├── README.md ├── electricsql-quill ├── .env ├── .eslintrc.cjs ├── .gitignore ├── .prettierignore ├── LICENSE ├── README.md ├── db │ └── migrations │ │ └── 01-create_docs.sql ├── index.html ├── package-lock.json ├── package.json ├── public │ ├── favicon.ico │ └── robots.txt ├── src │ ├── App.tsx │ ├── DocPicker.css │ ├── DocPicker.tsx │ ├── Loader.tsx │ ├── assets │ │ └── logo.svg │ ├── auth.ts │ ├── main.tsx │ ├── quill │ │ ├── ElectricQuill.tsx │ │ └── quill_wrapper.ts │ ├── units.ts │ └── vite-env.d.ts ├── tsconfig.json ├── tsconfig.node.json └── vite.config.ts ├── replicache-quill ├── .gitignore ├── LICENSE ├── README.md ├── client │ ├── .eslintignore │ ├── index.html │ ├── package.json │ ├── public │ │ └── replicache-logo-96.png │ ├── src │ │ ├── assert.ts │ │ ├── index.ts │ │ ├── quill_wrapper.ts │ │ ├── space.ts │ │ └── vite-env.d.ts │ ├── tsconfig.json │ ├── tsconfig.node.json │ └── vite.config.ts ├── package-lock.json ├── package.json ├── render.yaml ├── server │ ├── .eslintignore │ ├── endpoints │ │ ├── handle-poke.ts │ │ ├── handle-request.ts │ │ ├── handle-space.ts │ │ ├── replicache-pull.ts │ │ └── replicache-push.ts │ ├── nodemon.json │ ├── package.json │ ├── src │ │ ├── data.test.ts │ │ ├── data.ts │ │ ├── main.ts │ │ ├── pg.ts │ │ ├── pgconfig │ │ │ ├── pgconfig.ts │ │ │ ├── pgmem.ts │ │ │ └── postgres.ts │ │ ├── poke.ts │ │ ├── postgres-storage.ts │ │ ├── pull.ts │ │ ├── push.ts │ │ └── schema.ts │ └── tsconfig.json └── shared │ ├── .eslintignore │ ├── package.json │ ├── src │ ├── index.ts │ ├── mutators.ts │ └── rich_text.ts │ └── tsconfig.json ├── suggested-changes ├── .gitignore ├── .prettierignore ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── src │ ├── common │ │ ├── block_text.ts │ │ └── messages.ts │ ├── server │ │ ├── rich_text_server.ts │ │ └── server.ts │ └── site │ │ ├── index.html │ │ ├── main.ts │ │ ├── prosemirror_wrapper.ts │ │ ├── schema.ts │ │ └── suggestion.ts ├── tsconfig.json ├── tsconfig.server.json ├── tsconfig.webpack-config.json └── webpack.config.ts ├── triplit-quill ├── .gitignore ├── LICENSE ├── README.md ├── index.html ├── package-lock.json ├── package.json ├── src │ ├── main.ts │ ├── quill_wrapper.ts │ └── vite-env.d.ts ├── triplit │ └── schema.ts └── tsconfig.json ├── websocket-prosemirror-blocks ├── .gitignore ├── .prettierignore ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── src │ ├── common │ │ ├── block_text.ts │ │ └── messages.ts │ ├── server │ │ ├── rich_text_server.ts │ │ └── server.ts │ └── site │ │ ├── index.html │ │ ├── main.ts │ │ ├── prosemirror_wrapper.ts │ │ └── schema.ts ├── tsconfig.json ├── tsconfig.server.json ├── tsconfig.webpack-config.json └── webpack.config.ts ├── websocket-prosemirror-log ├── .gitignore ├── .prettierignore ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── src │ ├── common │ │ ├── messages.ts │ │ └── mutation.ts │ ├── server │ │ ├── rich_text_server.ts │ │ └── server.ts │ └── site │ │ ├── index.html │ │ ├── main.ts │ │ ├── prosemirror_wrapper.ts │ │ └── web_socket_client.ts ├── tsconfig.json ├── tsconfig.server.json ├── tsconfig.webpack-config.json └── webpack.config.ts └── websocket-quill ├── .gitignore ├── .prettierignore ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── src ├── common │ └── messages.ts ├── server │ ├── rich_text_server.ts │ └── server.ts └── site │ ├── index.html │ ├── main.ts │ └── quill_wrapper.ts ├── tsconfig.json ├── tsconfig.server.json ├── tsconfig.webpack-config.json └── webpack.config.ts /README.md: -------------------------------------------------------------------------------- 1 | # list-positions Demos 2 | 3 | Demos using the [list-positions](https://github.com/mweidner037/list-positions#readme) library and its companion library ([@list-positions/formatting](https://github.com/mweidner037/list-positions-formatting#readme).) 4 | 5 | Note: These demos are prototypes. I am working on "bindings" for specific rich-text editors, which aim to be more thoroughly tested and to support additional features (e.g., embedded media). 6 | 7 | - [`websocket-quill/`](./websocket-quill#readme): Basic collaborative rich-text editor using a WebSocket server and Quill. 8 | - [`websocket-prosemirror-log/`](./websocket-prosemirror-log#readme): Basic collaborative rich-text editor using a WebSocket server and ProseMirror, with support for arbitrary schemas, using a log of mutations. 9 | - [`websocket-prosemirror-blocks/`](./websocket-prosemirror-blocks#readme): Basic collaborative rich-text editor using a WebSocket server and ProseMirror, with a simple block-based schema. 10 | - [`triplit-quill/`](./triplit-quill#readme): Collaborative rich-text editor that synchronizes using the [Triplit](https://www.triplit.dev/) fullstack database. 11 | - [`replicache-quill/`](./replicache-quill#readme): Collaborative rich-text editor that synchronizes using the [Replicache](https://replicache.dev/) client-side sync framework. 12 | - [`electricsql-quill/`](./electricsql-quill#readme): Collaborative rich-text editor that synchronizes using the [ElectricSQL](https://electric-sql.com/) database sync service. 13 | - [`suggested-changes/`](./suggested-changes/#readme): Extension of `websocket-prosemirror-blocks` that adds "Suggested Changes". 14 | -------------------------------------------------------------------------------- /electricsql-quill/.env: -------------------------------------------------------------------------------- 1 | ELECTRIC_SERVICE=http://localhost:5133 2 | ELECTRIC_PG_PROXY_PORT=65432 -------------------------------------------------------------------------------- /electricsql-quill/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { browser: true, es2020: true }, 4 | extends: [ 5 | "eslint:recommended", 6 | "plugin:@typescript-eslint/recommended", 7 | "plugin:react-hooks/recommended", 8 | ], 9 | ignorePatterns: ["dist", ".eslintrc.cjs", "src/generated"], 10 | parser: "@typescript-eslint/parser", 11 | plugins: ["react-refresh"], 12 | rules: { 13 | "react-refresh/only-export-components": [ 14 | "warn", 15 | { allowConstantExport: true }, 16 | ], 17 | }, 18 | }; 19 | -------------------------------------------------------------------------------- /electricsql-quill/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | 26 | # Wasm 27 | public/wa-sqlite-async.wasm 28 | 29 | # Env files 30 | .env.local 31 | .env.*.local 32 | 33 | # Generated code 34 | src/generated 35 | -------------------------------------------------------------------------------- /electricsql-quill/.prettierignore: -------------------------------------------------------------------------------- 1 | src/generated 2 | -------------------------------------------------------------------------------- /electricsql-quill/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2024 Matthew Weidner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /electricsql-quill/README.md: -------------------------------------------------------------------------------- 1 | # ElectricSQL-Quill 2 | 3 | Basic collaborative rich-text editor using [list-positions](https://github.com/mweidner037/list-positions#readme) and [@list-positions/formatting](https://github.com/mweidner037/list-positions-formatting#readme), the [ElectricSQL](https://electric-sql.com/) database sync service, and [Quill](https://quilljs.com/). 4 | 5 | This is a web application using ElectricSQL in the browser with [wa-sqlite](https://electric-sql.com/docs/integrations/drivers/web/wa-sqlite). 6 | The editor state is stored in a SQL database with three tables: 7 | 8 | - `bunches` for list-positions's [BunchMeta](https://github.com/mweidner037/list-positions#managing-metadata). 9 | - `char_entries` for the characters. Each character gets its own row. 10 | - `formatting_marks` for the formatting marks. 11 | 12 | See the schema in [`db/migrations/01-create_docs.ts`](./db/migrations/01-create_docs.sql). 13 | 14 | Local updates are synced to the local database. When any table changes, a [live query](https://electric-sql.com/docs/usage/data-access/queries#live-queries) in `src/quill/ElectricQuill.tsx` updates the Quill state. Since subscriptions are not incremental (they always return the whole state), we diff against the previous state to figure out what changed. 15 | 16 | ## Pre-reqs 17 | 18 | The instructions below are unchanged from the [ElectricSQL Quick Start](https://electric-sql.com/docs/quickstart) (specifically, `npx create-electric-app@latest my-app --template react`). 19 | 20 | You need [NodeJS >= 16.11 and Docker Compose v2](https://electric-sql.com/docs/usage/installation/prereqs). 21 | 22 | ## Install 23 | 24 | Install the dependencies: 25 | 26 | ```sh 27 | npm install 28 | ``` 29 | 30 | ## Setup 31 | 32 | Start Postgres and Electric using Docker (see [running the examples](https://electric-sql.com/docs/examples/notes/running) for more options): 33 | 34 | ```shell 35 | npm run backend:up 36 | # Or `npm run backend:start` to foreground 37 | ``` 38 | 39 | Note that, if useful, you can connect to Postgres using: 40 | 41 | ```shell 42 | npm run db:psql 43 | ``` 44 | 45 | Setup your [database schema](https://electric-sql.com/docs/usage/data-modelling): 46 | 47 | ```shell 48 | npm run db:migrate 49 | ``` 50 | 51 | Generate your [type-safe client](https://electric-sql.com/docs/usage/data-access/client): 52 | 53 | ```shell 54 | npm run client:generate 55 | # or `npm run client:watch`` to re-generate whenever the DB schema changes 56 | ``` 57 | 58 | ## Run 59 | 60 | Start your app: 61 | 62 | ```sh 63 | npm run dev 64 | ``` 65 | 66 | 67 | 68 | Open [localhost:5173](http://localhost:5173) in your web browser. 69 | 70 | ## Develop 71 | 72 | This template contains the basic Electric code which you can adapt to your use case. For more information see the: 73 | 74 | - [Documentation](https://electric-sql.com/docs) 75 | - [Quickstart](https://electric-sql.com/docs/quickstart) 76 | - [Usage guide](https://electric-sql.com/docs/usage) 77 | 78 | If you need help [let ElectricSQL know on Discord](https://discord.electric-sql.com). 79 | -------------------------------------------------------------------------------- /electricsql-quill/db/migrations/01-create_docs.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE docs ( 2 | id UUID PRIMARY KEY, 3 | docName TEXT NOT NULL 4 | ); 5 | 6 | ALTER TABLE docs ENABLE ELECTRIC; 7 | 8 | -- Rich-text tables for the docs. 9 | 10 | CREATE TABLE bunches ( 11 | id TEXT PRIMARY KEY, 12 | -- Another bunchId or "ROOT". 13 | parent_id TEXT NOT NULL, 14 | the_offset INTEGER NOT NULL, 15 | doc_id UUID NOT NULL REFERENCES docs(id) ON DELETE CASCADE 16 | ); 17 | 18 | ALTER TABLE bunches ENABLE ELECTRIC; 19 | 20 | -- To allow merging concurrent deletions within the same bunch, 21 | -- we unfortunately need to store each (Position, char) pair as 22 | -- its own row, instead of as fields within the bunch. 23 | CREATE TABLE char_entries ( 24 | -- String encoding of the Position, used since we need a primary key 25 | -- but don't want to waste space on a UUID. 26 | pos TEXT PRIMARY KEY, 27 | -- Electric does not support CHAR(1), so use TEXT instead. 28 | char TEXT NOT NULL, 29 | -- Store doc IDs so we can delete cascade. 30 | doc_id UUID NOT NULL REFERENCES docs(id) ON DELETE CASCADE 31 | ); 32 | 33 | ALTER TABLE char_entries ENABLE ELECTRIC; 34 | 35 | -- Add-only log of TimestampMarks from @list-positions/formatting. 36 | CREATE TABLE formatting_marks ( 37 | -- String encoding of (creatorID, timestamp), used since we need a primary key 38 | -- but don't want to waste space on a UUID. 39 | id TEXT PRIMARY KEY, 40 | start_pos TEXT NOT NULL, 41 | start_before BOOLEAN NOT NULL, 42 | end_pos TEXT NOT NULL, 43 | end_before BOOLEAN NOT NULL, 44 | the_key TEXT NOT NULL, 45 | -- JSON encoded. 46 | the_value TEXT NOT NULL, 47 | -- Store doc IDs so we can delete cascade. 48 | doc_id UUID NOT NULL REFERENCES docs(id) ON DELETE CASCADE 49 | ); 50 | 51 | ALTER TABLE formatting_marks ENABLE ELECTRIC; -------------------------------------------------------------------------------- /electricsql-quill/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 17 | ElectricSQL-Quill 18 | 19 | 20 |
21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /electricsql-quill/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "electricsql", 3 | "version": "0.1.0", 4 | "author": "ElectricSQL", 5 | "type": "module", 6 | "scripts": { 7 | "backend:start": "npx electric-sql start --with-postgres", 8 | "backend:stop": "npx electric-sql stop", 9 | "backend:up": "npx electric-sql start --with-postgres --detach", 10 | "backend:down": "npx electric-sql stop --remove", 11 | "client:generate": "npx electric-sql generate", 12 | "client:watch": "npx electric-sql generate --watch", 13 | "db:migrate": "npx electric-sql with-config \"npx pg-migrations apply --database {{ELECTRIC_PROXY}} --directory ./db/migrations\"", 14 | "db:psql": "npx electric-sql psql", 15 | "electric:start": "npx electric-sql start", 16 | "dev": "vite", 17 | "build": "tsc && vite build", 18 | "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", 19 | "preview": "vite preview" 20 | }, 21 | "dependencies": { 22 | "electric-sql": "^0.12.0", 23 | "@list-positions/formatting": "^1.0.0", 24 | "list-positions": "^1.0.0", 25 | "quill": "^2.0.2", 26 | "react": "^18.2.0", 27 | "react-dom": "^18.2.0", 28 | "wa-sqlite": "github:rhashimoto/wa-sqlite#semver:^0.9.8" 29 | }, 30 | "devDependencies": { 31 | "@databases/pg-migrations": "^5.0.3", 32 | "@types/react": "^18.2.57", 33 | "@types/react-dom": "^18.2.19", 34 | "@typescript-eslint/eslint-plugin": "^6.21.0", 35 | "@typescript-eslint/parser": "^6.21.0", 36 | "@vitejs/plugin-react": "^4.2.1", 37 | "eslint": "^8.56.0", 38 | "eslint-plugin-react-hooks": "^4.6.0", 39 | "eslint-plugin-react-refresh": "^0.4.5", 40 | "typescript": "^5.3.3", 41 | "vite": "^5.1.4" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /electricsql-quill/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mweidner037/list-positions-demos/135f838515968d22758b10f9977078e90cbc50ac/electricsql-quill/public/favicon.ico -------------------------------------------------------------------------------- /electricsql-quill/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /electricsql-quill/src/App.tsx: -------------------------------------------------------------------------------- 1 | import { Loader } from "./Loader"; 2 | import { DocPicker } from "./DocPicker"; 3 | 4 | export default function App() { 5 | return ( 6 | 7 | 8 | 9 | ); 10 | } 11 | -------------------------------------------------------------------------------- /electricsql-quill/src/DocPicker.css: -------------------------------------------------------------------------------- 1 | .Picker { 2 | text-align: center; 3 | } 4 | 5 | .Picker-logo { 6 | height: min(160px, 30vmin); 7 | pointer-events: none; 8 | margin-top: min(30px, 5vmin); 9 | margin-bottom: min(30px, 5vmin); 10 | } 11 | 12 | .Picker-header { 13 | background-color: #1c1e20; 14 | min-height: 100vh; 15 | display: flex; 16 | flex-direction: column; 17 | align-items: top; 18 | justify-content: top; 19 | font-size: calc(10px + 2vmin); 20 | color: white; 21 | } 22 | 23 | .controls { 24 | margin-bottom: 1.5rem; 25 | } 26 | 27 | .button { 28 | display: inline-block; 29 | line-height: 1.3; 30 | text-align: center; 31 | text-decoration: none; 32 | vertical-align: middle; 33 | cursor: pointer; 34 | user-select: none; 35 | width: calc(15vw + 100px); 36 | margin-right: 0.5rem !important; 37 | margin-left: 0.5rem !important; 38 | border-radius: 32px; 39 | text-shadow: 2px 6px 20px rgba(0, 0, 0, 0.4); 40 | box-shadow: rgba(0, 0, 0, 0.5) 1px 2px 8px 0px; 41 | background: #1e2123; 42 | border: 2px solid #229089; 43 | color: #f9fdff; 44 | font-size: 16px; 45 | font-weight: 500; 46 | padding: 10px 18px; 47 | } 48 | 49 | .docP { 50 | display: block; 51 | line-height: 1.3; 52 | text-align: center; 53 | vertical-align: middle; 54 | width: calc(30vw - 1.5rem + 200px); 55 | margin-right: auto; 56 | margin-left: auto; 57 | border-radius: 32px; 58 | border: 1.5px solid #bbb; 59 | box-shadow: rgba(0, 0, 0, 0.3) 1px 2px 8px 0px; 60 | color: #f9fdff; 61 | font-size: 13px; 62 | padding: 10px 18px; 63 | } 64 | -------------------------------------------------------------------------------- /electricsql-quill/src/DocPicker.tsx: -------------------------------------------------------------------------------- 1 | import { useLiveQuery } from "electric-sql/react"; 2 | import { genUUID } from "electric-sql/util"; 3 | import { useState } from "react"; 4 | 5 | import { useElectric } from "./Loader"; 6 | import { Docs as Doc } from "./generated/client"; 7 | 8 | import logo from "./assets/logo.svg"; 9 | import "./DocPicker.css"; 10 | import { ElectricQuill } from "./quill/ElectricQuill"; 11 | 12 | export function DocPicker() { 13 | const [pickedId, setPickedId] = useState(); 14 | 15 | if (pickedId) { 16 | return ; 17 | } else { 18 | return ( 19 | { 21 | setPickedId(pickedId); 22 | }} 23 | /> 24 | ); 25 | } 26 | } 27 | 28 | function NotYetPicked({ onPick }: { onPick: (docId: string) => void }) { 29 | const { db } = useElectric()!; 30 | 31 | const { results } = useLiveQuery( 32 | db.docs.liveMany({ orderBy: { docname: "asc" } }) 33 | ); 34 | 35 | const addDoc = async () => { 36 | const id = genUUID(); 37 | await db.docs.create({ 38 | data: { 39 | id, 40 | docname: new Date().toISOString(), 41 | }, 42 | }); 43 | }; 44 | 45 | const docs: Doc[] = results ?? []; 46 | 47 | return ( 48 |
49 |
50 | logo 51 |
52 |
53 | 56 |
57 | {docs.map((doc) => ( 58 |

onPick(doc.id)}> 59 | {doc.docname} 60 |

61 | ))} 62 |
63 |
64 |
65 | ); 66 | } 67 | -------------------------------------------------------------------------------- /electricsql-quill/src/Loader.tsx: -------------------------------------------------------------------------------- 1 | import { ReactNode, useEffect, useState } from "react"; 2 | 3 | import { makeElectricContext } from "electric-sql/react"; 4 | import { uniqueTabId } from "electric-sql/util"; 5 | import { LIB_VERSION } from "electric-sql/version"; 6 | import { ElectricDatabase, electrify } from "electric-sql/wa-sqlite"; 7 | 8 | import { authToken } from "./auth"; 9 | import { Electric, schema } from "./generated/client"; 10 | 11 | const { ElectricProvider, useElectric } = makeElectricContext(); 12 | 13 | // eslint-disable-next-line react-refresh/only-export-components 14 | export { useElectric }; 15 | 16 | export const Loader = ({ children }: { children: ReactNode }) => { 17 | const [electric, setElectric] = useState(); 18 | 19 | useEffect(() => { 20 | let isMounted = true; 21 | 22 | const init = async () => { 23 | const config = { 24 | debug: import.meta.env.DEV, 25 | url: import.meta.env.ELECTRIC_SERVICE, 26 | }; 27 | 28 | const { tabId } = uniqueTabId(); 29 | const scopedDbName = `basic-${LIB_VERSION}-${tabId}.db`; 30 | 31 | const conn = await ElectricDatabase.init(scopedDbName); 32 | const electric = await electrify(conn, schema, config); 33 | await electric.connect(authToken()); 34 | 35 | // Establish sync with the remote DB using shapes. 36 | void electric.db.docs.sync({ 37 | include: { 38 | bunches: true, 39 | char_entries: true, 40 | formatting_marks: true, 41 | }, 42 | }); 43 | 44 | if (!isMounted) { 45 | return; 46 | } 47 | 48 | setElectric(electric); 49 | }; 50 | 51 | init(); 52 | 53 | return () => { 54 | isMounted = false; 55 | }; 56 | }, []); 57 | 58 | const [connected, setConnected] = useState(true); 59 | 60 | if (electric === undefined) { 61 | return null; 62 | } 63 | 64 | return ( 65 | <> 66 |
67 | {/* Connected checkbox, for testing concurrency. */} 68 | { 73 | if (e.currentTarget.checked) { 74 | electric.connect(authToken()); 75 | } else electric.disconnect(); 76 | setConnected(e.currentTarget.checked); 77 | }} 78 | /> 79 | 80 |
81 |
82 | {children} 83 | 84 | ); 85 | }; 86 | -------------------------------------------------------------------------------- /electricsql-quill/src/assets/logo.svg: -------------------------------------------------------------------------------- 1 | 5 | 8 | 11 | 12 | -------------------------------------------------------------------------------- /electricsql-quill/src/auth.ts: -------------------------------------------------------------------------------- 1 | import { insecureAuthToken } from 'electric-sql/auth' 2 | import { genUUID } from 'electric-sql/util' 3 | 4 | // Generate an insecure authentication JWT. 5 | // See https://electric-sql.com/docs/usage/auth for more details. 6 | export const authToken = () => { 7 | const subKey = '__electric_sub' 8 | let sub = window.sessionStorage.getItem(subKey) 9 | if (!sub) { 10 | // This is just a demo. In a real app, the user ID would 11 | // usually come from somewhere else :) 12 | sub = genUUID() 13 | window.sessionStorage.setItem(subKey, sub) 14 | } 15 | const claims = { sub } 16 | return insecureAuthToken(claims) 17 | } 18 | -------------------------------------------------------------------------------- /electricsql-quill/src/main.tsx: -------------------------------------------------------------------------------- 1 | import ReactDOM from "react-dom/client"; 2 | import App from "./App"; 3 | 4 | ReactDOM.createRoot(document.getElementById("root")!).render(); 5 | -------------------------------------------------------------------------------- /electricsql-quill/src/quill/ElectricQuill.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect, useRef } from "react"; 2 | 3 | import { useLiveQuery } from "electric-sql/react"; 4 | import { TimestampMark } from "@list-positions/formatting"; 5 | import { BunchMeta, Position, expandPositions } from "list-positions"; 6 | import { useElectric } from "../Loader"; 7 | import { QuillWrapper, WrapperOp } from "./quill_wrapper"; 8 | 9 | // TODO: Fix Quill double-toolbar in React strict mode. 10 | // For now we just disable strict mode. 11 | // See https://github.com/zenoamaro/react-quill/issues/784 12 | 13 | /** 14 | * The state of a long-lived "instance" of the ElectricQuill component, 15 | * associated to a particular Quill instance. 16 | */ 17 | type InstanceState = { 18 | wrapper: QuillWrapper; 19 | curBunchIDs: Set; 20 | curCharIDs: Set; 21 | curMarkIDs: Set; 22 | }; 23 | 24 | export function ElectricQuill({ 25 | docId, 26 | style, 27 | }: { 28 | docId: string; 29 | style?: React.CSSProperties; 30 | }) { 31 | const { db } = useElectric()!; 32 | 33 | const instanceStateRef = useRef(null); 34 | const quillRef = useRef(null); 35 | useEffect(() => { 36 | const wrapper = new QuillWrapper( 37 | quillRef.current!, 38 | onLocalOps, 39 | // Start with minimal initial state; existing db state loaded 40 | // in by queries below, analogous to new edits. 41 | QuillWrapper.makeInitialState() 42 | ); 43 | instanceStateRef.current = { 44 | wrapper, 45 | curBunchIDs: new Set(), 46 | curCharIDs: new Set(), 47 | curMarkIDs: new Set(), 48 | }; 49 | 50 | /** 51 | * Note: I use a strategy that describes the Quill state "transparently" 52 | * in the DB - e.g., there are rows corresponding to individual chars. 53 | * (Though it is not yet practical to query the text in order.) 54 | * 55 | * In principle, one could instead store the Quill state "opaquely", 56 | * by just storing the WrapperOps in an append-only log. 57 | * But one goal of list-positions is to allow transparent storage & updates, 58 | * instead of storing a CRDT library's opaque update blobs, 59 | * so that is what I demo here. 60 | */ 61 | 62 | // Send local ops to the DB. 63 | async function onLocalOps(ops: WrapperOp[]) { 64 | // Encoded Positions to delete. 65 | // We batch these into a single DB op at the end, to prevent 66 | // gradual backspacing when a collaborator deletes a large selection. 67 | const toDelete: string[] = []; 68 | 69 | for (const op of ops) { 70 | switch (op.type) { 71 | case "set": { 72 | const poss = expandPositions(op.startPos, op.chars.length); 73 | await db.char_entries.createMany({ 74 | data: poss.map((pos, i) => ({ 75 | pos: encodePos(pos), 76 | char: op.chars[i], 77 | doc_id: docId, 78 | })), 79 | }); 80 | break; 81 | } 82 | case "delete": 83 | for (const pos of expandPositions(op.startPos, op.count ?? 1)) { 84 | toDelete.push(encodePos(pos)); 85 | } 86 | break; 87 | case "metas": 88 | // Note: It is important that receivers apply all of these BunchMetas together, 89 | // in case they have internal dependencies. 90 | await db.bunches.createMany({ 91 | data: op.metas.map((meta) => ({ 92 | id: meta.bunchID, 93 | parent_id: meta.parentID, 94 | the_offset: meta.offset, 95 | doc_id: docId, 96 | })), 97 | }); 98 | break; 99 | case "marks": 100 | console.log(op.marks); 101 | await db.formatting_marks.createMany({ 102 | data: op.marks.map((mark) => ({ 103 | id: encodeMarkID(mark), 104 | start_pos: encodePos(mark.start.pos), 105 | start_before: mark.start.before, 106 | end_pos: encodePos(mark.end.pos), 107 | end_before: mark.end.before, 108 | the_key: mark.key, 109 | the_value: JSON.stringify(mark.value), 110 | doc_id: docId, 111 | })), 112 | }); 113 | break; 114 | } 115 | } 116 | 117 | // Batched delete. 118 | if (toDelete.length !== 0) { 119 | await db.char_entries.deleteMany({ 120 | where: { OR: toDelete.map((pos) => ({ pos })) }, 121 | }); 122 | } 123 | } 124 | 125 | return () => wrapper.destroy(); 126 | // eslint-disable-next-line react-hooks/exhaustive-deps 127 | }, [docId]); 128 | 129 | // Reflect DB ops in Quill. 130 | // Since queries are not incremental, we diff against the previous state 131 | // and process changed (inserted/deleted) ids. 132 | // Note that this will also capture local changes; QuillWrapper will ignore 133 | // those as redundant. 134 | const { results: bunches } = useLiveQuery( 135 | db.bunches.liveMany({ where: { doc_id: docId } }) 136 | ); 137 | const { results: charEntries } = useLiveQuery( 138 | db.char_entries.liveMany({ where: { doc_id: docId } }) 139 | ); 140 | const { results: marks } = useLiveQuery( 141 | db.formatting_marks.liveMany({ where: { doc_id: docId } }) 142 | ); 143 | 144 | if (instanceStateRef.current !== null) { 145 | const { wrapper, curBunchIDs, curCharIDs, curMarkIDs } = 146 | instanceStateRef.current; 147 | const newOps: WrapperOp[] = []; 148 | 149 | if (bunches) { 150 | const newBunchMetas: BunchMeta[] = []; 151 | for (const bunch of bunches) { 152 | if (!curBunchIDs.has(bunch.id)) { 153 | curBunchIDs.add(bunch.id); 154 | newBunchMetas.push({ 155 | bunchID: bunch.id, 156 | parentID: bunch.parent_id, 157 | offset: bunch.the_offset, 158 | }); 159 | } 160 | } 161 | if (newBunchMetas.length !== 0) { 162 | newOps.push({ type: "metas", metas: newBunchMetas }); 163 | } 164 | } 165 | 166 | if (charEntries) { 167 | const unseenCharIDs = new Set(curCharIDs); 168 | for (const charEntry of charEntries) { 169 | if (!curCharIDs.has(charEntry.pos)) { 170 | curCharIDs.add(charEntry.pos); 171 | newOps.push({ 172 | type: "set", 173 | startPos: decodePos(charEntry.pos), 174 | chars: charEntry.char, 175 | }); 176 | } 177 | unseenCharIDs.delete(charEntry.pos); 178 | } 179 | // unseenCharIDs is the diff in the other direction, used to find deleted rows. 180 | for (const unseenCharID of unseenCharIDs) { 181 | newOps.push({ 182 | type: "delete", 183 | startPos: decodePos(unseenCharID), 184 | }); 185 | curCharIDs.delete(unseenCharID); 186 | } 187 | } 188 | 189 | if (marks) { 190 | for (const mark of marks) { 191 | if (!curMarkIDs.has(mark.id)) { 192 | curMarkIDs.add(mark.id); 193 | newOps.push({ 194 | type: "marks", 195 | marks: [ 196 | { 197 | ...decodeMarkID(mark.id), 198 | start: { 199 | pos: decodePos(mark.start_pos), 200 | before: mark.start_before, 201 | }, 202 | end: { 203 | pos: decodePos(mark.end_pos), 204 | before: mark.end_before, 205 | }, 206 | key: mark.the_key, 207 | value: JSON.parse(mark.the_value), 208 | }, 209 | ], 210 | }); 211 | } 212 | } 213 | } 214 | 215 | if (newOps.length !== 0) wrapper.applyOps(newOps); 216 | } 217 | 218 | return
; 219 | } 220 | 221 | function encodePos(pos: Position): string { 222 | return `${pos.bunchID}_${pos.innerIndex.toString(36)}`; 223 | } 224 | 225 | function decodePos(encoded: string): Position { 226 | const sep = encoded.lastIndexOf("_"); 227 | const bunchID = encoded.slice(0, sep); 228 | const innerIndex = Number.parseInt(encoded.slice(sep + 1), 36); 229 | return { bunchID, innerIndex }; 230 | } 231 | 232 | function encodeMarkID(mark: TimestampMark): string { 233 | return `${mark.creatorID}_${mark.timestamp.toString(36)}`; 234 | } 235 | 236 | function decodeMarkID(encoded: string): { 237 | creatorID: string; 238 | timestamp: number; 239 | } { 240 | const sep = encoded.lastIndexOf("_"); 241 | const creatorID = encoded.slice(0, sep); 242 | const timestamp = Number.parseInt(encoded.slice(sep + 1), 36); 243 | return { creatorID, timestamp }; 244 | } 245 | -------------------------------------------------------------------------------- /electricsql-quill/src/units.ts: -------------------------------------------------------------------------------- 1 | export type Unit = "kg" | "g" | "mg" | "L" | "mL" | "ct"; 2 | export const AllUnits: Unit[] = ["kg", "g", "mg", "L", "mL", "ct"]; 3 | export const DEFAULT_UNIT: Unit = "g"; 4 | -------------------------------------------------------------------------------- /electricsql-quill/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /electricsql-quill/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "useDefineForClassFields": true, 5 | "lib": ["ES2020", "DOM", "DOM.Iterable"], 6 | "module": "ESNext", 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "resolveJsonModule": true, 13 | "isolatedModules": true, 14 | "noEmit": true, 15 | "jsx": "react-jsx", 16 | 17 | /* Linting */ 18 | "strict": true, 19 | "noUnusedLocals": true, 20 | "noUnusedParameters": true, 21 | "noFallthroughCasesInSwitch": true 22 | }, 23 | "include": ["src"], 24 | "references": [{ "path": "./tsconfig.node.json" }] 25 | } 26 | -------------------------------------------------------------------------------- /electricsql-quill/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | "skipLibCheck": true, 5 | "module": "ESNext", 6 | "moduleResolution": "bundler", 7 | "allowSyntheticDefaultImports": true 8 | }, 9 | "include": ["vite.config.ts"] 10 | } 11 | -------------------------------------------------------------------------------- /electricsql-quill/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import react from '@vitejs/plugin-react' 3 | 4 | // https://vitejs.dev/config/ 5 | export default defineConfig({ 6 | plugins: [react()], 7 | envPrefix: 'ELECTRIC_', 8 | optimizeDeps: { 9 | exclude: ['wa-sqlite'], 10 | }, 11 | }) 12 | -------------------------------------------------------------------------------- /replicache-quill/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | dist 84 | 85 | # Gatsby files 86 | .cache/ 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 88 | # https://nextjs.org/blog/next-9-1#public-directory-support 89 | # public 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless/ 96 | 97 | # FuseBox cache 98 | .fusebox/ 99 | 100 | # DynamoDB Local files 101 | .dynamodb/ 102 | 103 | # TernJS port file 104 | .tern-port 105 | -------------------------------------------------------------------------------- /replicache-quill/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2024 Matthew Weidner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /replicache-quill/README.md: -------------------------------------------------------------------------------- 1 | # Replicache-Quill 2 | 3 | Basic collaborative rich-text editor using [list-positions](https://github.com/mweidner037/list-positions#readme) and [@list-positions/formatting](https://github.com/mweidner037/list-positions-formatting#readme), the [Replicache](https://replicache.dev/) client-side sync framework, and [Quill](https://quilljs.com/). 4 | 5 | The editor state is stored in Replicache under two prefixes: 6 | 7 | - `bunch/` for the `List` state, grouped by bunch. Each entry corresponds to a [bunch](https://github.com/mweidner037/list-positions#bunches) from list-positions. It stores the bunch's [BunchMeta](https://github.com/mweidner037/list-positions#managing-metadata) fields, plus its current values (chars) as an object `{ [innerIndex: number]: string }`. 8 | - `mark/` for the formatting marks. Each entry stores a [TimestampMark](https://github.com/mweidner037/list-positions-formatting#class-timestampformatting) from @list-positions/formatting, keyed by an arbitrary unique ID. 9 | 10 | Replicache mutators correspond to the basic rich-text operations: 11 | 12 | - `createBunch` to create a new bunch with its BunchMeta. 13 | - `setValues` to set some Position-value pairs within a bunch. 14 | - `deleteValues` to delete some Position-value pairs within a bunch. 15 | - `addMarks` to add a formatting mark. 16 | 17 | See `shared/src/rich_text.ts` and `shared/src/mutators.ts`. 18 | 19 | The instructions below are from Replicache's [todo-wc](https://github.com/rocicorp/todo-wc) example, which we used as a template. 20 | 21 | ## 1. Setup 22 | 23 | #### Get your Replicache License Key 24 | 25 | ```bash 26 | $ npx replicache get-license 27 | ``` 28 | 29 | #### Set your `VITE_REPLICACHE_LICENSE_KEY` environment variable 30 | 31 | ```bash 32 | $ export VITE_REPLICACHE_LICENSE_KEY="" 33 | ``` 34 | 35 | (Or put it in `client/.env`, which is gitignored.) 36 | 37 | #### Install and Build 38 | 39 | ```bash 40 | $ npm install; npm run build; 41 | ``` 42 | 43 | ## 2.Start frontend and backend watcher 44 | 45 | ```bash 46 | $ npm run watch --ws 47 | ``` 48 | 49 | ## Deploying to Render 50 | 51 | A render blueprint example is provided to deploy the application. 52 | 53 | Open the `render.yaml` file and add your license key 54 | 55 | ``` 56 | - key: VITE_REPLICACHE_LICENSE_KEY 57 | value: 58 | ``` 59 | 60 | Commit the changes and follow the direction on [Deploying to Render](https://doc.replicache.dev/deploy-render) 61 | /client 62 | /shared 63 | /server 64 | package.json 65 | -------------------------------------------------------------------------------- /replicache-quill/client/.eslintignore: -------------------------------------------------------------------------------- 1 | dist 2 | vite.config.ts 3 | -------------------------------------------------------------------------------- /replicache-quill/client/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Replicache-Quill 8 | 9 | 10 | 11 |
12 | Info and source code 16 |
17 |
18 | 19 | 20 |
21 |
22 |
23 | 24 | 25 | -------------------------------------------------------------------------------- /replicache-quill/client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "client", 3 | "private": true, 4 | "version": "0.1.0", 5 | "type": "module", 6 | "scripts": { 7 | "lint": "eslint --ext .ts,.tsx .", 8 | "dev": "vite", 9 | "check-types": "tsc --noEmit", 10 | "build": "tsc && vite build", 11 | "build:server": "cd ../server && npm run build", 12 | "preview": "vite preview", 13 | "format": "prettier --write './src/**/*.{js,jsx,json,ts,tsx,html,css,md}' '*.{cjs,js,jsx,json,ts,tsx,html,css,md}'", 14 | "check-format": "prettier --check './src/**/*.{js,jsx,json,ts,tsx,html,css,md}' '*.{cjs,js,jsx,json,ts,tsx,html,css,md}'", 15 | "clean": "rm -rf ./dist; mkdir -p ./dist", 16 | "prod": "cp -r ./dist/ ../server/dist/; cd ../server; npm run prod", 17 | "server": "cd ../server && npm run dev", 18 | "watch": "concurrently --kill-others 'npm run server' 'npm run check-types -- --watch --preserveWatchOutput' 'sleep 3; npm run dev'" 19 | }, 20 | "dependencies": { 21 | "@list-positions/formatting": "^1.0.1", 22 | "list-positions": "^1.0.0", 23 | "nanoid": "^5.0.7", 24 | "quill": "^2.0.2", 25 | "replicache": ">=14.0.3", 26 | "shared": "^0.1.0" 27 | }, 28 | "devDependencies": { 29 | "@rocicorp/prettier-config": "^0.1.1", 30 | "@rocicorp/eslint-config": "^0.1.2", 31 | "prettier": "^2.2.1", 32 | "typescript": "^5.4.5", 33 | "vite": "^3.0.7", 34 | "concurrently": "^7.4.0" 35 | }, 36 | "eslintConfig": { 37 | "extends": "@rocicorp/eslint-config" 38 | }, 39 | "prettier": "@rocicorp/prettier-config" 40 | } 41 | -------------------------------------------------------------------------------- /replicache-quill/client/public/replicache-logo-96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mweidner037/list-positions-demos/135f838515968d22758b10f9977078e90cbc50ac/replicache-quill/client/public/replicache-logo-96.png -------------------------------------------------------------------------------- /replicache-quill/client/src/assert.ts: -------------------------------------------------------------------------------- 1 | type Truthy = T extends null | undefined | false | '' | 0 ? never : T; 2 | 3 | export function assert( 4 | b: T, 5 | msg = 'Assertion failed', 6 | ): asserts b is Truthy { 7 | if (!b) { 8 | throw new Error(msg); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /replicache-quill/client/src/index.ts: -------------------------------------------------------------------------------- 1 | import {TimestampMark} from '@list-positions/formatting'; 2 | import { 3 | ExperimentalDiffOperationAdd, 4 | ExperimentalDiffOperationChange, 5 | Replicache, 6 | } from 'replicache'; 7 | import {Bunch, allBunches, allMarks, mutators} from 'shared'; 8 | import {QuillWrapper, WrapperOp} from './quill_wrapper'; 9 | import {createSpace, spaceExists} from './space'; 10 | 11 | async function init() { 12 | const {pathname} = window.location; 13 | 14 | if (pathname === '/' || pathname === '') { 15 | window.location.href = '/list/' + (await createSpace()); 16 | return; 17 | } 18 | 19 | // URL layout is "/list/" 20 | const paths = pathname.split('/'); 21 | const [, listDir, listID] = paths; 22 | if ( 23 | listDir !== 'list' || 24 | listID === undefined || 25 | !(await spaceExists(listID)) 26 | ) { 27 | window.location.href = '/'; 28 | return; 29 | } 30 | 31 | // See https://doc.replicache.dev/licensing for how to get a license key. 32 | const licenseKey = import.meta.env.VITE_REPLICACHE_LICENSE_KEY; 33 | if (!licenseKey) { 34 | throw new Error('Missing VITE_REPLICACHE_LICENSE_KEY'); 35 | } 36 | 37 | const r = new Replicache({ 38 | licenseKey, 39 | pushURL: `/api/replicache/push?spaceID=${listID}`, 40 | pullURL: `/api/replicache/pull?spaceID=${listID}`, 41 | name: listID, 42 | mutators, 43 | }); 44 | 45 | // Implements a Replicache poke using Server-Sent Events. 46 | // If a "poke" message is received, it will pull from the server. 47 | const ev = new EventSource(`/api/replicache/poke?spaceID=${listID}`, { 48 | withCredentials: true, 49 | }); 50 | ev.onmessage = async event => { 51 | if (event.data === 'poke') { 52 | await r.pull(); 53 | } 54 | }; 55 | 56 | // Load initial state from Replicache. 57 | 58 | const richText = QuillWrapper.newRichText(); 59 | await r.query(async tx => { 60 | const bunches = await allBunches(tx); 61 | // First need to load all metas together, to avoid dependency ordering concerns. 62 | richText.order.addMetas(bunches.map(bunch => bunch.meta)); 63 | // Now load all values. 64 | for (const bunch of bunches) { 65 | // TODO: In list-positions, provide method to set a whole bunch's values quickly. 66 | for (const [indexStr, char] of Object.entries(bunch.values)) { 67 | const innerIndex = Number.parseInt(indexStr); 68 | richText.text.set({bunchID: bunch.meta.bunchID, innerIndex}, char); 69 | } 70 | } 71 | 72 | // Load all marks. 73 | const marks = await allMarks(tx); 74 | richText.formatting.load(marks); 75 | }); 76 | 77 | const quillWrapper = new QuillWrapper(onLocalOps, richText); 78 | 79 | // Send future Quill changes to Replicache. 80 | // Use a queue to avoid reordered mutations (since onLocalOps is sync 81 | // but mutations are async). 82 | 83 | let localOpsQueue: WrapperOp[] = []; 84 | let sendingLocalOps = false; 85 | function onLocalOps(ops: WrapperOp[]) { 86 | localOpsQueue.push(...ops); 87 | if (!sendingLocalOps) void sendLocalOps(); 88 | } 89 | 90 | async function sendLocalOps() { 91 | sendingLocalOps = true; 92 | try { 93 | while (localOpsQueue.length !== 0) { 94 | const ops = localOpsQueue; 95 | localOpsQueue = []; 96 | for (const op of ops) { 97 | switch (op.type) { 98 | case 'meta': 99 | await r.mutate.createBunch(op.meta); 100 | break; 101 | case 'set': 102 | await r.mutate.setValues({ 103 | startPos: op.startPos, 104 | values: [...op.chars], 105 | }); 106 | break; 107 | case 'delete': 108 | await r.mutate.deleteValues({ 109 | startPos: op.startPos, 110 | count: op.count, 111 | }); 112 | break; 113 | case 'marks': 114 | await r.mutate.addMarks({marks: op.marks}); 115 | break; 116 | } 117 | } 118 | } 119 | } finally { 120 | sendingLocalOps = false; 121 | } 122 | } 123 | 124 | // Send future Replicache changes to Quill. 125 | 126 | r.experimentalWatch(diff => { 127 | const wrapperOps: WrapperOp[] = []; 128 | for (const diffOp of diff) { 129 | if (diffOp.key.startsWith('bunch/')) { 130 | switch (diffOp.op) { 131 | case 'add': { 132 | const op = diffOp as ExperimentalDiffOperationAdd; 133 | wrapperOps.push({ 134 | type: 'meta', 135 | meta: op.newValue.meta, 136 | }); 137 | for (const [indexStr, char] of Object.entries(op.newValue.values)) { 138 | const innerIndex = Number.parseInt(indexStr); 139 | wrapperOps.push({ 140 | type: 'set', 141 | startPos: {bunchID: op.newValue.meta.bunchID, innerIndex}, 142 | chars: char, 143 | }); 144 | } 145 | break; 146 | } 147 | case 'change': { 148 | const op = diffOp as ExperimentalDiffOperationChange; 149 | // Need to manually diff op.oldValue and op.newValue. 150 | // Luckily, bunches are usually small (10-100 chars?). 151 | 152 | // deletedKeys collects keys present in oldValue but not newValue. 153 | const deletedKeys = new Set(Object.keys(op.oldValue.values)); 154 | for (const [indexStr, newChar] of Object.entries( 155 | op.newValue.values, 156 | )) { 157 | const innerIndex = Number.parseInt(indexStr); 158 | const oldChar = op.oldValue.values[innerIndex]; 159 | if (newChar !== oldChar) { 160 | wrapperOps.push({ 161 | type: 'set', 162 | startPos: {bunchID: op.newValue.meta.bunchID, innerIndex}, 163 | chars: newChar, 164 | }); 165 | } 166 | deletedKeys.delete(indexStr); 167 | } 168 | for (const indexStr of deletedKeys) { 169 | wrapperOps.push({ 170 | type: 'delete', 171 | startPos: { 172 | bunchID: op.newValue.meta.bunchID, 173 | innerIndex: Number.parseFloat(indexStr), 174 | }, 175 | count: 1, 176 | }); 177 | } 178 | break; 179 | } 180 | default: 181 | console.error('Unexpected op on bunch key:', diffOp.op, diffOp.key); 182 | } 183 | } else if (diffOp.key.startsWith('mark/')) { 184 | switch (diffOp.op) { 185 | case 'add': { 186 | // ReadonlyJSONValue is supposed to express that the value is deep-readonly. 187 | // Because of https://github.com/microsoft/TypeScript/issues/15300 , though, 188 | // it doesn't work on JSON objects whose type is (or includes) an interface. 189 | const op = diffOp as unknown as ExperimentalDiffOperationAdd< 190 | string, 191 | TimestampMark 192 | >; 193 | wrapperOps.push({type: 'marks', marks: [op.newValue]}); 194 | break; 195 | } 196 | default: 197 | console.error('Unexpected op on mark key:', diffOp.op, diffOp.key); 198 | } 199 | } else { 200 | console.error('Unexpected key:', diffOp.key); 201 | } 202 | } 203 | 204 | quillWrapper.applyOps(wrapperOps); 205 | }); 206 | } 207 | await init(); 208 | -------------------------------------------------------------------------------- /replicache-quill/client/src/space.ts: -------------------------------------------------------------------------------- 1 | export async function spaceExists(spaceID: string): Promise { 2 | const spaceExistRes = await fetchJSON('spaceExists', spaceID); 3 | if ( 4 | spaceExistRes && 5 | typeof spaceExistRes === 'object' && 6 | typeof spaceExistRes.spaceExists === 'boolean' 7 | ) { 8 | return spaceExistRes.spaceExists; 9 | } 10 | throw new Error('Bad response from spaceExists'); 11 | } 12 | 13 | export async function createSpace(spaceID?: string): Promise { 14 | const createSpaceRes = await fetchJSON('createSpace', spaceID); 15 | if ( 16 | createSpaceRes && 17 | typeof createSpaceRes === 'object' && 18 | typeof createSpaceRes.spaceID === 'string' 19 | ) { 20 | return createSpaceRes.spaceID; 21 | } 22 | throw new Error('Bad response from createSpace'); 23 | } 24 | 25 | async function fetchJSON(apiName: string, spaceID: string | undefined) { 26 | const res = await fetch(`/api/replicache/${apiName}`, { 27 | method: 'POST', 28 | headers: { 29 | 'Content-Type': 'application/json', 30 | }, 31 | body: 32 | spaceID && 33 | JSON.stringify({ 34 | spaceID, 35 | }), 36 | }); 37 | return await res.json(); 38 | } 39 | -------------------------------------------------------------------------------- /replicache-quill/client/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /replicache-quill/client/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "useDefineForClassFields": true, 5 | "lib": ["DOM", "DOM.Iterable", "ESNext"], 6 | "allowJs": false, 7 | "skipLibCheck": true, 8 | "esModuleInterop": false, 9 | "allowSyntheticDefaultImports": false, 10 | "strict": true, 11 | "forceConsistentCasingInFileNames": true, 12 | "module": "ESNext", 13 | "moduleResolution": "Node", 14 | "resolveJsonModule": true, 15 | "isolatedModules": true, 16 | "noEmit": true 17 | }, 18 | "include": ["src"], 19 | "references": [{"path": "./tsconfig.node.json"}] 20 | } 21 | -------------------------------------------------------------------------------- /replicache-quill/client/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | "module": "ESNext", 5 | "moduleResolution": "Node", 6 | "allowSyntheticDefaultImports": false 7 | }, 8 | "include": ["vite.config.ts"] 9 | } 10 | -------------------------------------------------------------------------------- /replicache-quill/client/vite.config.ts: -------------------------------------------------------------------------------- 1 | import {defineConfig} from 'vite'; 2 | 3 | // https://vitejs.dev/config/ 4 | export default defineConfig({ 5 | plugins: [], 6 | build: { 7 | target: 'esnext', 8 | }, 9 | server: { 10 | proxy: { 11 | '/api': { 12 | target: 'http://127.0.0.1:8080', 13 | }, 14 | }, 15 | }, 16 | }); 17 | -------------------------------------------------------------------------------- /replicache-quill/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "todo-wc", 3 | "version": "0.1.0", 4 | "devDependencies": { 5 | "@rocicorp/eslint-config": "^0.1.2", 6 | "@rocicorp/prettier-config": "^0.1.1", 7 | "typescript": "^5.4.5" 8 | }, 9 | "scripts": { 10 | "format": "npm run format --ws", 11 | "check-format": "npm run check-format --ws", 12 | "lint": "npm run lint --ws", 13 | "build": "npm run build -ws --if-present", 14 | "check-types": "npm run check-types --ws" 15 | }, 16 | "type": "module", 17 | "eslintConfig": { 18 | "extends": "@rocicorp/eslint-config" 19 | }, 20 | "prettier": "@rocicorp/prettier-config", 21 | "engines": { 22 | "node": ">=16.15.0", 23 | "npm": ">=7.0.0" 24 | }, 25 | "workspaces": [ 26 | "client", 27 | "server", 28 | "shared" 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /replicache-quill/render.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | - type: web 3 | name: replicache-quickstarts-todo 4 | env: node 5 | region: oregon # optional (defaults to oregon) 6 | plan: starter # optional (defaults to starter) 7 | buildCommand: "npm install && npm run build" # optional (defaults to Dockerfile command) 8 | startCommand: "npm run prod --ws" 9 | numInstances: 1 # optional (defaults to 1) 10 | envVars: 11 | - key: VITE_REPLICACHE_LICENSE_KEY 12 | value: # change e.g. (VITE_REPLICACHE_LICENSE_KEY: 1234567890) 13 | - key: NODE_VERSION 14 | value: 16.15.1 15 | - key: DATABASE_URL 16 | fromDatabase: 17 | name: replicache-todo 18 | property: connectionString 19 | 20 | databases: 21 | - name: replicache-todo 22 | databaseName: db 23 | -------------------------------------------------------------------------------- /replicache-quill/server/.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | out 3 | tool 4 | bin 5 | .eslintrc.cjs 6 | dist 7 | lib 8 | env.d.ts 9 | vite.config.ts -------------------------------------------------------------------------------- /replicache-quill/server/endpoints/handle-poke.ts: -------------------------------------------------------------------------------- 1 | import type Express from 'express'; 2 | import {getPokeBackend} from '../src/poke.js'; 3 | 4 | export async function handlePoke( 5 | req: Express.Request, 6 | res: Express.Response, 7 | ): Promise { 8 | if (req.query.spaceID === undefined) { 9 | res.status(400).send('Missing spaceID'); 10 | return; 11 | } 12 | const {spaceID} = req.query; 13 | 14 | res.setHeader('Access-Control-Allow-Origin', '*'); 15 | res.setHeader('Content-Type', 'text/event-stream;charset=utf-8'); 16 | res.setHeader('Cache-Control', 'no-cache, no-transform'); 17 | res.setHeader('X-Accel-Buffering', 'no'); 18 | 19 | res.write(`id: ${Date.now()}\n`); 20 | res.write(`data: hello\n\n`); 21 | 22 | const pokeBackend = getPokeBackend(); 23 | 24 | const unlisten = pokeBackend.addListener(spaceID as string, () => { 25 | console.log(`Sending poke for space ${spaceID}`); 26 | res.write(`id: ${Date.now()}\n`); 27 | res.write(`data: poke\n\n`); 28 | }); 29 | 30 | setInterval(() => { 31 | res.write(`id: ${Date.now()}\n`); 32 | res.write(`data: beat\n\n`); 33 | }, 30 * 1000); 34 | 35 | res.on('close', () => { 36 | console.log('Closing poke connection'); 37 | unlisten(); 38 | }); 39 | } 40 | -------------------------------------------------------------------------------- /replicache-quill/server/endpoints/handle-request.ts: -------------------------------------------------------------------------------- 1 | import type {MutatorDefs} from 'replicache'; 2 | import {handlePull} from './replicache-pull.js'; 3 | import {handlePush} from './replicache-push.js'; 4 | import type Express from 'express'; 5 | import {handleCreateSpace, handleSpaceExist} from './handle-space.js'; 6 | 7 | export async function handleRequest( 8 | req: Express.Request, 9 | res: Express.Response, 10 | next: Express.NextFunction, 11 | mutators: M, 12 | ): Promise { 13 | if (req.query === undefined) { 14 | res.status(400).send('Missing query'); 15 | return; 16 | } 17 | 18 | const {op} = req.params; 19 | console.log(`Handling request ${JSON.stringify(req.body)}, op: ${op}`); 20 | 21 | switch (op) { 22 | case 'push': 23 | return await handlePush(req, res, next, mutators); 24 | case 'pull': 25 | return await handlePull(req, res, next); 26 | case 'createSpace': 27 | return await handleCreateSpace(req, res, next); 28 | case 'spaceExists': 29 | return await handleSpaceExist(req, res, next); 30 | } 31 | 32 | res.status(400).send({error: 'Invalid op'}); 33 | } 34 | -------------------------------------------------------------------------------- /replicache-quill/server/endpoints/handle-space.ts: -------------------------------------------------------------------------------- 1 | import {nanoid} from 'nanoid'; 2 | import type Express from 'express'; 3 | import { transact } from '../src/pg.js'; 4 | import { getCookie, createSpace } from "../src/data.js"; 5 | 6 | export async function handleCreateSpace( 7 | req: Express.Request, 8 | res: Express.Response, 9 | next: Express.NextFunction 10 | ): Promise { 11 | let spaceID = nanoid(6); 12 | if (req.body.spaceID) { 13 | spaceID = req.body.spaceID; 14 | } 15 | if (spaceID.length > 10) { 16 | next(Error(`SpaceID must be 10 characters or less`)); 17 | } 18 | try { 19 | await transact(async (executor) => { 20 | await createSpace(executor, spaceID); 21 | }); 22 | res.status(200).send({ spaceID }); 23 | } catch (e: any) { 24 | next(Error(`Failed to create space ${spaceID}`, e)); 25 | } 26 | } 27 | 28 | export async function handleSpaceExist( 29 | req: Express.Request, 30 | res: Express.Response, 31 | next: Express.NextFunction 32 | ): Promise { 33 | try { 34 | const cookie = await transact(async (executor) => { 35 | return await getCookie(executor, req.body.spaceID); 36 | }); 37 | const exists = cookie !== undefined; 38 | res.status(200).send({ spaceExists: exists }); 39 | } catch (e: any) { 40 | next(Error(`Failed to check space exists ${req.body.spaceID}`, e)); 41 | } 42 | } -------------------------------------------------------------------------------- /replicache-quill/server/endpoints/replicache-pull.ts: -------------------------------------------------------------------------------- 1 | import type Express from 'express'; 2 | import {pull} from '../src/pull.js'; 3 | 4 | export async function handlePull( 5 | req: Express.Request, 6 | res: Express.Response, 7 | next: Express.NextFunction 8 | ): Promise { 9 | if (req.query.spaceID === undefined) { 10 | res.status(400).json({ error: "spaceID is required" }); 11 | return; 12 | } 13 | const { spaceID } = req.query; 14 | try { 15 | const resp = await pull(spaceID as string, req.body); 16 | res.json(resp); 17 | } catch (e: any) { 18 | next(Error(e)); 19 | } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /replicache-quill/server/endpoints/replicache-push.ts: -------------------------------------------------------------------------------- 1 | import type {MutatorDefs} from 'replicache'; 2 | import {push} from '../src/push.js'; 3 | 4 | import type Express from 'express'; 5 | 6 | export async function handlePush( 7 | req: Express.Request, 8 | res: Express.Response, 9 | next: Express.NextFunction, 10 | mutators: M, 11 | ): Promise { 12 | if (req.query.spaceID === undefined) { 13 | res.status(400).send('Missing spaceID'); 14 | return; 15 | } 16 | const spaceID = req.query.spaceID.toString() as string; 17 | try { 18 | await push(spaceID, req.body, mutators); 19 | res.status(200).json({}); 20 | } catch (e: any) { 21 | next(Error(e)); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /replicache-quill/server/nodemon.json: -------------------------------------------------------------------------------- 1 | { 2 | "watch": ["src", "../shared"], 3 | "ext": "ts,json", 4 | "ignore": ["src/**/*.spec.ts"], 5 | "exec": "node --loader ts-node/esm --experimental-specifier-resolution=node ./src/main.ts" 6 | } 7 | -------------------------------------------------------------------------------- /replicache-quill/server/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "server", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "dotenv": "^16.0.1", 7 | "express": "^4.18.1", 8 | "@list-positions/formatting": "^1.0.1", 9 | "list-positions": "^1.0.0", 10 | "replicache": ">=14.0.3", 11 | "shared": "^0.1.0", 12 | "nanoid": "^4.0.0", 13 | "replicache-transaction": "^0.3.3" 14 | }, 15 | "devDependencies": { 16 | "shared": "^0.1.0", 17 | "@types/chai": "^4.3.0", 18 | "@types/mocha": "^9.1.0", 19 | "@types/pg": "^8.6.4", 20 | "@typescript-eslint/eslint-plugin": "^5.3.1", 21 | "@typescript-eslint/parser": "^5.18.0", 22 | "chai": "^4.3.6", 23 | "eslint": "^8.2.0", 24 | "mocha": "^9.2.1", 25 | "prettier": "^2.2.1", 26 | "pg": ">=8.6.0", 27 | "pg-mem": ">=2.5.0", 28 | "@rocicorp/eslint-config": "^0.1.2", 29 | "@rocicorp/prettier-config": "^0.1.1", 30 | "@types/express": "^4.17.13", 31 | "@types/node": "^16.11.50", 32 | "nodemon": "^2.0.19", 33 | "ts-node": "^10.9.1", 34 | "typescript": "^5.4.5", 35 | "zod": ">=3.17.3" 36 | }, 37 | "scripts": { 38 | "format": "prettier --write './src/**/*.{js,jsx,json,ts,tsx,html,css,md}' '*.{cjs,js,jsx,json,ts,tsx,html,css,md}'", 39 | "check-format": "prettier --check './src/**/*.{js,jsx,json,ts,tsx,html,css,md}' '*.{cjs,js,jsx,json,ts,tsx,html,css,md}'", 40 | "lint": "eslint --ext .ts,.tsx,.js,.jsx .", 41 | "build": "rm -rf ./dist && tsc", 42 | "check-types": "tsc --noEmit", 43 | "dev": "nodemon", 44 | "prod": "NODE_ENV=production node --loader ts-node/esm --experimental-specifier-resolution=node ./src/main.ts", 45 | "prepack": "npm run lint && npm run build", 46 | "pretest": "npm run build", 47 | "test": "mocha --ui=tdd 'dist/**/*.test.js'" 48 | }, 49 | "type": "module", 50 | "eslintConfig": { 51 | "extends": "@rocicorp/eslint-config" 52 | }, 53 | "prettier": "@rocicorp/prettier-config", 54 | "engines": { 55 | "node": ">=16.15.0" 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /replicache-quill/server/src/data.ts: -------------------------------------------------------------------------------- 1 | import type {JSONValue} from 'replicache'; 2 | import {z} from 'zod'; 3 | import type {Executor} from './pg.js'; 4 | 5 | export async function getEntry( 6 | executor: Executor, 7 | spaceid: string, 8 | key: string, 9 | ): Promise { 10 | const {rows} = await executor( 11 | 'select value from replicache_entry where spaceid = $1 and key = $2 and deleted = false', 12 | [spaceid, key], 13 | ); 14 | const value = rows[0]?.value; 15 | if (value === undefined) { 16 | return undefined; 17 | } 18 | return JSON.parse(value); 19 | } 20 | 21 | export async function putEntry( 22 | executor: Executor, 23 | spaceID: string, 24 | key: string, 25 | value: JSONValue, 26 | version: number, 27 | ): Promise { 28 | await executor( 29 | ` 30 | insert into replicache_entry (spaceid, key, value, deleted, version, lastmodified) 31 | values ($1, $2, $3, false, $4, now()) 32 | on conflict (spaceid, key) do update set 33 | value = $3, deleted = false, version = $4, lastmodified = now() 34 | `, 35 | [spaceID, key, JSON.stringify(value), version], 36 | ); 37 | } 38 | 39 | export async function delEntry( 40 | executor: Executor, 41 | spaceID: string, 42 | key: string, 43 | version: number, 44 | ): Promise { 45 | await executor( 46 | `update replicache_entry set deleted = true, version = $3 where spaceid = $1 and key = $2`, 47 | [spaceID, key, version], 48 | ); 49 | } 50 | 51 | export async function* getEntries( 52 | executor: Executor, 53 | spaceID: string, 54 | fromKey: string, 55 | ): AsyncIterable { 56 | const {rows} = await executor( 57 | `select key, value from replicache_entry where spaceid = $1 and key >= $2 and deleted = false order by key`, 58 | [spaceID, fromKey], 59 | ); 60 | for (const row of rows) { 61 | yield [row.key as string, JSON.parse(row.value) as JSONValue] as const; 62 | } 63 | } 64 | 65 | export async function getChangedEntries( 66 | executor: Executor, 67 | spaceID: string, 68 | prevVersion: number, 69 | ): Promise<[key: string, value: JSONValue, deleted: boolean][]> { 70 | const {rows} = await executor( 71 | `select key, value, deleted from replicache_entry where spaceid = $1 and version > $2`, 72 | [spaceID, prevVersion], 73 | ); 74 | return rows.map(row => [row.key, JSON.parse(row.value), row.deleted]); 75 | } 76 | 77 | export async function createSpace( 78 | executor: Executor, 79 | spaceID: string, 80 | ): Promise { 81 | console.log('creating space', spaceID); 82 | await executor( 83 | `insert into replicache_space (id, version, lastmodified) values ($1, 0, now())`, 84 | [spaceID], 85 | ); 86 | } 87 | 88 | export async function getCookie( 89 | executor: Executor, 90 | spaceID: string, 91 | ): Promise { 92 | const {rows} = await executor( 93 | `select version from replicache_space where id = $1`, 94 | [spaceID], 95 | ); 96 | const value = rows[0]?.version; 97 | if (value === undefined) { 98 | return undefined; 99 | } 100 | return z.number().parse(value); 101 | } 102 | 103 | export async function setCookie( 104 | executor: Executor, 105 | spaceID: string, 106 | version: number, 107 | ): Promise { 108 | await executor( 109 | `update replicache_space set version = $2, lastmodified = now() where id = $1`, 110 | [spaceID, version], 111 | ); 112 | } 113 | 114 | export async function getLastMutationID( 115 | executor: Executor, 116 | clientID: string, 117 | ): Promise { 118 | const {rows} = await executor( 119 | `select lastmutationid from replicache_client where id = $1`, 120 | [clientID], 121 | ); 122 | const value = rows[0]?.lastmutationid; 123 | if (value === undefined) { 124 | return undefined; 125 | } 126 | return z.number().parse(value); 127 | } 128 | 129 | export async function getLastMutationIDs( 130 | executor: Executor, 131 | clientIDs: string[], 132 | ) { 133 | return Object.fromEntries( 134 | await Promise.all( 135 | clientIDs.map(async cid => { 136 | const lmid = await getLastMutationID(executor, cid); 137 | return [cid, lmid ?? 0] as const; 138 | }), 139 | ), 140 | ); 141 | } 142 | 143 | export async function getLastMutationIDsSince( 144 | executor: Executor, 145 | clientGroupID: string, 146 | sinceVersion: number, 147 | ) { 148 | const {rows} = await executor( 149 | `select id, lastmutationid from replicache_client where clientgroupid = $1 and version > $2`, 150 | [clientGroupID, sinceVersion], 151 | ); 152 | return Object.fromEntries( 153 | rows.map(r => [r.id as string, r.lastmutationid as number] as const), 154 | ); 155 | } 156 | 157 | export async function setLastMutationID( 158 | executor: Executor, 159 | clientID: string, 160 | clientGroupID: string, 161 | lastMutationID: number, 162 | version: number, 163 | ): Promise { 164 | await executor( 165 | ` 166 | insert into replicache_client (id, clientgroupid, lastmutationid, version, lastmodified) 167 | values ($1, $2, $3, $4, now()) 168 | on conflict (id) do update set lastmutationid = $3, version = $4, lastmodified = now() 169 | `, 170 | [clientID, clientGroupID, lastMutationID, version], 171 | ); 172 | } 173 | 174 | export async function setLastMutationIDs( 175 | executor: Executor, 176 | clientGroupID: string, 177 | lmids: Record, 178 | version: number, 179 | ) { 180 | return await Promise.all( 181 | [...Object.entries(lmids)].map(([clientID, lmid]) => 182 | setLastMutationID(executor, clientID, clientGroupID, lmid, version), 183 | ), 184 | ); 185 | } 186 | -------------------------------------------------------------------------------- /replicache-quill/server/src/main.ts: -------------------------------------------------------------------------------- 1 | import 'dotenv/config'; 2 | import path from 'path'; 3 | import {mutators} from 'shared'; 4 | import {fileURLToPath} from 'url'; 5 | import express from 'express'; 6 | import type Express from 'express'; 7 | import {handleRequest} from '../endpoints/handle-request.js'; 8 | 9 | import fs from 'fs'; 10 | import {handlePoke} from '../endpoints/handle-poke'; 11 | const __filename = fileURLToPath(import.meta.url); 12 | const __dirname = path.dirname(__filename); 13 | const portEnv = parseInt(process.env.PORT || ''); 14 | const port = Number.isInteger(portEnv) ? portEnv : 8080; 15 | const options = { 16 | mutators, 17 | port, 18 | host: process.env.HOST || '0.0.0.0', 19 | }; 20 | 21 | const default_dist = path.join(__dirname, '../dist/dist'); 22 | 23 | const app = express(); 24 | 25 | const errorHandler = ( 26 | err: Error, 27 | _req: Express.Request, 28 | res: Express.Response, 29 | next: Express.NextFunction, 30 | ) => { 31 | res.status(500).send(err.message); 32 | next(err); 33 | }; 34 | 35 | app.use(express.urlencoded({extended: true}), express.json(), errorHandler); 36 | 37 | app.post( 38 | '/api/replicache/:op', 39 | async ( 40 | req: Express.Request, 41 | res: Express.Response, 42 | next: Express.NextFunction, 43 | ) => { 44 | await handleRequest(req, res, next, mutators); 45 | }, 46 | ); 47 | app.get( 48 | '/api/replicache/poke', 49 | async (req: Express.Request, res: Express.Response) => { 50 | await handlePoke(req, res); 51 | }, 52 | ); 53 | 54 | if (process.env.NODE_ENV === 'production') { 55 | app.use(express.static(default_dist)); 56 | app.get('/health', (_req, res) => { 57 | res.send('ok'); 58 | }); 59 | app.use('*', (_req, res) => { 60 | const index = path.join(default_dist, 'index.html'); 61 | const html = fs.readFileSync(index, 'utf8'); 62 | res.status(200).set({'Content-Type': 'text/html'}).end(html); 63 | }); 64 | } 65 | 66 | app.listen(options.port, options.host, () => { 67 | console.log(`Server listening on ${options.host}:${options.port}`); 68 | }); 69 | -------------------------------------------------------------------------------- /replicache-quill/server/src/pg.ts: -------------------------------------------------------------------------------- 1 | // Low-level config and utilities for Postgres. 2 | 3 | import type {Pool, QueryResult} from 'pg'; 4 | import {createDatabase} from './schema.js'; 5 | import {getDBConfig} from './pgconfig/pgconfig.js'; 6 | 7 | const pool = getPool(); 8 | 9 | async function getPool() { 10 | const global = globalThis as unknown as { 11 | // eslint-disable-next-line @typescript-eslint/naming-convention 12 | _pool: Pool; 13 | }; 14 | if (!global._pool) { 15 | global._pool = await initPool(); 16 | } 17 | return global._pool; 18 | } 19 | 20 | async function initPool() { 21 | console.log('creating global pool'); 22 | 23 | const dbConfig = getDBConfig(); 24 | const pool = dbConfig.initPool(); 25 | 26 | // the pool will emit an error on behalf of any idle clients 27 | // it contains if a backend error or network partition happens 28 | pool.on('error', err => { 29 | console.error('Unexpected error on idle client', err); 30 | process.exit(-1); 31 | }); 32 | pool.on('connect', async client => { 33 | // eslint-disable-next-line @typescript-eslint/no-floating-promises 34 | client.query( 35 | 'SET SESSION CHARACTERISTICS AS TRANSACTION ISOLATION LEVEL SERIALIZABLE', 36 | ); 37 | }); 38 | 39 | await withExecutorAndPool(async executor => { 40 | await transactWithExecutor(executor, async executor => { 41 | await createDatabase(executor, dbConfig); 42 | }); 43 | }, pool); 44 | 45 | return pool; 46 | } 47 | 48 | export async function withExecutor(f: (executor: Executor) => R) { 49 | const p = await pool; 50 | return withExecutorAndPool(f, p); 51 | } 52 | 53 | async function withExecutorAndPool( 54 | f: (executor: Executor) => R, 55 | p: Pool, 56 | ): Promise { 57 | const client = await p.connect(); 58 | 59 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 60 | const executor = async (sql: string, params?: any[]) => { 61 | try { 62 | return await client.query(sql, params); 63 | } catch (e) { 64 | throw new Error( 65 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 66 | `Error executing SQL: ${sql}: ${(e as unknown as any).toString()}`, 67 | ); 68 | } 69 | }; 70 | 71 | try { 72 | return await f(executor); 73 | } finally { 74 | client.release(); 75 | } 76 | } 77 | 78 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 79 | export type Executor = (sql: string, params?: any[]) => Promise; 80 | export type TransactionBodyFn = (executor: Executor) => Promise; 81 | 82 | /** 83 | * Invokes a supplied function within a transaction. 84 | * @param body Function to invoke. If this throws, the transaction will be rolled 85 | * back. The thrown error will be re-thrown. 86 | */ 87 | export async function transact(body: TransactionBodyFn) { 88 | return await withExecutor(async executor => { 89 | return await transactWithExecutor(executor, body); 90 | }); 91 | } 92 | 93 | async function transactWithExecutor( 94 | executor: Executor, 95 | body: TransactionBodyFn, 96 | ) { 97 | for (let i = 0; i < 10; i++) { 98 | try { 99 | await executor('begin'); 100 | try { 101 | const r = await body(executor); 102 | await executor('commit'); 103 | return r; 104 | } catch (e) { 105 | console.log('caught error', e, 'rolling back'); 106 | await executor('rollback'); 107 | throw e; 108 | } 109 | } catch (e) { 110 | if (shouldRetryTransaction(e)) { 111 | console.log( 112 | `Retrying transaction due to error ${e} - attempt number ${i}`, 113 | ); 114 | continue; 115 | } 116 | throw e; 117 | } 118 | } 119 | throw new Error('Tried to execute transacation too many times. Giving up.'); 120 | } 121 | 122 | //stackoverflow.com/questions/60339223/node-js-transaction-coflicts-in-postgresql-optimistic-concurrency-control-and 123 | function shouldRetryTransaction(err: unknown) { 124 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 125 | const code = typeof err === 'object' ? String((err as any).code) : null; 126 | return code === '40001' || code === '40P01'; 127 | } 128 | -------------------------------------------------------------------------------- /replicache-quill/server/src/pgconfig/pgconfig.ts: -------------------------------------------------------------------------------- 1 | import type {Pool} from 'pg'; 2 | import type {Executor} from '../pg.js'; 3 | import {PGMemConfig} from './pgmem.js'; 4 | import {PostgresDBConfig} from './postgres.js'; 5 | 6 | export interface PGConfig { 7 | initPool(): Pool; 8 | getSchemaVersion(executor: Executor): Promise; 9 | } 10 | 11 | export function getDBConfig(): PGConfig { 12 | const dbURL = process.env.DATABASE_URL; 13 | if (dbURL) { 14 | return new PostgresDBConfig(dbURL); 15 | } 16 | return new PGMemConfig(); 17 | } 18 | -------------------------------------------------------------------------------- /replicache-quill/server/src/pgconfig/pgmem.ts: -------------------------------------------------------------------------------- 1 | import type {Pool} from 'pg'; 2 | import {newDb} from 'pg-mem'; 3 | import type {PGConfig} from './pgconfig.js'; 4 | 5 | export class PGMemConfig implements PGConfig { 6 | constructor() { 7 | console.log('Creating PGMemConfig'); 8 | } 9 | 10 | initPool(): Pool { 11 | return new (newDb().adapters.createPg().Pool)() as Pool; 12 | } 13 | 14 | async getSchemaVersion(): Promise { 15 | // pg-mem lacks the system tables we normally use to introspect our 16 | // version. Luckily since pg-mem is in memory, we know that everytime we 17 | // start, we're starting fresh :). 18 | return 0; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /replicache-quill/server/src/pgconfig/postgres.ts: -------------------------------------------------------------------------------- 1 | import pg from 'pg'; 2 | import type {Executor} from '../pg.js'; 3 | import type {PGConfig} from './pgconfig.js'; 4 | 5 | /** 6 | * Implements PGConfig over a basic Postgres connection. 7 | */ 8 | export class PostgresDBConfig implements PGConfig { 9 | private _url: string; 10 | 11 | constructor(url: string) { 12 | console.log('Creating PostgresDBConfig with url', url); 13 | this._url = url; 14 | } 15 | 16 | initPool(): pg.Pool { 17 | const ssl = 18 | process.env.NODE_ENV === 'production' 19 | ? { 20 | rejectUnauthorized: false, 21 | } 22 | : undefined; 23 | return new pg.Pool({ 24 | connectionString: this._url, 25 | ssl, 26 | }); 27 | } 28 | 29 | async getSchemaVersion(executor: Executor): Promise { 30 | const metaExists = await executor(`select exists( 31 | select from pg_tables where schemaname = 'public' and tablename = 'replicache_meta')`); 32 | if (!metaExists.rows[0].exists) { 33 | return 0; 34 | } 35 | const qr = await executor( 36 | `select value from replicache_meta where key = 'schemaVersion'`, 37 | ); 38 | return qr.rows[0].value; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /replicache-quill/server/src/poke.ts: -------------------------------------------------------------------------------- 1 | export function getPokeBackend() { 2 | // The SSE impl has to keep process-wide state using the global object. 3 | // Otherwise the state is lost during hot reload in dev. 4 | const global = globalThis as unknown as { 5 | // eslint-disable-next-line @typescript-eslint/naming-convention 6 | _pokeBackend: PokeBackend; 7 | }; 8 | if (!global._pokeBackend) { 9 | global._pokeBackend = new PokeBackend(); 10 | } 11 | return global._pokeBackend; 12 | } 13 | 14 | type Listener = () => void; 15 | type ListenerMap = Map>; 16 | 17 | // Implements the poke backend using server-sent events. 18 | export class PokeBackend { 19 | private _listeners: ListenerMap; 20 | 21 | constructor() { 22 | this._listeners = new Map(); 23 | } 24 | 25 | addListener(spaceID: string, listener: () => void) { 26 | let set = this._listeners.get(spaceID); 27 | if (!set) { 28 | set = new Set(); 29 | this._listeners.set(spaceID, set); 30 | } 31 | set.add(listener); 32 | return () => this._removeListener(spaceID, listener); 33 | } 34 | 35 | poke(spaceID: string) { 36 | const set = this._listeners.get(spaceID); 37 | if (!set) { 38 | return; 39 | } 40 | for (const listener of set) { 41 | try { 42 | listener(); 43 | } catch (e) { 44 | console.error(e); 45 | } 46 | } 47 | } 48 | 49 | private _removeListener(spaceID: string, listener: () => void) { 50 | const set = this._listeners.get(spaceID); 51 | if (!set) { 52 | return; 53 | } 54 | set.delete(listener); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /replicache-quill/server/src/postgres-storage.ts: -------------------------------------------------------------------------------- 1 | import type {JSONValue} from 'replicache'; 2 | import type {Storage} from 'replicache-transaction'; 3 | import {putEntry, getEntry, delEntry, getEntries} from './data.js'; 4 | import type {Executor} from './pg.js'; 5 | 6 | // Implements the Storage interface required by replicache-transaction in terms 7 | // of our Postgres database. 8 | export class PostgresStorage implements Storage { 9 | private _spaceID: string; 10 | private _version: number; 11 | private _executor: Executor; 12 | 13 | constructor(spaceID: string, version: number, executor: Executor) { 14 | this._spaceID = spaceID; 15 | this._version = version; 16 | this._executor = executor; 17 | } 18 | 19 | putEntry(key: string, value: JSONValue): Promise { 20 | return putEntry(this._executor, this._spaceID, key, value, this._version); 21 | } 22 | 23 | async hasEntry(key: string): Promise { 24 | const v = await this.getEntry(key); 25 | return v !== undefined; 26 | } 27 | 28 | getEntry(key: string): Promise { 29 | return getEntry(this._executor, this._spaceID, key); 30 | } 31 | 32 | getEntries(fromKey: string): AsyncIterable { 33 | return getEntries(this._executor, this._spaceID, fromKey); 34 | } 35 | 36 | delEntry(key: string): Promise { 37 | return delEntry(this._executor, this._spaceID, key, this._version); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /replicache-quill/server/src/pull.ts: -------------------------------------------------------------------------------- 1 | import {transact} from './pg.js'; 2 | import {getChangedEntries, getCookie, getLastMutationIDsSince} from './data.js'; 3 | import {z} from 'zod'; 4 | import type {ClientID, PatchOperation} from 'replicache'; 5 | import type Express from 'express'; 6 | 7 | const pullRequest = z.object({ 8 | profileID: z.string(), 9 | clientGroupID: z.string(), 10 | cookie: z.union([z.number(), z.null()]), 11 | schemaVersion: z.string(), 12 | }); 13 | 14 | export type PullResponse = { 15 | cookie: number; 16 | lastMutationIDChanges: Record; 17 | patch: PatchOperation[]; 18 | }; 19 | 20 | export async function pull( 21 | spaceID: string, 22 | requestBody: Express.Request, 23 | ): Promise { 24 | console.log(`Processing pull`, JSON.stringify(requestBody, null, '')); 25 | 26 | const pull = pullRequest.parse(requestBody); 27 | const {cookie: requestCookie} = pull; 28 | 29 | console.log('spaceID', spaceID); 30 | 31 | const t0 = Date.now(); 32 | const sinceCookie = requestCookie ?? 0; 33 | 34 | const [entries, lastMutationIDChanges, responseCookie] = await transact( 35 | async executor => { 36 | return Promise.all([ 37 | getChangedEntries(executor, spaceID, sinceCookie), 38 | getLastMutationIDsSince(executor, pull.clientGroupID, sinceCookie), 39 | getCookie(executor, spaceID), 40 | ]); 41 | }, 42 | ); 43 | 44 | console.log('lastMutationIDChanges: ', lastMutationIDChanges); 45 | console.log('responseCookie: ', responseCookie); 46 | console.log('Read all objects in', Date.now() - t0); 47 | 48 | if (responseCookie === undefined) { 49 | throw new Error(`Unknown space ${spaceID}`); 50 | } 51 | 52 | const resp: PullResponse = { 53 | lastMutationIDChanges, 54 | cookie: responseCookie, 55 | patch: [], 56 | }; 57 | 58 | for (const [key, value, deleted] of entries) { 59 | if (deleted) { 60 | resp.patch.push({ 61 | op: 'del', 62 | key, 63 | }); 64 | } else { 65 | resp.patch.push({ 66 | op: 'put', 67 | key, 68 | value, 69 | }); 70 | } 71 | } 72 | 73 | console.log(`Returning`, JSON.stringify(resp, null, '')); 74 | return resp; 75 | } 76 | -------------------------------------------------------------------------------- /replicache-quill/server/src/push.ts: -------------------------------------------------------------------------------- 1 | import {transact} from './pg.js'; 2 | import {ReplicacheTransaction} from 'replicache-transaction'; 3 | import {z, ZodType} from 'zod'; 4 | import {getPokeBackend} from './poke.js'; 5 | import type {MutatorDefs, ReadonlyJSONValue} from 'replicache'; 6 | import {PostgresStorage} from './postgres-storage.js'; 7 | import { 8 | getCookie, 9 | getLastMutationIDs, 10 | setCookie, 11 | setLastMutationIDs, 12 | } from './data.js'; 13 | 14 | const mutationSchema = z.object({ 15 | clientID: z.string(), 16 | id: z.number(), 17 | name: z.string(), 18 | args: z.any(), 19 | }); 20 | 21 | const pushRequestSchema = z.object({ 22 | profileID: z.string(), 23 | clientGroupID: z.string(), 24 | mutations: z.array(mutationSchema), 25 | }); 26 | 27 | type PushRequest = z.infer; 28 | export type Error = 'SpaceNotFound'; 29 | 30 | export function parseIfDebug( 31 | schema: ZodType, 32 | val: T, 33 | ): T { 34 | if (globalThis.process?.env?.NODE_ENV !== 'production') { 35 | return schema.parse(val); 36 | } 37 | return val as T; 38 | } 39 | 40 | export async function push( 41 | spaceID: string, 42 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 43 | requestBody: any, 44 | mutators: M, 45 | ) { 46 | console.log('Processing push', JSON.stringify(requestBody, null, '')); 47 | 48 | const push = parseIfDebug(pushRequestSchema, requestBody); 49 | 50 | const {clientGroupID} = push; 51 | 52 | const t0 = Date.now(); 53 | await transact(async executor => { 54 | const prevVersion = await getCookie(executor, spaceID); 55 | if (prevVersion === undefined) { 56 | throw new Error(`Unknown space ${spaceID}`); 57 | } 58 | 59 | const nextVersion = prevVersion + 1; 60 | const clientIDs = [...new Set(push.mutations.map(m => m.clientID))]; 61 | 62 | const lastMutationIDs = await getLastMutationIDs(executor, clientIDs); 63 | 64 | console.log(JSON.stringify({prevVersion, nextVersion, lastMutationIDs})); 65 | 66 | const storage = new PostgresStorage(spaceID, nextVersion, executor); 67 | const tx = new ReplicacheTransaction(storage); 68 | 69 | for (let i = 0; i < push.mutations.length; i++) { 70 | const mutation = push.mutations[i]; 71 | const {clientID} = mutation; 72 | const lastMutationID = lastMutationIDs[clientID]; 73 | if (lastMutationID === undefined) { 74 | throw new Error( 75 | 'invalid state - lastMutationID not found for client: ' + clientID, 76 | ); 77 | } 78 | const expectedMutationID = lastMutationID + 1; 79 | 80 | if (mutation.id < expectedMutationID) { 81 | console.log( 82 | `Mutation ${mutation.id} has already been processed - skipping`, 83 | ); 84 | continue; 85 | } 86 | if (mutation.id > expectedMutationID) { 87 | console.warn(`Mutation ${mutation.id} is from the future - aborting`); 88 | break; 89 | } 90 | 91 | console.log('Processing mutation:', JSON.stringify(mutation, null, '')); 92 | 93 | const t1 = Date.now(); 94 | 95 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 96 | const mutator = (mutators as any)[mutation.name]; 97 | if (!mutator) { 98 | console.error(`Unknown mutator: ${mutation.name} - skipping`); 99 | } 100 | 101 | try { 102 | await mutator(tx, mutation.args); 103 | } catch (e) { 104 | console.error( 105 | `Error executing mutator: ${JSON.stringify(mutator)}: ${e}`, 106 | ); 107 | } 108 | 109 | lastMutationIDs[clientID] = expectedMutationID; 110 | console.log('Processed mutation in', Date.now() - t1); 111 | } 112 | 113 | await Promise.all([ 114 | setLastMutationIDs(executor, clientGroupID, lastMutationIDs, nextVersion), 115 | setCookie(executor, spaceID, nextVersion), 116 | tx.flush(), 117 | ]); 118 | 119 | const pokeBackend = getPokeBackend(); 120 | await pokeBackend.poke(spaceID); 121 | }); 122 | 123 | console.log('Processed all mutations in', Date.now() - t0); 124 | } 125 | -------------------------------------------------------------------------------- /replicache-quill/server/src/schema.ts: -------------------------------------------------------------------------------- 1 | import type {PGConfig} from './pgconfig/pgconfig.js'; 2 | import type {Executor} from './pg.js'; 3 | 4 | export async function createDatabase(executor: Executor, dbConfig: PGConfig) { 5 | console.log('Creating Database'); 6 | const schemaVersion = await dbConfig.getSchemaVersion(executor); 7 | const migrations = [createSchemaVersion1]; 8 | if (schemaVersion < 0 || schemaVersion > migrations.length) { 9 | throw new Error('Unexpected schema version: ' + schemaVersion); 10 | } 11 | await createSchemaVersion1(executor); 12 | } 13 | 14 | export async function createSchemaVersion1(executor: Executor) { 15 | await executor( 16 | `create table replicache_meta (key text primary key, value json)`, 17 | ); 18 | await executor( 19 | `insert into replicache_meta (key, value) values ('schemaVersion', '1')`, 20 | ); 21 | 22 | await executor(`create table replicache_space ( 23 | id text primary key not null, 24 | version integer not null, 25 | lastmodified timestamp(6) not null 26 | )`); 27 | 28 | await executor(`create table replicache_client ( 29 | id text primary key not null, 30 | lastmutationid integer not null, 31 | version integer not null, 32 | lastmodified timestamp(6) not null, 33 | clientgroupid text not null 34 | )`); 35 | 36 | await executor(`create table replicache_entry ( 37 | spaceid text not null, 38 | key text not null, 39 | value text not null, 40 | deleted boolean not null, 41 | version integer not null, 42 | lastmodified timestamp(6) not null 43 | )`); 44 | 45 | await executor(`create unique index on replicache_entry (spaceid, key)`); 46 | await executor(`create index on replicache_entry (spaceid)`); 47 | await executor(`create index on replicache_entry (deleted)`); 48 | await executor(`create index on replicache_entry (version)`); 49 | await executor(`create index on replicache_client (clientgroupid,version)`); 50 | } 51 | -------------------------------------------------------------------------------- /replicache-quill/server/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "esnext", 4 | "module": "esnext", 5 | "strict": true, 6 | "noUnusedLocals": true, 7 | "noUnusedParameters": true, 8 | "noImplicitReturns": true, 9 | "noFallthroughCasesInSwitch": true, 10 | "esModuleInterop": true, 11 | "declaration": true, 12 | "forceConsistentCasingInFileNames": true, 13 | "moduleResolution": "node", 14 | "outDir": "dist", 15 | "allowJs": true, 16 | // esnext for Object.fromEntries 17 | "lib": ["dom", "DOM.Iterable", "esnext"], 18 | "preserveSymlinks": true, 19 | "types": ["node"] 20 | }, 21 | "include": ["**/src/**/*.ts"], 22 | "exclude": ["node_modules", "lib"] 23 | } 24 | -------------------------------------------------------------------------------- /replicache-quill/shared/.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | out 3 | tool 4 | bin 5 | .eslintrc.cjs 6 | dist 7 | lib 8 | env.d.ts 9 | vite.config.ts -------------------------------------------------------------------------------- /replicache-quill/shared/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "shared", 3 | "version": "0.1.0", 4 | "private": true, 5 | "devDependencies": { 6 | "@list-positions/formatting": "^1.0.1", 7 | "list-positions": "^1.0.0", 8 | "@rocicorp/eslint-config": "^0.1.2", 9 | "@rocicorp/prettier-config": "^0.1.1", 10 | "typescript": "^5.4.5", 11 | "replicache": ">=14.0.3" 12 | }, 13 | "scripts": { 14 | "format": "prettier --write './src/**/*.{js,jsx,json,ts,tsx,html,css,md}' '*.{cjs,js,jsx,json,ts,tsx,html,css,md}'", 15 | "check-format": "prettier --check './src/**/*.{js,jsx,json,ts,tsx,html,css,md}' '*.{cjs,js,jsx,json,ts,tsx,html,css,md}'", 16 | "lint": "eslint --ext .ts,.tsx,.js,.jsx .", 17 | "check-types": "tsc --noEmit" 18 | }, 19 | "type": "module", 20 | "eslintConfig": { 21 | "extends": "@rocicorp/eslint-config" 22 | }, 23 | "prettier": "@rocicorp/prettier-config", 24 | "main": "src/index.ts" 25 | } 26 | -------------------------------------------------------------------------------- /replicache-quill/shared/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './mutators'; 2 | export * from './rich_text'; 3 | -------------------------------------------------------------------------------- /replicache-quill/shared/src/mutators.ts: -------------------------------------------------------------------------------- 1 | // This file defines our "mutators". 2 | // 3 | // Mutators are how you change data in Replicache apps. 4 | // 5 | // They are registered with Replicache at construction-time and callable like: 6 | // `myReplicache.mutate.createTodo({text: "foo"})`. 7 | // 8 | // Replicache runs each mutation immediately (optimistically) on the client, 9 | // against the local cache, and then later (usually moments later) sends a 10 | // description of the mutation (its name and arguments) to the server, so that 11 | // the server can *re-run* the mutation there against the authoritative 12 | // datastore. 13 | // 14 | // This re-running of mutations is how Replicache handles conflicts: the 15 | // mutators defensively check the database when they run and do the appropriate 16 | // thing. The Replicache sync protocol ensures that the server-side result takes 17 | // precedence over the client-side optimistic result. 18 | // 19 | // If the server is written in JavaScript, the mutator functions can be directly 20 | // reused on the server. This sample demonstrates the pattern by using these 21 | // mutators both with Replicache on the client (see [id]].tsx) and on the server 22 | // (see pages/api/replicache/[op].ts). 23 | // 24 | // See https://doc.replicache.dev/how-it-works#sync-details for all the detail 25 | // on how Replicache syncs and resolves conflicts, but understanding that is not 26 | // required to get up and running. 27 | 28 | import type {ReadonlyJSONValue, WriteTransaction} from 'replicache'; 29 | import { 30 | idOfMark, 31 | type AddMarks, 32 | type Bunch, 33 | type CreateBunch, 34 | type DeleteValues, 35 | type SetValues, 36 | } from './rich_text'; 37 | 38 | export type M = typeof mutators; 39 | 40 | export const mutators = { 41 | createBunch: async (tx: WriteTransaction, update: CreateBunch) => { 42 | const existing = await tx.get(`bunch/${update.bunchID}`); 43 | if (existing !== undefined) { 44 | console.warn('createBunch: Skipping duplicate bunchID:', update.bunchID); 45 | return; 46 | } 47 | const newBunch: Bunch = {meta: update, values: {}}; 48 | await tx.set(`bunch/${update.bunchID}`, newBunch); 49 | }, 50 | 51 | setValues: async (tx: WriteTransaction, update: SetValues) => { 52 | const existing = await tx.get(`bunch/${update.startPos.bunchID}`); 53 | if (existing === undefined) { 54 | console.error( 55 | 'setValues: Skipping unknown bunchID:', 56 | update.startPos.bunchID, 57 | ); 58 | return; 59 | } 60 | const values: Record = {...existing.values}; 61 | for (let i = 0; i < update.values.length; i++) { 62 | values[i + update.startPos.innerIndex] = update.values[i]; 63 | } 64 | const updated: Bunch = {...existing, values}; 65 | await tx.set(`bunch/${update.startPos.bunchID}`, updated); 66 | }, 67 | 68 | deleteValues: async (tx: WriteTransaction, update: DeleteValues) => { 69 | const existing = await tx.get(`bunch/${update.startPos.bunchID}`); 70 | if (existing === undefined) { 71 | console.error( 72 | 'setValues: Skipping unknown bunchID:', 73 | update.startPos.bunchID, 74 | ); 75 | return; 76 | } 77 | const values: Record = {...existing.values}; 78 | for (let i = 0; i < update.count; i++) { 79 | delete values[i + update.startPos.innerIndex]; 80 | } 81 | const updated: Bunch = {...existing, values}; 82 | await tx.set(`bunch/${update.startPos.bunchID}`, updated); 83 | }, 84 | 85 | addMarks: async (tx: WriteTransaction, update: AddMarks) => { 86 | for (const mark of update.marks) { 87 | const id = idOfMark(mark); 88 | const existing = await tx.get(`mark/${id}`); 89 | if (existing !== undefined) { 90 | console.warn('addMarks: Skipping duplicate mark ID:', id); 91 | continue; 92 | } 93 | // ReadonlyJSONValue is supposed to express that the value is deep-readonly. 94 | // Because of https://github.com/microsoft/TypeScript/issues/15300 , though, 95 | // it doesn't work on JSON objects whose type is (or includes) an interface. 96 | await tx.set(`mark/${id}`, mark as unknown as ReadonlyJSONValue); 97 | } 98 | }, 99 | }; 100 | -------------------------------------------------------------------------------- /replicache-quill/shared/src/rich_text.ts: -------------------------------------------------------------------------------- 1 | import type {TimestampMark} from '@list-positions/formatting'; 2 | import type {BunchMeta, Position} from 'list-positions'; 3 | import type {ReadTransaction} from 'replicache'; 4 | 5 | export type Bunch = { 6 | meta: BunchMeta; 7 | values: {[innerIndex: number]: string}; 8 | }; 9 | 10 | export type CreateBunch = BunchMeta; 11 | 12 | export type SetValues = { 13 | startPos: Position; 14 | values: string[]; 15 | }; 16 | 17 | export type DeleteValues = { 18 | startPos: Position; 19 | count: number; 20 | }; 21 | 22 | export async function allBunches(tx: ReadTransaction) { 23 | return await tx.scan({prefix: 'bunch/'}).values().toArray(); 24 | } 25 | 26 | export function idOfMark(mark: TimestampMark): string { 27 | return `${mark.timestamp + ',' + mark.creatorID}`; 28 | } 29 | 30 | export type AddMarks = { 31 | marks: TimestampMark[]; 32 | }; 33 | 34 | export async function allMarks(tx: ReadTransaction): Promise { 35 | // ReadonlyJSONValue is supposed to express that the value is deep-readonly. 36 | // Because of https://github.com/microsoft/TypeScript/issues/15300 , though, 37 | // it doesn't work on JSON objects whose type is (or includes) an interface. 38 | return (await tx 39 | .scan({prefix: 'mark/'}) 40 | .values() 41 | .toArray()) as unknown as TimestampMark[]; 42 | } 43 | -------------------------------------------------------------------------------- /replicache-quill/shared/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "esnext", 4 | "module": "esnext", 5 | "strict": true, 6 | "noUnusedLocals": true, 7 | "noUnusedParameters": true, 8 | "noImplicitReturns": true, 9 | "noFallthroughCasesInSwitch": true, 10 | "esModuleInterop": true, 11 | "declaration": true, 12 | "forceConsistentCasingInFileNames": true, 13 | "moduleResolution": "node", 14 | "outDir": "lib", 15 | "allowJs": true, 16 | // esnext for Object.fromEntries 17 | "lib": ["dom", "DOM.Iterable", "esnext"], 18 | "preserveSymlinks": true 19 | }, 20 | "include": ["**/src/**/*.ts"], 21 | "exclude": ["node_modules", "lib"] 22 | } 23 | -------------------------------------------------------------------------------- /suggested-changes/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /suggested-changes/.prettierignore: -------------------------------------------------------------------------------- 1 | /dist/* 2 | -------------------------------------------------------------------------------- /suggested-changes/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2024 Matthew Weidner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /suggested-changes/README.md: -------------------------------------------------------------------------------- 1 | # Suggested Changes 2 | 3 | Extension of [websocket-prosemirror-blocks](../websocket-prosemirror-blocks/) that adds "Suggested Changes". 4 | 5 | Select some text and click "Suggest change" to create a suggestion on the right side. Edit it and accept/reject. 6 | 7 | Internally, each suggestion (class [`Suggestion`](./src/site/suggestion.ts)) first copies the selected portion of the main text into a new `ProseMirrorWrapper`. Edits to the suggestion are tracked as `Message`s, then committed to the main text if accepted - as if performed by a collaborator. Suggestions also update live to reflect their merge with the main text, via `src/site/main.ts`'s `onLocalChange` function. 8 | 9 | Code organization: 10 | 11 | - `src/common/`: Messages shared between clients and the server. 12 | - `src/server/`: WebSocket server. 13 | - `src/site/`: ProseMirror client. 14 | - `src/site/suggestion.ts`: Class managing the GUI and state for an individual suggestion. 15 | 16 | ## Future Plans 17 | 18 | 19 | 20 | - [ ] Make suggestions collaborative. 21 | - [ ] Show where suggestions are with a highlight in the main text. Increase highlight on the focused suggestion. 22 | - [ ] Formatting in suggestions - currently only keyboard shortcuts (Ctrl+I/B) are supported. 23 | - [ ] Testing/cleanup. 24 | 25 | ## Installation 26 | 27 | First, install [Node.js](https://nodejs.org/). Then run `npm i`. 28 | 29 | ## Commands 30 | 31 | ### `npm run dev` 32 | 33 | Build the app from `src/`, in [development mode](https://webpack.js.org/guides/development/). 34 | 35 | ### `npm run build` 36 | 37 | Build the app from `src/`, in [production mode](https://webpack.js.org/guides/production/). 38 | 39 | ### `npm start` 40 | 41 | Run the server on [http://localhost:3000/](http://localhost:3000/). Use multiple browser windows at once to test collaboration. 42 | 43 | To change the port, set the `$PORT` environment variable. 44 | 45 | ### `npm run clean` 46 | 47 | Delete `dist/`. 48 | -------------------------------------------------------------------------------- /suggested-changes/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "license": "MIT", 3 | "dependencies": { 4 | "express": "^4.18.2", 5 | "@list-positions/formatting": "^1.0.0", 6 | "list-positions": "^1.0.0", 7 | "maybe-random-string": "^1.0.0", 8 | "prosemirror-commands": "^1.5.2", 9 | "prosemirror-keymap": "^1.2.2", 10 | "prosemirror-model": "^1.19.4", 11 | "prosemirror-state": "^1.4.3", 12 | "prosemirror-transform": "^1.8.0", 13 | "prosemirror-view": "^1.32.7", 14 | "ws": "^8.13.0" 15 | }, 16 | "devDependencies": { 17 | "@types/express": "^4.17.17", 18 | "@types/node": "^15.6.1", 19 | "@types/webpack": "^5.28.0", 20 | "@types/webpack-env": "^1.16.2", 21 | "@types/ws": "^8.5.10", 22 | "copy-webpack-plugin": "^11.0.0", 23 | "cross-env": "^7.0.3", 24 | "css-loader": "^6.2.0", 25 | "html-webpack-plugin": "^5.3.2", 26 | "npm-run-all": "^4.1.5", 27 | "prettier": "^2.2.1", 28 | "rimraf": "^2.7.1", 29 | "source-map-loader": "^3.0.0", 30 | "style-loader": "^3.3.3", 31 | "ts-loader": "^9.2.5", 32 | "ts-node": "^10.1.0", 33 | "typescript": "^4.3.5", 34 | "webpack": "^5.50.0", 35 | "webpack-cli": "^4.10.0" 36 | }, 37 | "scripts": { 38 | "start": "ts-node -P tsconfig.server.json src/server/server.ts", 39 | "dev": "cross-env TS_NODE_PROJECT='tsconfig.webpack-config.json' webpack", 40 | "build": "cross-env TS_NODE_PROJECT='tsconfig.webpack-config.json' webpack --mode production --devtool source-map", 41 | "test": "npm-run-all test:*", 42 | "test:server-tsc": "tsc -p tsconfig.server.json", 43 | "test:format": "prettier --check .", 44 | "fix": "npm-run-all fix:*", 45 | "fix:format": "prettier --write .", 46 | "clean": "rimraf dist" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /suggested-changes/src/common/block_text.ts: -------------------------------------------------------------------------------- 1 | import { TimestampFormattingSavedState } from "@list-positions/formatting"; 2 | import { ListSavedState, OrderSavedState } from "list-positions"; 3 | 4 | /** 5 | * Immutable - don't mutate attrs directly. 6 | */ 7 | export type BlockMarker = { 8 | readonly type: string; 9 | readonly attrs?: Record; 10 | /** 11 | * Lamport timestamp for LWW. 12 | */ 13 | readonly timestamp: number; 14 | readonly creatorID: string; 15 | }; 16 | 17 | export type BlockTextSavedState = { 18 | readonly order: OrderSavedState; 19 | readonly text: ListSavedState; 20 | readonly blockMarkers: ListSavedState; 21 | readonly formatting: TimestampFormattingSavedState; 22 | }; 23 | -------------------------------------------------------------------------------- /suggested-changes/src/common/messages.ts: -------------------------------------------------------------------------------- 1 | import { TimestampMark } from "@list-positions/formatting"; 2 | import { BunchMeta, Position } from "list-positions"; 3 | import { BlockMarker, BlockTextSavedState } from "./block_text"; 4 | 5 | export type SetMessage = { 6 | type: "set"; 7 | startPos: Position; 8 | chars: string; 9 | meta?: BunchMeta; 10 | }; 11 | 12 | export type SetMarkerMessage = { 13 | type: "setMarker"; 14 | pos: Position; 15 | marker: BlockMarker; 16 | meta?: BunchMeta; 17 | }; 18 | 19 | export type DeleteMessage = { 20 | type: "delete"; 21 | pos: Position; 22 | }; 23 | 24 | export type MarkMessage = { 25 | type: "mark"; 26 | mark: TimestampMark; 27 | }; 28 | 29 | export type WelcomeMessage = { 30 | type: "welcome"; 31 | savedState: BlockTextSavedState; 32 | }; 33 | 34 | export type Message = 35 | | SetMessage 36 | | SetMarkerMessage 37 | | DeleteMessage 38 | | MarkMessage 39 | | WelcomeMessage; 40 | -------------------------------------------------------------------------------- /suggested-changes/src/server/rich_text_server.ts: -------------------------------------------------------------------------------- 1 | import { TimestampMark } from "@list-positions/formatting"; 2 | import { List, Order } from "list-positions"; 3 | import { WebSocket, WebSocketServer } from "ws"; 4 | import { BlockMarker } from "../common/block_text"; 5 | import { Message } from "../common/messages"; 6 | 7 | const heartbeatInterval = 30000; 8 | 9 | export class RichTextServer { 10 | // To easily save and send the state to new clients, store as Lists. 11 | private readonly order: Order; 12 | private readonly text: List; 13 | private readonly blockMarkers: List; 14 | // We don't need to inspect the formatting, so just store the marks directly. 15 | private readonly marks: TimestampMark[]; 16 | 17 | private clients = new Set(); 18 | 19 | constructor(readonly wss: WebSocketServer) { 20 | this.order = new Order(); 21 | this.text = new List(this.order); 22 | this.blockMarkers = new List(this.order); 23 | this.marks = []; 24 | 25 | // Initial state: a single paragraph, to match Prosemirror's starting state. 26 | this.blockMarkers.insertAt(0, { 27 | type: "paragraph", 28 | timestamp: 1, 29 | creatorID: "INIT", 30 | }); 31 | 32 | this.wss.on("connection", (ws) => { 33 | if (ws.readyState === WebSocket.OPEN) { 34 | this.wsOpen(ws); 35 | } else ws.on("open", () => this.wsOpen(ws)); 36 | ws.on("message", (data) => this.wsReceive(ws, data.toString())); 37 | ws.on("close", () => this.wsClose(ws)); 38 | ws.on("error", (err) => { 39 | console.error(err); 40 | this.wsClose(ws); 41 | }); 42 | }); 43 | } 44 | 45 | private sendMessage(ws: WebSocket, msg: Message) { 46 | if (ws.readyState == WebSocket.OPEN) { 47 | ws.send(JSON.stringify(msg)); 48 | } 49 | } 50 | 51 | private echo(origin: WebSocket, data: string) { 52 | for (const ws of this.clients) { 53 | if (ws === origin) continue; 54 | if (ws.readyState == WebSocket.OPEN) { 55 | ws.send(data); 56 | } 57 | } 58 | } 59 | 60 | private wsOpen(ws: WebSocket) { 61 | this.startHeartbeats(ws); 62 | 63 | // Send the current state. 64 | this.sendMessage(ws, { 65 | type: "welcome", 66 | savedState: { 67 | order: this.order.save(), 68 | text: this.text.save(), 69 | blockMarkers: this.blockMarkers.save(), 70 | formatting: this.marks, 71 | }, 72 | }); 73 | 74 | this.clients.add(ws); 75 | } 76 | 77 | /** 78 | * Ping to keep connection alive. 79 | * 80 | * This is necessary on at least Heroku, which has a 55 second timeout: 81 | * https://devcenter.heroku.com/articles/websockets#timeouts 82 | */ 83 | private startHeartbeats(ws: WebSocket) { 84 | const interval = setInterval(() => { 85 | if (ws.readyState === WebSocket.OPEN) { 86 | ws.ping(); 87 | } else clearInterval(interval); 88 | }, heartbeatInterval); 89 | } 90 | 91 | private wsReceive(ws: WebSocket, data: string) { 92 | const msg = JSON.parse(data) as Message; 93 | switch (msg.type) { 94 | case "set": 95 | if (msg.meta) this.order.addMetas([msg.meta]); 96 | this.text.set(msg.startPos, ...msg.chars); 97 | this.echo(ws, data); 98 | // Because a Position is only ever set once (when it's created) and 99 | // the server does no validation, the origin's optimistically-updated 100 | // state is already correct: msg.startPos is set to msg.chars. 101 | // If that were not true, we would need to send a message to origin 102 | // telling it how to repair its optimistically-updated state. 103 | break; 104 | case "setMarker": 105 | if (msg.meta) this.order.addMetas([msg.meta]); 106 | this.blockMarkers.set(msg.pos, msg.marker); 107 | this.echo(ws, data); 108 | // Because a Position is only ever set once (when it's created) and 109 | // the server does no validation, the origin's optimistically-updated 110 | // state is already correct: msg.pos is set to msg.marker. 111 | // If that were not true, we would need to send a message to origin 112 | // telling it how to repair its optimistically-updated state. 113 | break; 114 | case "delete": 115 | // Pos might belong to either list; try to delete from both. 116 | this.text.delete(msg.pos); 117 | this.blockMarkers.delete(msg.pos); 118 | this.echo(ws, data); 119 | // Because deletes are permanant and the server does no validation, 120 | // the origin's optimistically-updated state is already correct. 121 | break; 122 | case "mark": 123 | this.marks.push(msg.mark); 124 | this.echo(ws, data); 125 | // Because marks are permanant and the server does no validation, 126 | // the origin's optimistically-updated state is already correct. 127 | break; 128 | default: 129 | throw new Error("Unknown message type: " + msg.type); 130 | } 131 | } 132 | 133 | private wsClose(ws: WebSocket) { 134 | this.clients.delete(ws); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /suggested-changes/src/server/server.ts: -------------------------------------------------------------------------------- 1 | import express from "express"; 2 | import path from "path"; 3 | import { WebSocketServer } from "ws"; 4 | import { RichTextServer } from "./rich_text_server"; 5 | 6 | const port = process.env.PORT || 3000; 7 | 8 | // Server dist/ with a simple express server. 9 | const app = express(); 10 | app.use("/", express.static(path.join(__dirname, "../../dist"))); 11 | const server = app.listen(port, () => 12 | console.log(`Listening at http://localhost:${port}/`) 13 | ); 14 | 15 | // Run the WebSocket server. 16 | const wss = new WebSocketServer({ server }); 17 | new RichTextServer(wss); 18 | -------------------------------------------------------------------------------- /suggested-changes/src/site/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 36 | 37 | Suggested Changes 38 | 39 | 40 | 41 |
42 | Info and source code 46 |
47 |
48 | 49 |
50 |
51 | 52 |
53 | 54 | 55 | 56 | 57 |
58 | 59 |
60 | 61 | 62 | 63 |
64 |

Suggested Changes

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /suggested-changes/src/site/main.ts: -------------------------------------------------------------------------------- 1 | import { Message } from "../common/messages"; 2 | import { ProseMirrorWrapper } from "./prosemirror_wrapper"; 3 | import { Suggestion } from "./suggestion"; 4 | 5 | const wsURL = location.origin.replace(/^http/, "ws"); 6 | const ws = new WebSocket(wsURL); 7 | 8 | const suggestionsDiv = document.getElementById("suggestions") as HTMLDivElement; 9 | 10 | let wrapper!: ProseMirrorWrapper; 11 | const suggestions = new Set(); 12 | 13 | function welcomeListener(e: MessageEvent) { 14 | const msg = JSON.parse(e.data) as Message; 15 | if (msg.type === "welcome") { 16 | // Got the initial state. Start Quill. 17 | ws.removeEventListener("message", welcomeListener); 18 | wrapper = new ProseMirrorWrapper( 19 | document.querySelector("#editor")!, 20 | { savedState: msg.savedState }, 21 | onLocalChange 22 | ); 23 | ws.addEventListener("message", (e: MessageEvent) => { 24 | onWsMessage(e); 25 | }); 26 | 27 | for (const type of ["h1", "h2", "ul", "ol"]) { 28 | document.getElementById("button_" + type)!.onclick = () => 29 | setBlockType(type); 30 | } 31 | 32 | // Enable "suggest changes" button only when the selection is nontrivial. 33 | const suggestChanges = document.getElementById( 34 | "button_suggest" 35 | ) as HTMLButtonElement; 36 | wrapper.onSelectionChange = () => { 37 | const pmSel = wrapper.view.state.selection; 38 | suggestChanges.disabled = pmSel.from === pmSel.to; 39 | }; 40 | suggestChanges.onclick = () => { 41 | const suggestion = new Suggestion( 42 | suggestionsDiv, 43 | wrapper, 44 | onAccept, 45 | onReject 46 | ); 47 | suggestions.add(suggestion); 48 | }; 49 | } else { 50 | console.error("Received non-welcome message first: " + msg.type); 51 | } 52 | } 53 | ws.addEventListener("message", welcomeListener); 54 | 55 | // For this basic demo, we don't allow disconnection tests or 56 | // attempt to reconnect the WebSocket ever. 57 | // That would require buffering updates and/or logic to 58 | // "merge" in the Welcome state received after reconnecting. 59 | 60 | function onLocalChange(msgs: Message[]) { 61 | send(msgs); 62 | // OPT: use formatting spans to quickly get the affected suggestions, 63 | // instead of looping over all of them? 64 | // Or, only update the suggestions in view. 65 | // Likewise in onWsMessage. 66 | for (const suggestion of suggestions) suggestion.applyOriginMessages(msgs); 67 | } 68 | 69 | function send(msgs: Message[]): void { 70 | if (ws.readyState === WebSocket.OPEN) { 71 | for (const msg of msgs) { 72 | ws.send(JSON.stringify(msg)); 73 | } 74 | } 75 | } 76 | 77 | // OPT: batch delivery, wrapped in wrapper.update(). 78 | function onWsMessage(e: MessageEvent): void { 79 | const msg = JSON.parse(e.data) as Message; 80 | wrapper.applyMessage(msg); 81 | for (const suggestion of suggestions) suggestion.applyOriginMessages([msg]); 82 | } 83 | 84 | /** 85 | * Called when a suggested change is accepted, with the given changes. 86 | */ 87 | function onAccept(caller: Suggestion, msgs: Message[]): void { 88 | // Apply the changes locally. 89 | wrapper.update(() => { 90 | for (const msg of msgs) wrapper.applyMessage(msg); 91 | }); 92 | 93 | // Apply the changes remotely and to other suggestions. 94 | onLocalChange(msgs); 95 | 96 | suggestions.delete(caller); 97 | } 98 | 99 | function onReject(caller: Suggestion): void { 100 | suggestions.delete(caller); 101 | } 102 | 103 | /** 104 | * Sets the currently selected block(s) to the given type. 105 | */ 106 | function setBlockType(type: string): void { 107 | // Cursors point to the Position on their left. 108 | // Affect all block markers between those immediately left (inclusive) 109 | // of anchor and head. 110 | const sel = wrapper.getSelection(); 111 | let [start, end] = [sel.anchor, sel.head]; 112 | if (wrapper.order.compare(start, end) > 0) [start, end] = [end, start]; 113 | 114 | const startBlock = wrapper.blockMarkers.indexOfPosition(start, "left"); 115 | const endBlock = wrapper.blockMarkers.indexOfPosition(end, "left"); 116 | const entries = [...wrapper.blockMarkers.entries(startBlock, endBlock + 1)]; 117 | 118 | // If they all have the given type, toggle it off. Else toggle it on. 119 | let allHaveType = true; 120 | for (const [, existing] of entries) { 121 | if (existing.type !== type) { 122 | allHaveType = false; 123 | break; 124 | } 125 | } 126 | const typeToSet = allHaveType ? "paragraph" : type; 127 | 128 | wrapper.update(() => { 129 | for (const [blockPos, existing] of wrapper.blockMarkers.entries( 130 | startBlock, 131 | endBlock + 1 132 | )) { 133 | if (existing.type !== typeToSet) { 134 | const marker = { ...existing, type: typeToSet }; 135 | wrapper.setMarker(blockPos, marker); 136 | onLocalChange([{ type: "setMarker", pos: blockPos, marker }]); 137 | } 138 | } 139 | }); 140 | } 141 | 142 | // TODO: show suggestions as gray highlight in main doc; when a selection 143 | // is focused, emphasize its highlight. 144 | -------------------------------------------------------------------------------- /suggested-changes/src/site/schema.ts: -------------------------------------------------------------------------------- 1 | import { DOMOutputSpec, MarkSpec, Schema } from "prosemirror-model"; 2 | 3 | const pDOM: DOMOutputSpec = ["p", 0]; 4 | const h1DOM: DOMOutputSpec = ["h1", 0]; 5 | const h2DOM: DOMOutputSpec = ["h2", 0]; 6 | 7 | const emDOM: DOMOutputSpec = ["em", 0]; 8 | const strongDOM: DOMOutputSpec = ["strong", 0]; 9 | 10 | export const schema = new Schema({ 11 | nodes: { 12 | doc: { content: "block+" }, 13 | paragraph: { 14 | group: "block", 15 | content: "text*", 16 | marks: "_", 17 | parseDOM: [{ tag: "p" }], 18 | toDOM() { 19 | return pDOM; 20 | }, 21 | }, 22 | h1: { 23 | group: "block", 24 | content: "text*", 25 | marks: "_", 26 | parseDOM: [{ tag: "h1" }], 27 | toDOM() { 28 | return h1DOM; 29 | }, 30 | }, 31 | h2: { 32 | group: "block", 33 | content: "text*", 34 | marks: "_", 35 | parseDOM: [{ tag: "h2" }], 36 | toDOM() { 37 | return h2DOM; 38 | }, 39 | }, 40 | ul: { 41 | group: "block", 42 | content: "text*", 43 | marks: "_", 44 | attrs: { 45 | /** 46 | * This is only set in sync(), *not* stored in the collaborative state. 47 | */ 48 | symbol: { default: "•" }, 49 | }, 50 | // TODO: ul vs ol 51 | parseDOM: [{ tag: "li" }], 52 | toDOM(node) { 53 | return [ 54 | "p", 55 | { 56 | class: "fakeLi", 57 | style: `--before-content: "${node.attrs.symbol}"`, 58 | }, 59 | 0, 60 | ]; 61 | }, 62 | }, 63 | ol: { 64 | group: "block", 65 | content: "text*", 66 | marks: "_", 67 | attrs: { 68 | /** 69 | * This is only set in sync(), *not* stored in the collaborative state. 70 | */ 71 | symbol: { default: "1." }, 72 | }, 73 | toDOM(node) { 74 | return [ 75 | "p", 76 | { 77 | class: "fakeLi", 78 | style: `--before-content: "${node.attrs.symbol}"`, 79 | }, 80 | 0, 81 | ]; 82 | }, 83 | }, 84 | text: {}, 85 | }, 86 | marks: { 87 | // em and strong marks copied from prosemirror-schema-basic. 88 | 89 | /// An emphasis mark. Rendered as an `` element. Has parse rules 90 | /// that also match `` and `font-style: italic`. 91 | em: { 92 | parseDOM: [ 93 | { tag: "i" }, 94 | { tag: "em" }, 95 | { style: "font-style=italic" }, 96 | { style: "font-style=normal", clearMark: (m) => m.type.name == "em" }, 97 | ], 98 | toDOM() { 99 | return emDOM; 100 | }, 101 | } as MarkSpec, 102 | 103 | /// A strong mark. Rendered as ``, parse rules also match 104 | /// `` and `font-weight: bold`. 105 | strong: { 106 | parseDOM: [ 107 | { tag: "strong" }, 108 | // This works around a Google Docs misbehavior where 109 | // pasted content will be inexplicably wrapped in `` 110 | // tags with a font-weight normal. 111 | { 112 | tag: "b", 113 | getAttrs: (node: HTMLElement) => 114 | node.style.fontWeight != "normal" && null, 115 | }, 116 | { style: "font-weight=400", clearMark: (m) => m.type.name == "strong" }, 117 | { 118 | style: "font-weight", 119 | getAttrs: (value: string) => 120 | /^(bold(er)?|[5-9]\d{2,})$/.test(value) && null, 121 | }, 122 | ], 123 | toDOM() { 124 | return strongDOM; 125 | }, 126 | } as MarkSpec, 127 | }, 128 | }); 129 | -------------------------------------------------------------------------------- /suggested-changes/src/site/suggestion.ts: -------------------------------------------------------------------------------- 1 | import { TimestampFormatting } from "@list-positions/formatting"; 2 | import { 3 | List, 4 | MAX_POSITION, 5 | MIN_POSITION, 6 | Order, 7 | Position, 8 | positionEquals, 9 | } from "list-positions"; 10 | import { maybeRandomString } from "maybe-random-string"; 11 | import { BlockMarker } from "../common/block_text"; 12 | import { Message } from "../common/messages"; 13 | import { ProseMirrorWrapper } from "./prosemirror_wrapper"; 14 | 15 | export class Suggestion { 16 | readonly container: HTMLDivElement; 17 | readonly wrapper: ProseMirrorWrapper; 18 | readonly messages: Message[] = []; 19 | 20 | /** 21 | * Our range in origin.list is described as an *open* interval (beforePos, afterPos). 22 | * Openness makes it easy to handle prepends and appends, but from the user's 23 | * perspective, a closed interval makes more sense. 24 | * To emulate that, we use hacked beforePos/afterPos that try to stay next to 25 | * the closed interval even when there are concurrent edits next to 26 | * our range in origin.list. 27 | */ 28 | readonly beforePos: Position; 29 | readonly afterPos: Position; 30 | /** 31 | * The last block Position <= startPos. 32 | */ 33 | firstBlockPos: Position; 34 | 35 | constructor( 36 | parent: HTMLElement, 37 | readonly origin: ProseMirrorWrapper, 38 | private readonly onAccept: (caller: Suggestion, msgs: Message[]) => void, 39 | private readonly onReject: (caller: Suggestion) => void 40 | ) { 41 | this.container = document.createElement("div"); 42 | 43 | // Extract initial state: the selection in origin. 44 | let selStart = origin.list.indexOfCursor(origin.getSelection().anchor); 45 | let selEnd = origin.list.indexOfCursor(origin.getSelection().head); 46 | if (selStart > selEnd) [selStart, selEnd] = [selEnd, selStart]; 47 | if (selStart === selEnd) { 48 | throw new Error("Selection is empty"); 49 | } 50 | 51 | // Use a forked Order, for privacy (our metas are not synced until we merge the change) 52 | // and to prevent the origin from using one of our not-yet-synced metas as a dependency. 53 | const list = new List(new Order()); 54 | list.order.load(origin.order.save()); 55 | for (const [pos, value] of origin.list.entries(selStart, selEnd)) { 56 | list.set(pos, value); 57 | } 58 | // Also extract the block marker just before the selection, to format 59 | // the first block. 60 | const blockIndex = origin.blockMarkers.indexOfPosition( 61 | origin.list.positionAt(selStart), 62 | "left" 63 | ); 64 | this.firstBlockPos = origin.blockMarkers.positionAt(blockIndex); 65 | list.set(this.firstBlockPos, origin.blockMarkers.get(this.firstBlockPos)!); 66 | 67 | const formatting = new TimestampFormatting(list.order); 68 | // OPT: only extract formatting spans in range. 69 | formatting.load(origin.formatting.save()); 70 | 71 | // Set beforePos and afterPos. 72 | this.beforePos = this.createBeforePos(list.order, selStart); 73 | this.afterPos = this.createAfterPos(list.order, selEnd); 74 | 75 | // Construct our GUI. 76 | this.wrapper = new ProseMirrorWrapper( 77 | this.container, 78 | { refState: { list, formatting } }, 79 | (msgs) => this.messages.push(...msgs), 80 | { beforePos: this.beforePos, afterPos: this.afterPos } 81 | ); 82 | 83 | const buttonDiv = document.createElement("div"); 84 | const acceptButton = document.createElement("button"); 85 | acceptButton.innerText = "✅️"; 86 | acceptButton.onclick = () => this.accept(); 87 | buttonDiv.appendChild(acceptButton); 88 | // TODO: padding between 89 | const rejectButton = document.createElement("button"); 90 | rejectButton.innerText = "❌️"; 91 | rejectButton.onclick = () => this.reject(); 92 | buttonDiv.appendChild(rejectButton); 93 | this.container.appendChild(buttonDiv); 94 | 95 | parent.appendChild(this.container); 96 | } 97 | 98 | private createBeforePos(order: Order, selStart: number): Position { 99 | // The first position in our list. 100 | const startPos = this.origin.list.positionAt(selStart); 101 | 102 | // Create a Position before startPos that is hacked to be closer to it 103 | // than almost any concurrent or future Position. 104 | // We do so by creating a left child whose bunchID (tiebreaker) starts with "|", 105 | // which is < all other default bunchID chars. 106 | 107 | // If this hack was used before, a sibling might already start with "|". 108 | // Use more "|"s than it. 109 | // (Technically, we only need to consider siblings with the same offset, but we're lazy.) 110 | let existingPipes = 0; 111 | const parent = order.getNodeFor(startPos); 112 | for (let i = 0; i < parent.childrenLength; i++) { 113 | const sibling = parent.getChild(i); 114 | let siblingPipes = 0; 115 | for (const char of sibling.bunchID) { 116 | if (char === "|") siblingPipes++; 117 | } 118 | existingPipes = Math.max(existingPipes, siblingPipes); 119 | } 120 | 121 | // Make a bunchID out of the pipes and a random string. 122 | const pipes = new Array(existingPipes + 1).fill("|").join(""); 123 | const bunchID = pipes + maybeRandomString(); 124 | 125 | // Use a crafted Order.createPositions call that creates a new bunch as a 126 | // left child of startPos, according to the Fugue algorithm. 127 | const [beforePos, newMeta] = order.createPositions( 128 | MIN_POSITION, 129 | startPos, 130 | 1, 131 | { bunchID } 132 | ); 133 | 134 | // TODO: use dedicated meta message instead of this empty set. 135 | this.messages.push({ 136 | type: "set", 137 | startPos: beforePos, 138 | chars: "", 139 | meta: newMeta!, 140 | }); 141 | return beforePos; 142 | } 143 | 144 | private createAfterPos(order: Order, selEnd: number): Position { 145 | // The last position in our list. 146 | const endPosIncl = this.origin.list.positionAt(selEnd - 1); 147 | 148 | // Create a Position after endPosIncl that is hacked to be closer to it 149 | // than almost any concurrent or future Position. 150 | // We do so by creating a right child whose bunchID (tiebreaker) starts with " ", 151 | // which is < all other default bunchID chars. 152 | 153 | // If this hack was used before, a sibling might already start with " ". 154 | // Use more " "s than it. 155 | // (Technically, we only need to consider siblings with the same offset, but we're lazy.) 156 | let existingSpaces = 0; 157 | const parent = order.getNodeFor(endPosIncl); 158 | for (let i = 0; i < parent.childrenLength; i++) { 159 | const sibling = parent.getChild(i); 160 | let siblingSpaces = 0; 161 | for (const char of sibling.bunchID) { 162 | if (char === " ") siblingSpaces++; 163 | } 164 | existingSpaces = Math.max(existingSpaces, siblingSpaces); 165 | } 166 | 167 | // Make a bunchID out of the pipes and a random string. 168 | const spaces = new Array(existingSpaces + 1).fill(" ").join(""); 169 | const bunchID = spaces + maybeRandomString(); 170 | 171 | // Use a crafted Order.createPositions call that creates a new bunch as a 172 | // right child of endPosIncl, according to the Fugue algorithm. 173 | const [afterPos, newMeta] = order.createPositions( 174 | endPosIncl, 175 | MAX_POSITION, 176 | 1, 177 | { bunchID } 178 | ); 179 | 180 | // TODO: use dedicated meta message instead of this empty set. 181 | this.messages.push({ 182 | type: "set", 183 | startPos: afterPos, 184 | chars: "", 185 | meta: newMeta!, 186 | }); 187 | return afterPos; 188 | } 189 | 190 | isInRange(pos: Position): boolean { 191 | return ( 192 | this.wrapper.order.compare(this.beforePos, pos) < 0 && 193 | this.wrapper.order.compare(pos, this.afterPos) < 0 194 | ); 195 | } 196 | 197 | /** 198 | * Updates our state to reflect ops on the origin doc. 199 | */ 200 | applyOriginMessages(msgs: Message[]): void { 201 | this.wrapper.update(() => { 202 | for (const msg of msgs) { 203 | switch (msg.type) { 204 | case "set": 205 | if (msg.meta) { 206 | this.wrapper.order.addMetas([msg.meta]); 207 | } 208 | // Note: this assumes a new position, so that it's either all in range or all not. 209 | if (this.isInRange(msg.startPos)) { 210 | this.wrapper.set(msg.startPos, msg.chars); 211 | } 212 | break; 213 | case "setMarker": 214 | // meta is already applied via the origin's set method. 215 | // TODO: for case of == this.firstBlockPos, if the suggestion has changed 216 | // the block type, ignore it so that the suggesion's addition can win? 217 | // Or re-do our own block marker set, so it will LWW win in the end. 218 | if ( 219 | positionEquals(msg.pos, this.firstBlockPos) || 220 | this.isInRange(msg.pos) 221 | ) { 222 | this.wrapper.setMarker(msg.pos, msg.marker); 223 | } 224 | break; 225 | case "delete": 226 | if (positionEquals(msg.pos, this.firstBlockPos)) { 227 | // Before deleting, need to fill in the previous blockPos, so that 228 | // wrapper.list always starts with a block marker. 229 | const curIndex = this.origin.blockMarkers.indexOfPosition( 230 | this.firstBlockPos, 231 | "right" 232 | ); 233 | this.firstBlockPos = this.origin.blockMarkers.positionAt( 234 | curIndex - 1 235 | ); 236 | this.wrapper.setMarker( 237 | this.firstBlockPos, 238 | this.origin.blockMarkers.get(this.firstBlockPos)! 239 | ); 240 | } 241 | this.wrapper.delete(msg.pos); 242 | break; 243 | case "mark": 244 | // OPT: only add formatting spans in range. 245 | this.wrapper.addMark(msg.mark); 246 | break; 247 | default: 248 | console.error("Unexpected message type:", msg.type, msg); 249 | } 250 | } 251 | }); 252 | } 253 | 254 | private accept(): void { 255 | this.onAccept(this, this.messages); 256 | this.destroy(); 257 | } 258 | 259 | private reject(): void { 260 | this.onReject(this); 261 | this.destroy(); 262 | } 263 | 264 | private destroy() { 265 | this.container.parentElement?.removeChild(this.container); 266 | this.wrapper.view.destroy(); 267 | } 268 | } 269 | -------------------------------------------------------------------------------- /suggested-changes/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | /* Recommend by Webpack. */ 5 | "module": "ES6", 6 | /* Needed with module: ES6 or else compilation breaks. */ 7 | "moduleResolution": "node", 8 | /* Enable strict type checking. */ 9 | "strict": true, 10 | /* Prevent errors caused by other libraries. */ 11 | "skipLibCheck": true, 12 | /* Enable interop with dependencies using different module systems. */ 13 | "esModuleInterop": true, 14 | /* We don't need to emit declarations. */ 15 | "declaration": false, 16 | /* Emit sourcemap files. */ 17 | "sourceMap": true, 18 | "rootDir": "src" 19 | }, 20 | "include": ["src/site", "src/common", "src/site/old.ts"] 21 | } 22 | -------------------------------------------------------------------------------- /suggested-changes/tsconfig.server.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "ES6", 5 | /* Enable strict type checking. */ 6 | "strict": true, 7 | /* Prevent errors caused by other libraries. */ 8 | "skipLibCheck": true, 9 | /* Enable interop with dependencies using different module systems. */ 10 | "esModuleInterop": true, 11 | /* Use tsc (npm run build) to typecheck only. */ 12 | "noEmit": true, 13 | "rootDir": "src" 14 | }, 15 | "include": ["src/server", "src/common"] 16 | } 17 | -------------------------------------------------------------------------------- /suggested-changes/tsconfig.webpack-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "ES6", 5 | /* Enable strict type checking. */ 6 | "strict": true, 7 | /* Prevent errors caused by other libraries. */ 8 | "skipLibCheck": true, 9 | /* Enable interop with dependencies using different module systems. */ 10 | "esModuleInterop": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /suggested-changes/webpack.config.ts: -------------------------------------------------------------------------------- 1 | import HtmlWebpackPlugin from "html-webpack-plugin"; 2 | import * as path from "path"; 3 | import * as webpack from "webpack"; 4 | 5 | // Basic Webpack config for TypeScript, based on 6 | // https://webpack.js.org/guides/typescript/ . 7 | const config: webpack.Configuration = { 8 | // mode and devtool are overridden by `npm run build` for production mode. 9 | mode: "development", 10 | devtool: "eval-source-map", 11 | entry: "./src/site/main.ts", 12 | module: { 13 | rules: [ 14 | { 15 | test: /\.tsx?$/, 16 | use: "ts-loader", 17 | exclude: /node_modules/, 18 | }, 19 | { 20 | test: /\.js$/, 21 | enforce: "pre", 22 | use: ["source-map-loader"], 23 | }, 24 | { 25 | test: /\.css$/, 26 | use: ["style-loader", "css-loader"], 27 | }, 28 | ], 29 | }, 30 | resolve: { 31 | extensions: [".tsx", ".ts", ".js"], 32 | }, 33 | output: { 34 | filename: "[name].bundle.js", 35 | path: path.resolve(__dirname, "dist"), 36 | clean: true, 37 | }, 38 | plugins: [ 39 | // Use src/index.html as the entry point. 40 | new HtmlWebpackPlugin({ 41 | template: "./src/site/index.html", 42 | }), 43 | ], 44 | }; 45 | 46 | export default config; 47 | -------------------------------------------------------------------------------- /triplit-quill/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | .env 15 | 16 | # Editor directories and files 17 | .vscode/* 18 | !.vscode/extensions.json 19 | .idea 20 | .DS_Store 21 | *.suo 22 | *.ntvs* 23 | *.njsproj 24 | *.sln 25 | *.sw? 26 | -------------------------------------------------------------------------------- /triplit-quill/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2024 Matthew Weidner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /triplit-quill/README.md: -------------------------------------------------------------------------------- 1 | # Triplit-Quill 2 | 3 | Basic collaborative rich-text editor using [list-positions](https://github.com/mweidner037/list-positions#readme) and [@list-positions/formatting](https://github.com/mweidner037/list-positions-formatting#readme), the [Triplit](https://www.triplit.dev/) fullstack database, and [Quill](https://quilljs.com/). 4 | 5 | The editor state is stored in a Triplit database with three tables: 6 | 7 | - `bunches` for list-positions's [BunchMeta](https://github.com/mweidner037/list-positions#managing-metadata). 8 | - `values` for the values (characters). For simplicity, each character gets its own row. (It's probably possible to instead store one row per bunch instead, using an `S.Set` to track which chars are present/deleted.) 9 | - `marks` for the formatting marks. 10 | 11 | See `triplit/schema.ts`. 12 | 13 | Local updates are synced to the local database. When any table changes, a [subscription](https://www.triplit.dev/docs/fetching-data/subscriptions) in `src/main.ts` updates the Quill state. Since subscriptions are not incremental (they always return the whole state), we diff against the previous state to figure out what changed. 14 | 15 | > Note: Rapidly inserting/deleting characters (by holding down a keyboard key) currently causes some weird behaviors. 16 | 17 | ## Setup 18 | 19 | 1. Install with `npm i`. 20 | 2. (Optional) To sync to the Triplit cloud, create a `.env` file with the content given in your Triplit project's dashboard (Vite version). Note that `.env` is gitignored. 21 | 22 | ## Commands 23 | 24 | These are unchanged from the `npm create triplit-app` setup (vanilla version). 25 | 26 | ### `npm run dev` 27 | 28 | Start a Vite development server with auto-reloading. 29 | 30 | ### `npm run build` 31 | 32 | Build the app to `dist/`. 33 | 34 | ### `npm run preview` 35 | 36 | Preview the app built to `dist/`. 37 | -------------------------------------------------------------------------------- /triplit-quill/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Triplit-Quill 8 | 9 | 10 | 11 |
12 | Info and source code 16 |
17 |
18 | 19 | 20 |
21 |
22 |
23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /triplit-quill/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "triplit-quill", 3 | "private": true, 4 | "version": "0.0.0", 5 | "license": "MIT", 6 | "type": "module", 7 | "scripts": { 8 | "dev": "vite", 9 | "build": "tsc && vite build", 10 | "preview": "vite preview" 11 | }, 12 | "devDependencies": { 13 | "typescript": "^5.2.2", 14 | "vite": "^5.0.8", 15 | "@triplit/cli": "^0.3.1" 16 | }, 17 | "dependencies": { 18 | "@triplit/client": "^0.3.1", 19 | "@list-positions/formatting": "^1.0.0", 20 | "list-positions": "^1.0.0", 21 | "quill": "^2.0.2" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /triplit-quill/src/main.ts: -------------------------------------------------------------------------------- 1 | import { ClientFetchResult, TriplitClient } from "@triplit/client"; 2 | import { RichText } from "@list-positions/formatting"; 3 | import { 4 | MAX_POSITION, 5 | MIN_POSITION, 6 | Position, 7 | expandPositions, 8 | } from "list-positions"; 9 | import { schema } from "../triplit/schema"; 10 | import { QuillWrapper, WrapperOp } from "./quill_wrapper"; 11 | 12 | const client = new TriplitClient({ 13 | schema, 14 | serverUrl: import.meta.env.VITE_TRIPLIT_SERVER_URL, 15 | token: import.meta.env.VITE_TRIPLIT_TOKEN, 16 | }); 17 | 18 | const quillWrapper = new QuillWrapper(onLocalOps, makeInitialState()); 19 | 20 | // Send Triplit changes to Quill. 21 | // Since queries are not incremental, we diff against the previous state 22 | // and process changed (inserted/deleted) ids. 23 | // Note that this will also capture local changes; quillWrapper will ignore 24 | // those as redundant. 25 | 26 | const bunches = client.query("bunches").build(); 27 | let lastBunchResults: ClientFetchResult = new Map(); 28 | client.subscribe(bunches, (results) => { 29 | const ops: WrapperOp[] = []; 30 | for (const [id, row] of results) { 31 | if (!lastBunchResults.has(id)) { 32 | // Process inserted row. 33 | ops.push({ 34 | type: "meta", 35 | meta: { bunchID: row.id, parentID: row.parentID, offset: row.offset }, 36 | }); 37 | } 38 | } 39 | // Rows are never deleted, so no need to diff those. 40 | lastBunchResults = results; 41 | // TODO: are rows guaranteed to be in causal order? 42 | // Since we batch the applyOps call, it's okay if not, so long as the 43 | // whole table is causally consistent. 44 | quillWrapper.applyOps(ops); 45 | }); 46 | 47 | const values = client.query("values").build(); 48 | let lastValuesResults: ClientFetchResult = new Map(); 49 | client.subscribe(values, (results) => { 50 | const ops: WrapperOp[] = []; 51 | for (const [id, row] of results) { 52 | if (!lastValuesResults.has(id)) { 53 | // Process inserted row. 54 | ops.push({ 55 | type: "set", 56 | startPos: { bunchID: row.bunchID, innerIndex: row.innerIndex }, 57 | chars: row.value, 58 | }); 59 | } 60 | } 61 | // Diff in the other direction to find deleted rows. 62 | for (const [id, row] of lastValuesResults) { 63 | if (!results.has(id)) { 64 | // Process deleted row. 65 | ops.push({ 66 | type: "delete", 67 | pos: { bunchID: row.bunchID, innerIndex: row.innerIndex }, 68 | }); 69 | } 70 | } 71 | lastValuesResults = results; 72 | // TODO: Are value & mark rows guaranteed to be updated after the bunch rows 73 | // that they depend on, given that our tx does so? 74 | // If not, we might get errors from missing BunchMeta dependencies. 75 | quillWrapper.applyOps(ops); 76 | }); 77 | 78 | const marks = client.query("marks").build(); 79 | let lastMarksResults: ClientFetchResult = new Map(); 80 | client.subscribe(marks, (results) => { 81 | const ops: WrapperOp[] = []; 82 | for (const [id, row] of results) { 83 | if (!lastMarksResults.has(id)) { 84 | // Process inserted row. 85 | ops.push({ 86 | type: "mark", 87 | mark: { 88 | start: { 89 | pos: { bunchID: row.startBunchID, innerIndex: row.startInnerIndex }, 90 | before: row.startBefore, 91 | }, 92 | end: { 93 | pos: { bunchID: row.endBunchID, innerIndex: row.endInnerIndex }, 94 | before: row.endBefore, 95 | }, 96 | key: row.key, 97 | value: JSON.parse(row.value), 98 | creatorID: row.creatorID, 99 | timestamp: row.timestamp, 100 | }, 101 | }); 102 | } 103 | } 104 | // Rows are never deleted, so no need to diff those. 105 | lastMarksResults = results; 106 | quillWrapper.applyOps(ops); 107 | }); 108 | 109 | // Send Quill changes to Triplit. 110 | // Use a queue to avoid overlapping transactions (since onLocalOps is sync 111 | // but transactions are async). 112 | 113 | // TODO: Despite avoiding overlapping transactions and explicit fetches, I still 114 | // get ReadWriteConflictErrors if I type/delete quickly (by holding down a 115 | // keyboard key). Are tx writes conflicting with subscribe's reads? 116 | 117 | let localOpsQueue: WrapperOp[] = []; 118 | let sendingLocalOps = false; 119 | function onLocalOps(ops: WrapperOp[]) { 120 | localOpsQueue.push(...ops); 121 | if (!sendingLocalOps) void sendLocalOps(); 122 | } 123 | 124 | async function sendLocalOps() { 125 | sendingLocalOps = true; 126 | try { 127 | while (localOpsQueue.length !== 0) { 128 | const ops = localOpsQueue; 129 | localOpsQueue = []; 130 | await client.transact(async (tx) => { 131 | for (const op of ops) { 132 | switch (op.type) { 133 | case "meta": 134 | await tx.insert("bunches", { 135 | id: op.meta.bunchID, 136 | parentID: op.meta.parentID, 137 | offset: op.meta.offset, 138 | }); 139 | break; 140 | case "set": 141 | let i = 0; 142 | for (const pos of expandPositions(op.startPos, op.chars.length)) { 143 | await tx.insert("values", { 144 | id: idOfPos(pos), 145 | bunchID: pos.bunchID, 146 | innerIndex: pos.innerIndex, 147 | value: op.chars[i], 148 | }); 149 | i++; 150 | } 151 | break; 152 | case "delete": 153 | await tx.delete("values", idOfPos(op.pos)); 154 | break; 155 | case "mark": 156 | await tx.insert("marks", { 157 | startBunchID: op.mark.start.pos.bunchID, 158 | startInnerIndex: op.mark.start.pos.innerIndex, 159 | startBefore: op.mark.start.before, 160 | endBunchID: op.mark.end.pos.bunchID, 161 | endInnerIndex: op.mark.end.pos.innerIndex, 162 | endBefore: op.mark.end.before, 163 | key: op.mark.key, 164 | value: JSON.stringify(op.mark.value), 165 | creatorID: op.mark.creatorID, 166 | timestamp: op.mark.timestamp, 167 | }); 168 | break; 169 | } 170 | } 171 | }); 172 | } 173 | } finally { 174 | sendingLocalOps = false; 175 | } 176 | } 177 | 178 | /** 179 | * Fake initial saved state that's identical on all replicas: a single 180 | * "\n", to match Quill's initial state. 181 | */ 182 | function makeInitialState() { 183 | const richText = new RichText(); 184 | // Use the same bunchID & BunchMeta on all replicas. 185 | const [pos] = richText.order.createPositions(MIN_POSITION, MAX_POSITION, 1, { 186 | bunchID: "INIT", 187 | }); 188 | richText.text.set(pos, "\n"); 189 | return richText.save(); 190 | } 191 | 192 | function idOfPos(pos: Position): string { 193 | return `${pos.innerIndex},${pos.bunchID}`; 194 | } 195 | -------------------------------------------------------------------------------- /triplit-quill/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /triplit-quill/triplit/schema.ts: -------------------------------------------------------------------------------- 1 | import { Schema as S } from "@triplit/db"; 2 | 3 | /** 4 | * Define your schema here. To use your schema, you can either: 5 | * - Directly import your schema into your app 6 | * - Run 'triplit migrate create' to generate migrations (recommended for production apps) 7 | * 8 | * For more information on schemas, see the docs: https://www.triplit.dev/docs/schemas 9 | */ 10 | export const schema = { 11 | bunches: { 12 | schema: S.Schema({ 13 | // The bunchID. 14 | id: S.Id(), 15 | parentID: S.String(), 16 | offset: S.Number(), 17 | }), 18 | }, 19 | values: { 20 | schema: S.Schema({ 21 | // A concatenation of bunchID and innerIndex (idOfPos), so that we can 22 | // delete a Position without doing a fetch for the ID. 23 | id: S.Id(), 24 | // Foreign key @ bunches table. 25 | bunchID: S.String(), 26 | innerIndex: S.Number(), 27 | value: S.String(), 28 | }), 29 | }, 30 | marks: { 31 | schema: S.Schema({ 32 | // Unused. 33 | id: S.Id(), 34 | // TODO: use nested records for anchors? 35 | // Foreign key @ bunches table. 36 | startBunchID: S.String(), 37 | startInnerIndex: S.Number(), 38 | startBefore: S.Boolean(), 39 | // Foreign key @ bunches table. 40 | endBunchID: S.String(), 41 | endInnerIndex: S.Number(), 42 | endBefore: S.Boolean(), 43 | key: S.String(), 44 | // JSON-ified any. 45 | value: S.String(), 46 | creatorID: S.String(), 47 | timestamp: S.Number(), 48 | }), 49 | }, 50 | }; 51 | -------------------------------------------------------------------------------- /triplit-quill/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "useDefineForClassFields": true, 5 | "module": "ESNext", 6 | "lib": ["ES2020", "DOM", "DOM.Iterable"], 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "resolveJsonModule": true, 13 | "isolatedModules": true, 14 | "noEmit": true, 15 | 16 | /* Linting */ 17 | "strict": true, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "noFallthroughCasesInSwitch": true 21 | }, 22 | "include": ["src"] 23 | } 24 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/.prettierignore: -------------------------------------------------------------------------------- 1 | /dist/* 2 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2024 Matthew Weidner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/README.md: -------------------------------------------------------------------------------- 1 | # WebSocket-Prosemirror-Blocks 2 | 3 | A basic collaborative rich-text editor using [list-positions](https://github.com/mweidner037/list-positions#readme) and [@list-positions/formatting](https://github.com/mweidner037/list-positions-formatting#readme), a WebSocket server, and [ProseMirror](https://prosemirror.net/). It supports only a simple block-based schema, stored in a "list-positions-native" way. 4 | 5 | When a client makes a change, a description is sent to the server in JSON format. The server echoes that change to all other connected clients. The server also updates its own copy of the rich-text state; this is sent to new clients when they load the page. 6 | 7 | A client optimistically updates its own state before sending its change to the server. The demo is simple enough that these optimistic updates are always "correct" (match the server's eventual state) - see the comments in [`src/server/rich_text_server.ts`](./src/server/rich_text_server.ts). A more complicated app might need to "repair" optimistic updates that are rejected or modified by the server, e.g., due to permissions issues. 8 | 9 | The collaborative state is linear, not a tree like ProseMirror's own state; it is stored as Lists in [`ProsemirrorWrapper`](./src/site/prosemirror_wrapper.ts). The collaborative state uses special "block markers" to indicate the start of each block and its type (paragraph, h1, h2, ul, ol). In particular, bullets and numbering are stored as a series of unordered or ordered list blocks, with no explicit list start/end; at render time, we fill in numbers and render the numbers/bullets using a CSS ::before element on a normal paragraph. 10 | 11 | The ProseMirror wrapper uses its copy of the collaborative state as the source-of-truth for ProseMirror. Whenever that state changes, `ProsemirrorWrapper.sync()` recomputes the ProseMirror state and sends it to ProseMirror. When ProseMirror generates a transaction due to a local change (e.g. typing), `ProsemirrorWrapper.onLocalTr` converts that transaction into changes to the collaborative state, then updates the server and calls `ProsemirrorWrapper.sync()`. Calling `sync()` is technically redundant, but it ensures that the two states don't diverge, and it makes the data flow consistent between local vs remote changes. 12 | 13 | _References: unpublished notes by Martin Kleppmann (2022); [Notion's data model](https://www.notion.so/blog/data-model-behind-notion); [y-prosemirror](https://github.com/yjs/y-prosemirror)_ 14 | 15 | Code organization: 16 | 17 | - `src/common/`: Messages shared between clients and the server. 18 | - `src/server/`: WebSocket server. 19 | - `src/site/`: ProseMirror client. 20 | 21 | ## Installation 22 | 23 | First, install [Node.js](https://nodejs.org/). Then run `npm i`. 24 | 25 | ## Commands 26 | 27 | ### `npm run dev` 28 | 29 | Build the app from `src/`, in [development mode](https://webpack.js.org/guides/development/). 30 | 31 | ### `npm run build` 32 | 33 | Build the app from `src/`, in [production mode](https://webpack.js.org/guides/production/). 34 | 35 | ### `npm start` 36 | 37 | Run the server on [http://localhost:3000/](http://localhost:3000/). Use multiple browser windows at once to test collaboration. 38 | 39 | To change the port, set the `$PORT` environment variable. 40 | 41 | ### `npm run clean` 42 | 43 | Delete `dist/`. 44 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "license": "MIT", 3 | "dependencies": { 4 | "express": "^4.18.2", 5 | "@list-positions/formatting": "^1.0.0", 6 | "list-positions": "^1.0.0", 7 | "maybe-random-string": "^1.0.0", 8 | "prosemirror-commands": "^1.5.2", 9 | "prosemirror-keymap": "^1.2.2", 10 | "prosemirror-model": "^1.19.4", 11 | "prosemirror-state": "^1.4.3", 12 | "prosemirror-transform": "^1.8.0", 13 | "prosemirror-view": "^1.32.7", 14 | "ws": "^8.13.0" 15 | }, 16 | "devDependencies": { 17 | "@types/express": "^4.17.17", 18 | "@types/node": "^15.6.1", 19 | "@types/webpack": "^5.28.0", 20 | "@types/webpack-env": "^1.16.2", 21 | "@types/ws": "^8.5.10", 22 | "copy-webpack-plugin": "^11.0.0", 23 | "cross-env": "^7.0.3", 24 | "css-loader": "^6.2.0", 25 | "html-webpack-plugin": "^5.3.2", 26 | "npm-run-all": "^4.1.5", 27 | "prettier": "^2.2.1", 28 | "rimraf": "^2.7.1", 29 | "source-map-loader": "^3.0.0", 30 | "style-loader": "^3.3.3", 31 | "ts-loader": "^9.2.5", 32 | "ts-node": "^10.1.0", 33 | "typescript": "^4.3.5", 34 | "webpack": "^5.50.0", 35 | "webpack-cli": "^4.10.0" 36 | }, 37 | "scripts": { 38 | "start": "ts-node -P tsconfig.server.json src/server/server.ts", 39 | "dev": "cross-env TS_NODE_PROJECT='tsconfig.webpack-config.json' webpack", 40 | "build": "cross-env TS_NODE_PROJECT='tsconfig.webpack-config.json' webpack --mode production --devtool source-map", 41 | "test": "npm-run-all test:*", 42 | "test:server-tsc": "tsc -p tsconfig.server.json", 43 | "test:format": "prettier --check .", 44 | "fix": "npm-run-all fix:*", 45 | "fix:format": "prettier --write .", 46 | "clean": "rimraf dist" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/src/common/block_text.ts: -------------------------------------------------------------------------------- 1 | import { TimestampFormattingSavedState } from "@list-positions/formatting"; 2 | import { ListSavedState, OrderSavedState } from "list-positions"; 3 | 4 | /** 5 | * Immutable - don't mutate attrs directly. 6 | */ 7 | export type BlockMarker = { 8 | readonly type: string; 9 | readonly attrs?: Record; 10 | /** 11 | * Lamport timestamp for LWW. 12 | */ 13 | readonly timestamp: number; 14 | readonly creatorID: string; 15 | }; 16 | 17 | export type BlockTextSavedState = { 18 | readonly order: OrderSavedState; 19 | readonly text: ListSavedState; 20 | readonly blockMarkers: ListSavedState; 21 | readonly formatting: TimestampFormattingSavedState; 22 | }; 23 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/src/common/messages.ts: -------------------------------------------------------------------------------- 1 | import { TimestampMark } from "@list-positions/formatting"; 2 | import { BunchMeta, Position } from "list-positions"; 3 | import { BlockMarker, BlockTextSavedState } from "./block_text"; 4 | 5 | export type SetMessage = { 6 | type: "set"; 7 | startPos: Position; 8 | chars: string; 9 | meta?: BunchMeta; 10 | }; 11 | 12 | export type SetMarkerMessage = { 13 | type: "setMarker"; 14 | pos: Position; 15 | marker: BlockMarker; 16 | meta?: BunchMeta; 17 | }; 18 | 19 | export type DeleteMessage = { 20 | type: "delete"; 21 | pos: Position; 22 | }; 23 | 24 | export type MarkMessage = { 25 | type: "mark"; 26 | mark: TimestampMark; 27 | }; 28 | 29 | export type WelcomeMessage = { 30 | type: "welcome"; 31 | savedState: BlockTextSavedState; 32 | }; 33 | 34 | export type Message = 35 | | SetMessage 36 | | SetMarkerMessage 37 | | DeleteMessage 38 | | MarkMessage 39 | | WelcomeMessage; 40 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/src/server/rich_text_server.ts: -------------------------------------------------------------------------------- 1 | import { TimestampMark } from "@list-positions/formatting"; 2 | import { List, Order } from "list-positions"; 3 | import { WebSocket, WebSocketServer } from "ws"; 4 | import { BlockMarker } from "../common/block_text"; 5 | import { Message } from "../common/messages"; 6 | 7 | const heartbeatInterval = 30000; 8 | 9 | export class RichTextServer { 10 | // To easily save and send the state to new clients, store as Lists. 11 | private readonly order: Order; 12 | private readonly text: List; 13 | private readonly blockMarkers: List; 14 | // We don't need to inspect the formatting, so just store the marks directly. 15 | private readonly marks: TimestampMark[]; 16 | 17 | private clients = new Set(); 18 | 19 | constructor(readonly wss: WebSocketServer) { 20 | this.order = new Order(); 21 | this.text = new List(this.order); 22 | this.blockMarkers = new List(this.order); 23 | this.marks = []; 24 | 25 | // Initial state: a single paragraph, to match Prosemirror's starting state. 26 | this.blockMarkers.insertAt(0, { 27 | type: "paragraph", 28 | timestamp: 1, 29 | creatorID: "INIT", 30 | }); 31 | 32 | this.wss.on("connection", (ws) => { 33 | if (ws.readyState === WebSocket.OPEN) { 34 | this.wsOpen(ws); 35 | } else ws.on("open", () => this.wsOpen(ws)); 36 | ws.on("message", (data) => this.wsReceive(ws, data.toString())); 37 | ws.on("close", () => this.wsClose(ws)); 38 | ws.on("error", (err) => { 39 | console.error(err); 40 | this.wsClose(ws); 41 | }); 42 | }); 43 | } 44 | 45 | private sendMessage(ws: WebSocket, msg: Message) { 46 | if (ws.readyState == WebSocket.OPEN) { 47 | ws.send(JSON.stringify(msg)); 48 | } 49 | } 50 | 51 | private echo(origin: WebSocket, data: string) { 52 | for (const ws of this.clients) { 53 | if (ws === origin) continue; 54 | if (ws.readyState == WebSocket.OPEN) { 55 | ws.send(data); 56 | } 57 | } 58 | } 59 | 60 | private wsOpen(ws: WebSocket) { 61 | this.startHeartbeats(ws); 62 | 63 | // Send the current state. 64 | this.sendMessage(ws, { 65 | type: "welcome", 66 | savedState: { 67 | order: this.order.save(), 68 | text: this.text.save(), 69 | blockMarkers: this.blockMarkers.save(), 70 | formatting: this.marks, 71 | }, 72 | }); 73 | 74 | this.clients.add(ws); 75 | } 76 | 77 | /** 78 | * Ping to keep connection alive. 79 | * 80 | * This is necessary on at least Heroku, which has a 55 second timeout: 81 | * https://devcenter.heroku.com/articles/websockets#timeouts 82 | */ 83 | private startHeartbeats(ws: WebSocket) { 84 | const interval = setInterval(() => { 85 | if (ws.readyState === WebSocket.OPEN) { 86 | ws.ping(); 87 | } else clearInterval(interval); 88 | }, heartbeatInterval); 89 | } 90 | 91 | private wsReceive(ws: WebSocket, data: string) { 92 | const msg = JSON.parse(data) as Message; 93 | switch (msg.type) { 94 | case "set": 95 | if (msg.meta) this.order.addMetas([msg.meta]); 96 | this.text.set(msg.startPos, ...msg.chars); 97 | this.echo(ws, data); 98 | // Because a Position is only ever set once (when it's created) and 99 | // the server does no validation, the origin's optimistically-updated 100 | // state is already correct: msg.startPos is set to msg.chars. 101 | // If that were not true, we would need to send a message to origin 102 | // telling it how to repair its optimistically-updated state. 103 | break; 104 | case "setMarker": 105 | if (msg.meta) this.order.addMetas([msg.meta]); 106 | this.blockMarkers.set(msg.pos, msg.marker); 107 | this.echo(ws, data); 108 | // Because a Position is only ever set once (when it's created) and 109 | // the server does no validation, the origin's optimistically-updated 110 | // state is already correct: msg.pos is set to msg.marker. 111 | // If that were not true, we would need to send a message to origin 112 | // telling it how to repair its optimistically-updated state. 113 | break; 114 | case "delete": 115 | // Pos might belong to either list; try to delete from both. 116 | this.text.delete(msg.pos); 117 | this.blockMarkers.delete(msg.pos); 118 | this.echo(ws, data); 119 | // Because deletes are permanant and the server does no validation, 120 | // the origin's optimistically-updated state is already correct. 121 | break; 122 | case "mark": 123 | this.marks.push(msg.mark); 124 | this.echo(ws, data); 125 | // Because marks are permanant and the server does no validation, 126 | // the origin's optimistically-updated state is already correct. 127 | break; 128 | default: 129 | throw new Error("Unknown message type: " + msg.type); 130 | } 131 | } 132 | 133 | private wsClose(ws: WebSocket) { 134 | this.clients.delete(ws); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/src/server/server.ts: -------------------------------------------------------------------------------- 1 | import express from "express"; 2 | import path from "path"; 3 | import { WebSocketServer } from "ws"; 4 | import { RichTextServer } from "./rich_text_server"; 5 | 6 | const port = process.env.PORT || 3000; 7 | 8 | // Server dist/ with a simple express server. 9 | const app = express(); 10 | app.use("/", express.static(path.join(__dirname, "../../dist"))); 11 | const server = app.listen(port, () => 12 | console.log(`Listening at http://localhost:${port}/`) 13 | ); 14 | 15 | // Run the WebSocket server. 16 | const wss = new WebSocketServer({ server }); 17 | new RichTextServer(wss); 18 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/src/site/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 30 | 31 | WebSocket-Prosemirror-Blocks 32 | 33 | 34 | 35 |
36 | Info and source code 40 |
41 |
42 | 43 | 44 |
45 | 46 | 47 | 48 | 49 |
50 | 51 |
52 | 53 | 54 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/src/site/main.ts: -------------------------------------------------------------------------------- 1 | import { Message } from "../common/messages"; 2 | import { ProseMirrorWrapper } from "./prosemirror_wrapper"; 3 | 4 | const wsURL = location.origin.replace(/^http/, "ws"); 5 | const ws = new WebSocket(wsURL); 6 | 7 | function welcomeListener(e: MessageEvent) { 8 | const msg = JSON.parse(e.data) as Message; 9 | if (msg.type === "welcome") { 10 | // Got the initial state. Start ProseMirror. 11 | ws.removeEventListener("message", welcomeListener); 12 | const wrapper = new ProseMirrorWrapper(msg.savedState, onLocalChange); 13 | ws.addEventListener("message", (e: MessageEvent) => { 14 | onMessage(e, wrapper); 15 | }); 16 | 17 | for (const type of ["h1", "h2", "ul", "ol"]) { 18 | document.getElementById("button_" + type)!.onclick = () => 19 | setBlockType(wrapper, type); 20 | } 21 | } else { 22 | console.error("Received non-welcome message first: " + msg.type); 23 | } 24 | } 25 | ws.addEventListener("message", welcomeListener); 26 | 27 | // For this basic demo, we don't allow disconnection tests or 28 | // attempt to reconnect the WebSocket ever. 29 | // That would require buffering updates and/or logic to 30 | // "merge" in the Welcome state received after reconnecting. 31 | 32 | function onLocalChange(msgs: Message[]) { 33 | send(msgs); 34 | } 35 | 36 | function send(msgs: Message[]): void { 37 | if (ws.readyState === WebSocket.OPEN) { 38 | for (const msg of msgs) { 39 | ws.send(JSON.stringify(msg)); 40 | } 41 | } 42 | } 43 | 44 | // OPT: batch delivery, wrapped in wrapper.update(). 45 | function onMessage(e: MessageEvent, wrapper: ProseMirrorWrapper): void { 46 | const msg = JSON.parse(e.data) as Message; 47 | switch (msg.type) { 48 | case "set": 49 | if (msg.meta) wrapper.order.addMetas([msg.meta]); 50 | wrapper.set(msg.startPos, msg.chars); 51 | break; 52 | case "setMarker": 53 | if (msg.meta) wrapper.order.addMetas([msg.meta]); 54 | wrapper.setMarker(msg.pos, msg.marker); 55 | break; 56 | case "delete": 57 | wrapper.delete(msg.pos); 58 | break; 59 | case "mark": 60 | wrapper.addMark(msg.mark); 61 | break; 62 | default: 63 | console.error("Unexpected message type:", msg.type, msg); 64 | } 65 | } 66 | 67 | /** 68 | * Sets the currently selected block(s) to the given type. 69 | */ 70 | function setBlockType(wrapper: ProseMirrorWrapper, type: string): void { 71 | // Cursors point to the Position on their left. 72 | // Affect all block markers between those immediately left (inclusive) 73 | // of anchor and head. 74 | const sel = wrapper.getSelection(); 75 | let [start, end] = [sel.anchor, sel.head]; 76 | if (wrapper.order.compare(start, end) > 0) [start, end] = [end, start]; 77 | 78 | const startBlock = wrapper.blockMarkers.indexOfPosition(start, "left"); 79 | const endBlock = wrapper.blockMarkers.indexOfPosition(end, "left"); 80 | const entries = [...wrapper.blockMarkers.entries(startBlock, endBlock + 1)]; 81 | 82 | // If they all have the given type, toggle it off. Else toggle it on. 83 | let allHaveType = true; 84 | for (const [, existing] of entries) { 85 | if (existing.type !== type) { 86 | allHaveType = false; 87 | break; 88 | } 89 | } 90 | const typeToSet = allHaveType ? "paragraph" : type; 91 | 92 | wrapper.update(() => { 93 | for (const [blockPos, existing] of wrapper.blockMarkers.entries( 94 | startBlock, 95 | endBlock + 1 96 | )) { 97 | if (existing.type !== typeToSet) { 98 | const marker = { ...existing, type: typeToSet }; 99 | wrapper.setMarker(blockPos, marker); 100 | send([{ type: "setMarker", pos: blockPos, marker }]); 101 | } 102 | } 103 | }); 104 | } 105 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/src/site/schema.ts: -------------------------------------------------------------------------------- 1 | import { DOMOutputSpec, MarkSpec, Schema } from "prosemirror-model"; 2 | 3 | const pDOM: DOMOutputSpec = ["p", 0]; 4 | const h1DOM: DOMOutputSpec = ["h1", 0]; 5 | const h2DOM: DOMOutputSpec = ["h2", 0]; 6 | 7 | const emDOM: DOMOutputSpec = ["em", 0]; 8 | const strongDOM: DOMOutputSpec = ["strong", 0]; 9 | 10 | export const schema = new Schema({ 11 | nodes: { 12 | doc: { content: "block+" }, 13 | paragraph: { 14 | group: "block", 15 | content: "text*", 16 | marks: "_", 17 | parseDOM: [{ tag: "p" }], 18 | toDOM() { 19 | return pDOM; 20 | }, 21 | }, 22 | h1: { 23 | group: "block", 24 | content: "text*", 25 | marks: "_", 26 | parseDOM: [{ tag: "h1" }], 27 | toDOM() { 28 | return h1DOM; 29 | }, 30 | }, 31 | h2: { 32 | group: "block", 33 | content: "text*", 34 | marks: "_", 35 | parseDOM: [{ tag: "h2" }], 36 | toDOM() { 37 | return h2DOM; 38 | }, 39 | }, 40 | ul: { 41 | group: "block", 42 | content: "text*", 43 | marks: "_", 44 | attrs: { 45 | /** 46 | * This is only set in sync(), *not* stored in the collaborative state. 47 | */ 48 | symbol: { default: "•" }, 49 | }, 50 | // TODO: ul vs ol 51 | parseDOM: [{ tag: "li" }], 52 | toDOM(node) { 53 | return [ 54 | "p", 55 | { 56 | class: "fakeLi", 57 | style: `--before-content: "${node.attrs.symbol}"`, 58 | }, 59 | 0, 60 | ]; 61 | }, 62 | }, 63 | ol: { 64 | group: "block", 65 | content: "text*", 66 | marks: "_", 67 | attrs: { 68 | /** 69 | * This is only set in sync(), *not* stored in the collaborative state. 70 | */ 71 | symbol: { default: "1." }, 72 | }, 73 | toDOM(node) { 74 | return [ 75 | "p", 76 | { 77 | class: "fakeLi", 78 | style: `--before-content: "${node.attrs.symbol}"`, 79 | }, 80 | 0, 81 | ]; 82 | }, 83 | }, 84 | text: {}, 85 | }, 86 | marks: { 87 | // em and strong marks copied from prosemirror-schema-basic. 88 | 89 | /// An emphasis mark. Rendered as an `` element. Has parse rules 90 | /// that also match `` and `font-style: italic`. 91 | em: { 92 | parseDOM: [ 93 | { tag: "i" }, 94 | { tag: "em" }, 95 | { style: "font-style=italic" }, 96 | { style: "font-style=normal", clearMark: (m) => m.type.name == "em" }, 97 | ], 98 | toDOM() { 99 | return emDOM; 100 | }, 101 | } as MarkSpec, 102 | 103 | /// A strong mark. Rendered as ``, parse rules also match 104 | /// `` and `font-weight: bold`. 105 | strong: { 106 | parseDOM: [ 107 | { tag: "strong" }, 108 | // This works around a Google Docs misbehavior where 109 | // pasted content will be inexplicably wrapped in `` 110 | // tags with a font-weight normal. 111 | { 112 | tag: "b", 113 | getAttrs: (node: HTMLElement) => 114 | node.style.fontWeight != "normal" && null, 115 | }, 116 | { style: "font-weight=400", clearMark: (m) => m.type.name == "strong" }, 117 | { 118 | style: "font-weight", 119 | getAttrs: (value: string) => 120 | /^(bold(er)?|[5-9]\d{2,})$/.test(value) && null, 121 | }, 122 | ], 123 | toDOM() { 124 | return strongDOM; 125 | }, 126 | } as MarkSpec, 127 | }, 128 | }); 129 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | /* Recommend by Webpack. */ 5 | "module": "ES6", 6 | /* Needed with module: ES6 or else compilation breaks. */ 7 | "moduleResolution": "node", 8 | /* Enable strict type checking. */ 9 | "strict": true, 10 | /* Prevent errors caused by other libraries. */ 11 | "skipLibCheck": true, 12 | /* Enable interop with dependencies using different module systems. */ 13 | "esModuleInterop": true, 14 | /* We don't need to emit declarations. */ 15 | "declaration": false, 16 | /* Emit sourcemap files. */ 17 | "sourceMap": true, 18 | "rootDir": "src" 19 | }, 20 | "include": ["src/site", "src/common", "src/site/old.ts"] 21 | } 22 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/tsconfig.server.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "ES6", 5 | /* Enable strict type checking. */ 6 | "strict": true, 7 | /* Prevent errors caused by other libraries. */ 8 | "skipLibCheck": true, 9 | /* Enable interop with dependencies using different module systems. */ 10 | "esModuleInterop": true, 11 | /* Use tsc (npm run build) to typecheck only. */ 12 | "noEmit": true, 13 | "rootDir": "src" 14 | }, 15 | "include": ["src/server", "src/common"] 16 | } 17 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/tsconfig.webpack-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "ES6", 5 | /* Enable strict type checking. */ 6 | "strict": true, 7 | /* Prevent errors caused by other libraries. */ 8 | "skipLibCheck": true, 9 | /* Enable interop with dependencies using different module systems. */ 10 | "esModuleInterop": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /websocket-prosemirror-blocks/webpack.config.ts: -------------------------------------------------------------------------------- 1 | import HtmlWebpackPlugin from "html-webpack-plugin"; 2 | import * as path from "path"; 3 | import * as webpack from "webpack"; 4 | 5 | // Basic Webpack config for TypeScript, based on 6 | // https://webpack.js.org/guides/typescript/ . 7 | const config: webpack.Configuration = { 8 | // mode and devtool are overridden by `npm run build` for production mode. 9 | mode: "development", 10 | devtool: "eval-source-map", 11 | entry: "./src/site/main.ts", 12 | module: { 13 | rules: [ 14 | { 15 | test: /\.tsx?$/, 16 | use: "ts-loader", 17 | exclude: /node_modules/, 18 | }, 19 | { 20 | test: /\.js$/, 21 | enforce: "pre", 22 | use: ["source-map-loader"], 23 | }, 24 | { 25 | test: /\.css$/, 26 | use: ["style-loader", "css-loader"], 27 | }, 28 | ], 29 | }, 30 | resolve: { 31 | extensions: [".tsx", ".ts", ".js"], 32 | }, 33 | output: { 34 | filename: "[name].bundle.js", 35 | path: path.resolve(__dirname, "dist"), 36 | clean: true, 37 | }, 38 | plugins: [ 39 | // Use src/index.html as the entry point. 40 | new HtmlWebpackPlugin({ 41 | template: "./src/site/index.html", 42 | }), 43 | ], 44 | }; 45 | 46 | export default config; 47 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/.prettierignore: -------------------------------------------------------------------------------- 1 | /dist/* 2 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2024 Matthew Weidner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/README.md: -------------------------------------------------------------------------------- 1 | # WebSocket-Prosemirror-Log 2 | 3 | A basic collaborative rich-text editor using [list-positions](https://github.com/mweidner037/list-positions#readme), a WebSocket server, and [ProseMirror](https://prosemirror.net/). It supports arbitrary schemas and works similarly to ProseMirror's built-in collaboration system, using a server-authoritative log of changes. 4 | 5 | When a client makes a change, a _mutation_ describing that change is sent to the server in JSON format. The server assigns that mutation a sequence number in the log and echoes it to all connected clients. It also stores the log to send to future clients. (In principle, the server could instead store the literal ProseMirror & list-positions states.) 6 | 7 | A client's state is always given by: 8 | 9 | - First, apply all mutations received from (or confirmed by) the server, in the same order as the server's log. 10 | - Next, apply all pending local mutations, which have been performed by the local user but not yet confirmed by the server. 11 | 12 | To process a remote message from the server, the pending local mutations are undone, the remote message is applied, and then the pending local mutations are redone. If a mutation no longer makes sense in its current state, it is skipped. 13 | 14 | Internally, each mutation consists of ordinary ProseMirror [steps](https://prosemirror.net/docs/guide/#transform), but with their list indices (ProseMirror positions) replaced by Positions from list-positions. That way, it is always possible to "rebase" a step on top of the server's latest state: just look up the new indices corresponding to each steps' Positions. (Internally, the lookup uses an [Outline](https://github.com/mweidner037/list-positions#outline).) 15 | 16 | Overall, this strategy is the same as [ProseMirror's built-in collaboration system](https://prosemirror.net/docs/guide/#collab), but using immutable Positions (CRDT-style) instead of indices that are transformed during rebasing (OT-style). As a result, clients never need to rebase and resubmit steps: steps can be rebased "as-is". 17 | 18 | Using ProseMirror's built-in steps lets many ProseMirror features work out-of-the-box, just like with ProseMirror's built-in collaboration. In contrast, [y-prosemirror](https://github.com/yjs/y-prosemirror) and [websocket-prosemirror-blocks](../websocket-prosemirror-blocks#readme) rewrite the ProseMirror state directly, which breaks the default cursor tracking, undo/redo, and [some other features](https://discuss.yjs.dev/t/decorationsets-and-remapping-broken-with-y-sync-plugin/845). 19 | 20 | _References: [Collaborative Editing in ProseMirror](https://marijnhaverbeke.nl/blog/collaborative-editing.html); [Replicache's sync strategy](https://rocicorp.dev/blog/ready-player-two)_ 21 | 22 | Code organization: 23 | 24 | - `src/common/`: Messages shared between clients and the server. 25 | - `src/server/`: WebSocket server. 26 | - `src/site/`: ProseMirror client. 27 | 28 | ## Installation 29 | 30 | First, install [Node.js](https://nodejs.org/). Then run `npm i`. 31 | 32 | ## Commands 33 | 34 | ### `npm run dev` 35 | 36 | Build the app from `src/`, in [development mode](https://webpack.js.org/guides/development/). You can also use `npm run watch`. 37 | 38 | ### `npm run build` 39 | 40 | Build the app from `src/`, in [production mode](https://webpack.js.org/guides/production/). 41 | 42 | ### `npm start` 43 | 44 | Run the server on [http://localhost:3000/](http://localhost:3000/). Use multiple browser windows at once to test collaboration. 45 | 46 | To change the port, set the `$PORT` environment variable. 47 | 48 | ### `npm run clean` 49 | 50 | Delete `dist/`. 51 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "license": "MIT", 3 | "dependencies": { 4 | "express": "^4.18.2", 5 | "@list-positions/formatting": "^1.0.0", 6 | "list-positions": "^1.0.0", 7 | "maybe-random-string": "^1.0.0", 8 | "prosemirror-commands": "^1.5.2", 9 | "prosemirror-example-setup": "^1.2.2", 10 | "prosemirror-keymap": "^1.2.2", 11 | "prosemirror-menu": "^1.2.4", 12 | "prosemirror-model": "^1.19.4", 13 | "prosemirror-schema-basic": "^1.2.2", 14 | "prosemirror-schema-list": "^1.3.0", 15 | "prosemirror-state": "^1.4.3", 16 | "prosemirror-transform": "^1.8.0", 17 | "prosemirror-view": "^1.32.7", 18 | "ws": "^8.13.0" 19 | }, 20 | "devDependencies": { 21 | "@types/express": "^4.17.17", 22 | "@types/node": "^15.6.1", 23 | "@types/webpack": "^5.28.0", 24 | "@types/webpack-env": "^1.16.2", 25 | "@types/ws": "^8.5.10", 26 | "copy-webpack-plugin": "^11.0.0", 27 | "cross-env": "^7.0.3", 28 | "css-loader": "^6.2.0", 29 | "html-webpack-plugin": "^5.3.2", 30 | "npm-run-all": "^4.1.5", 31 | "prettier": "^2.2.1", 32 | "rimraf": "^2.7.1", 33 | "source-map-loader": "^3.0.0", 34 | "style-loader": "^3.3.3", 35 | "ts-loader": "^9.2.5", 36 | "ts-node": "^10.1.0", 37 | "typescript": "^4.3.5", 38 | "webpack": "^5.50.0", 39 | "webpack-cli": "^4.10.0" 40 | }, 41 | "scripts": { 42 | "start": "ts-node -P tsconfig.server.json src/server/server.ts", 43 | "watch": "cross-env TS_NODE_PROJECT='tsconfig.webpack-config.json' webpack --watch", 44 | "dev": "cross-env TS_NODE_PROJECT='tsconfig.webpack-config.json' webpack", 45 | "build": "cross-env TS_NODE_PROJECT='tsconfig.webpack-config.json' webpack --mode production --devtool source-map", 46 | "test": "npm-run-all test:*", 47 | "test:server-tsc": "tsc -p tsconfig.server.json", 48 | "test:format": "prettier --check .", 49 | "fix": "npm-run-all fix:*", 50 | "fix:format": "prettier --write .", 51 | "clean": "rimraf dist" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/src/common/messages.ts: -------------------------------------------------------------------------------- 1 | import type { BunchMeta, Position } from "list-positions"; 2 | import { Mutation } from "./mutation"; 3 | 4 | export type MutationMessage = { 5 | type: "mutation"; 6 | mutation: Mutation; 7 | }; 8 | 9 | export type WelcomeMessage = { 10 | type: "welcome"; 11 | mutations: Mutation[]; 12 | }; 13 | 14 | export type Message = MutationMessage | WelcomeMessage; 15 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/src/common/mutation.ts: -------------------------------------------------------------------------------- 1 | import type { BunchMeta, Position } from "list-positions"; 2 | 3 | /** 4 | * Positions for a ReplaceStep or one part of a ReplaceAroundStep. 5 | * 6 | * At least one of insert or delete is guaranteed to be defined. 7 | */ 8 | export type ReplacePositions = { 9 | insert?: { meta: BunchMeta | null; startPos: Position }; 10 | delete?: { 11 | // Right cursor - doesn't expand. 12 | startPos: Position; 13 | // Left cursor - doesn't expand. 14 | endPos: Position; 15 | }; 16 | }; 17 | 18 | /** 19 | * A ProseMirror step, annotated with Positions in place of list indices (PM positions). 20 | * 21 | * The Positions let us rebase steps "as-is", without explicitly transforming indices. 22 | * In other words, an AnnotatedStep is CRDT-style, while a plain step is OT-style. 23 | */ 24 | export type AnnotatedStep = 25 | | { 26 | type: "replace"; 27 | positions: ReplacePositions; 28 | sliceJSON: unknown; 29 | structure: boolean; 30 | } 31 | | { 32 | type: "replaceAround"; 33 | leftPositions: ReplacePositions; 34 | rightPositions: ReplacePositions; 35 | sliceJSON: unknown; 36 | // This is just an index into the slice, so we don't need to CRDT-ify it. 37 | sliceInsert: number; 38 | structure: boolean; 39 | } 40 | | { 41 | // TODO: If an insertion is applied after a concurrent changeMark, it won't 42 | // get the mark (according to PM's default logic). 43 | // Can we change it to get the mark anyway (Peritext-style)? 44 | type: "changeMark"; 45 | // Else remove. 46 | isAdd: boolean; 47 | // Right cursor - doesn't expand. 48 | fromPos: Position; 49 | // Left cursor - doesn't expand. 50 | // TODO: do expand for e.g. bold? 51 | toPos: Position; 52 | markJSON: unknown; 53 | } 54 | | { 55 | type: "changeNodeMark"; 56 | // Else remove. 57 | isAdd: boolean; 58 | pos: Position; 59 | markJSON: unknown; 60 | } 61 | | { 62 | type: "attr"; 63 | pos: Position; 64 | attr: string; 65 | value: unknown; 66 | } 67 | | { 68 | type: "docAttr"; 69 | attr: string; 70 | value: unknown; 71 | }; 72 | 73 | export type Mutation = { 74 | annSteps: AnnotatedStep[]; 75 | clientID: string; 76 | clientCounter: number; 77 | }; 78 | 79 | export function idEquals(a: Mutation, b: Mutation): boolean { 80 | return a.clientID === b.clientID && a.clientCounter === b.clientCounter; 81 | } 82 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/src/server/rich_text_server.ts: -------------------------------------------------------------------------------- 1 | import { WebSocket, WebSocketServer } from "ws"; 2 | import { Message } from "../common/messages"; 3 | import { Mutation } from "../common/mutation"; 4 | 5 | const heartbeatInterval = 30000; 6 | 7 | /** 8 | * Server that assigns mutations a sequence number and echoes them to all 9 | * clients in order. 10 | * 11 | * We store the full Mutation log for welcoming future clients. In principle, 12 | * you could instead store just the current ProseMirror + Outline states and 13 | * use those to welcome clients. (For reconnections, you would also need a vector 14 | * clock or similar, to tell clients which of their past mutations have been acked.) 15 | */ 16 | export class RichTextServer { 17 | private readonly mutations: Mutation[] = []; 18 | 19 | private clients = new Set(); 20 | 21 | constructor(readonly wss: WebSocketServer) { 22 | this.wss.on("connection", (ws) => { 23 | if (ws.readyState === WebSocket.OPEN) { 24 | this.wsOpen(ws); 25 | } else ws.on("open", () => this.wsOpen(ws)); 26 | ws.on("message", (data) => this.wsReceive(ws, data.toString())); 27 | ws.on("close", () => this.wsClose(ws)); 28 | ws.on("error", (err) => { 29 | console.error(err); 30 | this.wsClose(ws); 31 | }); 32 | }); 33 | } 34 | 35 | private sendMessage(ws: WebSocket, msg: Message) { 36 | if (ws.readyState == WebSocket.OPEN) { 37 | ws.send(JSON.stringify(msg)); 38 | } 39 | } 40 | 41 | private echo(origin: WebSocket, data: string) { 42 | for (const ws of this.clients) { 43 | if (ws.readyState == WebSocket.OPEN) { 44 | ws.send(data); 45 | } 46 | } 47 | } 48 | 49 | private wsOpen(ws: WebSocket) { 50 | this.startHeartbeats(ws); 51 | 52 | // Send the current state. 53 | this.sendMessage(ws, { 54 | type: "welcome", 55 | mutations: this.mutations, 56 | }); 57 | 58 | this.clients.add(ws); 59 | } 60 | 61 | /** 62 | * Ping to keep connection alive. 63 | * 64 | * This is necessary on at least Heroku, which has a 55 second timeout: 65 | * https://devcenter.heroku.com/articles/websockets#timeouts 66 | */ 67 | private startHeartbeats(ws: WebSocket) { 68 | const interval = setInterval(() => { 69 | if (ws.readyState === WebSocket.OPEN) { 70 | ws.ping(); 71 | } else clearInterval(interval); 72 | }, heartbeatInterval); 73 | } 74 | 75 | private wsReceive(ws: WebSocket, data: string) { 76 | const msg = JSON.parse(data) as Message; 77 | switch (msg.type) { 78 | case "mutation": 79 | // Here is where you can choose to reject/alter the mutation, before 80 | // adding it to the log (which is the source of truth) and 81 | // broadcasting it. 82 | // Note: Even if you reject the change, you should keep the BunchMeta, 83 | // in case this client's future changes depend on it. 84 | // TODO: Need a way to tell a client when one of its mutations has 85 | // been acknowledged but not accepted as-is, so that the client can remove 86 | // that mutation from its pendingMutations. 87 | this.mutations.push(msg.mutation); 88 | this.echo(ws, data); 89 | break; 90 | default: 91 | console.error("Unknown message type: " + msg.type); 92 | } 93 | } 94 | 95 | private wsClose(ws: WebSocket) { 96 | this.clients.delete(ws); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/src/server/server.ts: -------------------------------------------------------------------------------- 1 | import express from "express"; 2 | import path from "path"; 3 | import { WebSocketServer } from "ws"; 4 | import { RichTextServer } from "./rich_text_server"; 5 | 6 | const port = process.env.PORT || 3000; 7 | 8 | // Server dist/ with a simple express server. 9 | const app = express(); 10 | app.use("/", express.static(path.join(__dirname, "../../dist"))); 11 | const server = app.listen(port, () => 12 | console.log(`Listening at http://localhost:${port}/`) 13 | ); 14 | 15 | // Run the WebSocket server. 16 | const wss = new WebSocketServer({ server }); 17 | new RichTextServer(wss); 18 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/src/site/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 31 | 32 | WebSocket-Prosemirror-Log 33 | 34 | 35 | 36 |
37 | 38 | 39 | | 40 | Info and source code 44 |
45 |
46 | 47 |
48 |
49 |
50 | 51 | 52 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/src/site/main.ts: -------------------------------------------------------------------------------- 1 | import { Message } from "../common/messages"; 2 | import { Mutation } from "../common/mutation"; 3 | import { ProseMirrorWrapper } from "./prosemirror_wrapper"; 4 | import { WebSocketClient } from "./web_socket_client"; 5 | 6 | const wsURL = location.origin.replace(/^http/, "ws"); 7 | const client = new WebSocketClient(wsURL); 8 | 9 | client.onMessage = (data) => { 10 | const msg = JSON.parse(data) as Message; 11 | if (msg.type === "welcome") { 12 | // Got the initial state. Start ProseMirror. 13 | const wrapper = new ProseMirrorWrapper(onLocalMutation); 14 | wrapper.receive(msg.mutations); 15 | client.onMessage = (data) => onMessage(data, wrapper); 16 | } else { 17 | console.error("Received non-welcome message first: " + msg.type); 18 | } 19 | }; 20 | 21 | function onMessage(data: string, wrapper: ProseMirrorWrapper): void { 22 | const msg = JSON.parse(data) as Message; 23 | switch (msg.type) { 24 | case "mutation": 25 | wrapper.receive([msg.mutation]); 26 | break; 27 | default: 28 | console.error("Unexpected message type:", msg.type, msg); 29 | } 30 | } 31 | 32 | function onLocalMutation(mutation: Mutation) { 33 | send([{ type: "mutation", mutation }]); 34 | } 35 | 36 | function send(msgs: Message[]): void { 37 | for (const msg of msgs) { 38 | client.send(JSON.stringify(msg)); 39 | } 40 | } 41 | 42 | // --- "Connected" checkbox for testing concurrency --- 43 | 44 | const connected = document.getElementById("connected") as HTMLInputElement; 45 | connected.addEventListener("click", () => { 46 | client.testConnected = !client.testConnected; 47 | }); 48 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/src/site/web_socket_client.ts: -------------------------------------------------------------------------------- 1 | // For this basic demo, we don't attempt to reconnect the WebSocket ever. 2 | // That would require buffering updates and/or logic to 3 | // "merge" in the Welcome state received after reconnecting. 4 | // However, we do allow "test" disconnection that just buffers messages 5 | // internally (without actually disconnecting the WebSocket). 6 | 7 | export class WebSocketClient { 8 | ws: WebSocket; 9 | 10 | onMessage?: (message: string) => void; 11 | 12 | private _testConnected = true; 13 | private sendQueue: string[] = []; 14 | private receiveQueue: string[] = []; 15 | 16 | constructor(readonly wsURL: string) { 17 | this.ws = new WebSocket(wsURL); 18 | this.ws.addEventListener("message", (e) => this.messageHandler(e)); 19 | } 20 | 21 | private messageHandler(e: MessageEvent): void { 22 | const message = e.data; 23 | if (this._testConnected) { 24 | this.onMessage?.(message); 25 | } else { 26 | this.receiveQueue.push(message); 27 | } 28 | } 29 | 30 | private sendInternal(message: string): void { 31 | if (this.ws.readyState === WebSocket.OPEN) { 32 | this.ws.send(message); 33 | } else { 34 | console.log("WebSocket not open, skipping send"); 35 | } 36 | } 37 | 38 | send(message: string): void { 39 | if (this._testConnected) { 40 | this.sendInternal(message); 41 | } else { 42 | this.sendQueue.push(message); 43 | } 44 | } 45 | 46 | set testConnected(conn: boolean) { 47 | if (conn !== this._testConnected) { 48 | this._testConnected = conn; 49 | if (conn) { 50 | // Send queued messages. 51 | for (const message of this.sendQueue) { 52 | this.sendInternal(message); 53 | } 54 | this.sendQueue = []; 55 | 56 | // Received queued messages. 57 | const toReceive = this.receiveQueue; 58 | this.receiveQueue = []; 59 | for (const message of toReceive) { 60 | this.onMessage?.(message); 61 | } 62 | } 63 | } 64 | } 65 | 66 | get testConnected(): boolean { 67 | return this._testConnected; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2022", 4 | /* Recommend by Webpack. */ 5 | "module": "ES6", 6 | /* Needed with module: ES6 or else compilation breaks. */ 7 | "moduleResolution": "node", 8 | /* Enable strict type checking. */ 9 | "strict": true, 10 | /* Prevent errors caused by other libraries. */ 11 | "skipLibCheck": true, 12 | /* Enable interop with dependencies using different module systems. */ 13 | "esModuleInterop": true, 14 | /* We don't need to emit declarations. */ 15 | "declaration": false, 16 | /* Emit sourcemap files. */ 17 | "sourceMap": true, 18 | "rootDir": "src" 19 | }, 20 | "include": ["src/site", "src/common", "src/site/old.ts"] 21 | } 22 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/tsconfig.server.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "ES6", 5 | /* Enable strict type checking. */ 6 | "strict": true, 7 | /* Prevent errors caused by other libraries. */ 8 | "skipLibCheck": true, 9 | /* Enable interop with dependencies using different module systems. */ 10 | "esModuleInterop": true, 11 | /* Use tsc (npm run build) to typecheck only. */ 12 | "noEmit": true, 13 | "rootDir": "src" 14 | }, 15 | "include": ["src/server", "src/common"] 16 | } 17 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/tsconfig.webpack-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "ES6", 5 | /* Enable strict type checking. */ 6 | "strict": true, 7 | /* Prevent errors caused by other libraries. */ 8 | "skipLibCheck": true, 9 | /* Enable interop with dependencies using different module systems. */ 10 | "esModuleInterop": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /websocket-prosemirror-log/webpack.config.ts: -------------------------------------------------------------------------------- 1 | import HtmlWebpackPlugin from "html-webpack-plugin"; 2 | import * as path from "path"; 3 | import * as webpack from "webpack"; 4 | 5 | // Basic Webpack config for TypeScript, based on 6 | // https://webpack.js.org/guides/typescript/ . 7 | const config: webpack.Configuration = { 8 | // mode and devtool are overridden by `npm run build` for production mode. 9 | mode: "development", 10 | devtool: "eval-source-map", 11 | entry: "./src/site/main.ts", 12 | module: { 13 | rules: [ 14 | { 15 | test: /\.tsx?$/, 16 | use: "ts-loader", 17 | exclude: /node_modules/, 18 | }, 19 | { 20 | test: /\.js$/, 21 | enforce: "pre", 22 | use: ["source-map-loader"], 23 | }, 24 | { 25 | test: /\.css$/, 26 | use: ["style-loader", "css-loader"], 27 | }, 28 | ], 29 | }, 30 | resolve: { 31 | extensions: [".tsx", ".ts", ".js"], 32 | }, 33 | output: { 34 | filename: "[name].bundle.js", 35 | path: path.resolve(__dirname, "dist"), 36 | clean: true, 37 | }, 38 | plugins: [ 39 | // Use src/index.html as the entry point. 40 | new HtmlWebpackPlugin({ 41 | template: "./src/site/index.html", 42 | }), 43 | ], 44 | }; 45 | 46 | export default config; 47 | -------------------------------------------------------------------------------- /websocket-quill/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /websocket-quill/.prettierignore: -------------------------------------------------------------------------------- 1 | /dist/* 2 | -------------------------------------------------------------------------------- /websocket-quill/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2024 Matthew Weidner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /websocket-quill/README.md: -------------------------------------------------------------------------------- 1 | # WebSocket-Quill 2 | 3 | A basic collaborative rich-text editor using [list-positions](https://github.com/mweidner037/list-positions#readme) and [@list-positions/formatting](https://github.com/mweidner037/list-positions-formatting#readme), a WebSocket server, and [Quill](https://quilljs.com/). 4 | 5 | When a client makes a change, a description is sent to the server in JSON format. The server echoes that change to all other connected clients. The server also updates its own copy of the rich-text state; this is sent to new clients when they load the page. 6 | 7 | A client optimistically updates its own state before sending its change to the server. The demo is simple enough that these optimistic updates are always "correct" (match the server's eventual state) - see the comments in [`src/server/rich_text_server.ts`](./src/server/rich_text_server.ts). A more complicated app might need to "repair" optimistic updates that are rejected or modified by the server, e.g., due to permissions issues. 8 | 9 | Code organization: 10 | 11 | - `src/common/messages.ts`: Messages sent between clients and the server. 12 | - `src/server/`: WebSocket server. 13 | - `src/site/`: Quill client. 14 | 15 | ## Setup 16 | 17 | Install with `npm i`. 18 | 19 | ## Commands 20 | 21 | ### `npm run dev` 22 | 23 | Build the site in [development mode](https://webpack.js.org/guides/development/). 24 | 25 | ### `npm run build` 26 | 27 | Build the site in [production mode](https://webpack.js.org/guides/production/). 28 | 29 | ### `npm start` 30 | 31 | Run the server on [http://localhost:3000/](http://localhost:3000/). Use multiple browser windows at once to test collaboration. 32 | 33 | To change the port, set the `$PORT` environment variable. 34 | 35 | ### `npm run clean` 36 | 37 | Delete `dist/`. 38 | -------------------------------------------------------------------------------- /websocket-quill/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "license": "MIT", 3 | "dependencies": { 4 | "@list-positions/formatting": "^1.0.0", 5 | "express": "^4.18.2", 6 | "list-positions": "^1.0.0", 7 | "quill": "^2.0.2", 8 | "quill-cursors": "^4.0.2", 9 | "ws": "^8.13.0" 10 | }, 11 | "devDependencies": { 12 | "@types/express": "^4.17.17", 13 | "@types/node": "^15.6.1", 14 | "@types/webpack": "^5.28.0", 15 | "@types/webpack-env": "^1.16.2", 16 | "@types/ws": "^8.5.10", 17 | "copy-webpack-plugin": "^11.0.0", 18 | "cross-env": "^7.0.3", 19 | "css-loader": "^6.2.0", 20 | "html-webpack-plugin": "^5.3.2", 21 | "npm-run-all": "^4.1.5", 22 | "prettier": "^2.2.1", 23 | "rimraf": "^2.7.1", 24 | "source-map-loader": "^3.0.0", 25 | "style-loader": "^3.3.3", 26 | "ts-loader": "^9.5.1", 27 | "ts-node": "^10.9.2", 28 | "typescript": "^5.4.5", 29 | "webpack": "^5.50.0", 30 | "webpack-cli": "^4.10.0" 31 | }, 32 | "scripts": { 33 | "start": "ts-node -P tsconfig.server.json src/server/server.ts", 34 | "dev": "cross-env TS_NODE_PROJECT='tsconfig.webpack-config.json' webpack", 35 | "build": "cross-env TS_NODE_PROJECT='tsconfig.webpack-config.json' webpack --mode production --devtool source-map", 36 | "test": "npm-run-all test:*", 37 | "test:server-tsc": "tsc -p tsconfig.server.json", 38 | "test:format": "prettier --check .", 39 | "fix": "npm-run-all fix:*", 40 | "fix:format": "prettier --write .", 41 | "clean": "rimraf dist" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /websocket-quill/src/common/messages.ts: -------------------------------------------------------------------------------- 1 | import { 2 | TimestampFormattingSavedState, 3 | TimestampMark, 4 | } from "@list-positions/formatting"; 5 | import { 6 | BunchMeta, 7 | OrderSavedState, 8 | Position, 9 | TextSavedState, 10 | } from "list-positions"; 11 | 12 | export type SetMessage = { 13 | type: "set"; 14 | startPos: Position; 15 | chars: string; 16 | meta?: BunchMeta; 17 | }; 18 | 19 | export type DeleteMessage = { 20 | type: "delete"; 21 | pos: Position; 22 | }; 23 | 24 | export type MarkMessage = { 25 | type: "mark"; 26 | mark: TimestampMark; 27 | }; 28 | 29 | export type WelcomeMessage = { 30 | type: "welcome"; 31 | order: OrderSavedState; 32 | text: TextSavedState; 33 | formatting: TimestampFormattingSavedState; 34 | }; 35 | 36 | export type Message = SetMessage | DeleteMessage | MarkMessage | WelcomeMessage; 37 | -------------------------------------------------------------------------------- /websocket-quill/src/server/rich_text_server.ts: -------------------------------------------------------------------------------- 1 | import { TimestampMark } from "@list-positions/formatting"; 2 | import { Text } from "list-positions"; 3 | import { WebSocket, WebSocketServer } from "ws"; 4 | import { Message } from "../common/messages"; 5 | 6 | const heartbeatInterval = 30000; 7 | 8 | export class RichTextServer { 9 | // To easily save and send the state to new clients, store the 10 | // text in a List. 11 | private readonly text: Text; 12 | // We don't need to inspect the formatting, so just store the marks directly. 13 | private readonly marks: TimestampMark[]; 14 | 15 | private clients = new Set(); 16 | 17 | constructor(readonly wss: WebSocketServer) { 18 | this.text = new Text(); 19 | this.marks = []; 20 | 21 | // Initial state: a single "\n", to match Quill's initial state. 22 | this.text.insertAt(0, "\n"); 23 | 24 | this.wss.on("connection", (ws) => { 25 | if (ws.readyState === WebSocket.OPEN) { 26 | this.wsOpen(ws); 27 | } else ws.on("open", () => this.wsOpen(ws)); 28 | ws.on("message", (data) => this.wsReceive(ws, data.toString())); 29 | ws.on("close", () => this.wsClose(ws)); 30 | ws.on("error", (err) => { 31 | console.error(err); 32 | this.wsClose(ws); 33 | }); 34 | }); 35 | } 36 | 37 | private sendMessage(ws: WebSocket, msg: Message) { 38 | if (ws.readyState === WebSocket.OPEN) { 39 | ws.send(JSON.stringify(msg)); 40 | } 41 | } 42 | 43 | private echo(origin: WebSocket, data: string) { 44 | for (const ws of this.clients) { 45 | if (ws === origin) continue; 46 | if (ws.readyState === WebSocket.OPEN) { 47 | ws.send(data); 48 | } 49 | } 50 | } 51 | 52 | private wsOpen(ws: WebSocket) { 53 | this.startHeartbeats(ws); 54 | 55 | // Send the current state. 56 | this.sendMessage(ws, { 57 | type: "welcome", 58 | order: this.text.order.save(), 59 | text: this.text.save(), 60 | formatting: this.marks, 61 | }); 62 | 63 | this.clients.add(ws); 64 | } 65 | 66 | /** 67 | * Ping to keep connection alive. 68 | * 69 | * This is necessary on at least Heroku, which has a 55 second timeout: 70 | * https://devcenter.heroku.com/articles/websockets#timeouts 71 | */ 72 | private startHeartbeats(ws: WebSocket) { 73 | const interval = setInterval(() => { 74 | if (ws.readyState === WebSocket.OPEN) { 75 | ws.ping(); 76 | } else clearInterval(interval); 77 | }, heartbeatInterval); 78 | } 79 | 80 | private wsReceive(ws: WebSocket, data: string) { 81 | const msg = JSON.parse(data) as Message; 82 | switch (msg.type) { 83 | case "set": 84 | if (msg.meta) { 85 | this.text.order.addMetas([msg.meta]); 86 | } 87 | this.text.set(msg.startPos, msg.chars); 88 | this.echo(ws, data); 89 | // Because a Position is only ever set once (when it's created) and 90 | // the server does no validation, the origin's optimistically-updated 91 | // state is already correct: msg.startPos is set to msg.chars. 92 | // If that were not true, we would need to send a message to origin 93 | // telling it how to repair its optimistically-updated state. 94 | break; 95 | case "delete": 96 | this.text.delete(msg.pos); 97 | this.echo(ws, data); 98 | // Because deletes are permanant and the server does no validation, 99 | // the origin's optimistically-updated state is already correct. 100 | break; 101 | case "mark": 102 | this.marks.push(msg.mark); 103 | this.echo(ws, data); 104 | // Because marks are permanant and the server does no validation, 105 | // the origin's optimistically-updated state is already correct. 106 | break; 107 | default: 108 | throw new Error("Unknown message type: " + msg.type); 109 | } 110 | } 111 | 112 | private wsClose(ws: WebSocket) { 113 | this.clients.delete(ws); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /websocket-quill/src/server/server.ts: -------------------------------------------------------------------------------- 1 | import express from "express"; 2 | import path from "path"; 3 | import { WebSocketServer } from "ws"; 4 | import { RichTextServer } from "./rich_text_server"; 5 | 6 | const port = process.env.PORT || 3000; 7 | 8 | // Server dist/ with a simple express server. 9 | const app = express(); 10 | app.use("/", express.static(path.join(__dirname, "../../dist"))); 11 | const server = app.listen(port, () => 12 | console.log(`Listening at http://localhost:${port}/`) 13 | ); 14 | 15 | // Run the WebSocket server. 16 | const wss = new WebSocketServer({ server }); 17 | new RichTextServer(wss); 18 | -------------------------------------------------------------------------------- /websocket-quill/src/site/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | WebSocket-Quill 8 | 9 | 10 | 11 |
12 | Info and source code 16 |
17 |
18 | 19 | 20 |
21 |
22 |
23 | 24 | 25 | -------------------------------------------------------------------------------- /websocket-quill/src/site/main.ts: -------------------------------------------------------------------------------- 1 | import { Message } from "../common/messages"; 2 | import { QuillWrapper } from "./quill_wrapper"; 3 | 4 | const wsURL = location.origin.replace(/^http/, "ws"); 5 | const ws = new WebSocket(wsURL); 6 | 7 | function welcomeListener(e: MessageEvent) { 8 | const msg = JSON.parse(e.data) as Message; 9 | if (msg.type === "welcome") { 10 | // Got the initial state. Start Quill. 11 | ws.removeEventListener("message", welcomeListener); 12 | new QuillWrapper(ws, msg); 13 | } else { 14 | console.error("Received non-welcome message first: " + msg.type); 15 | } 16 | } 17 | ws.addEventListener("message", welcomeListener); 18 | 19 | // For this basic demo, we don't allow disconnection tests or 20 | // attempt to reconnect the WebSocket ever. 21 | // That would require buffering updates and/or logic to 22 | // "merge" in the Welcome state received after reconnecting. 23 | -------------------------------------------------------------------------------- /websocket-quill/src/site/quill_wrapper.ts: -------------------------------------------------------------------------------- 1 | import Quill from "quill"; 2 | 3 | // Quill CSS. 4 | import { 5 | type FormattedChars, 6 | RichText, 7 | sliceFromSpan, 8 | } from "@list-positions/formatting"; 9 | import "quill/dist/quill.snow.css"; 10 | import type { Message, WelcomeMessage } from "../common/messages"; 11 | import { Delta } from "quill/core"; 12 | 13 | export class QuillWrapper { 14 | readonly editor: Quill; 15 | readonly richText: RichText; 16 | 17 | constructor(readonly ws: WebSocket, welcome: WelcomeMessage) { 18 | this.richText = new RichText({ expandRules }); 19 | 20 | // Setup Quill. 21 | const editorContainer = document.getElementById("editor") as HTMLDivElement; 22 | this.editor = new Quill(editorContainer, { 23 | theme: "snow", 24 | modules: { 25 | toolbar: [ 26 | ["bold", "italic"], 27 | [{ header: "1" }, { header: "2" }], 28 | [{ list: "ordered" }, { list: "bullet" }], 29 | ], 30 | history: { 31 | userOnly: true, 32 | }, 33 | }, 34 | formats: ["bold", "italic", "header", "list"], 35 | }); 36 | 37 | // Load initial state into richText. 38 | this.richText.order.load(welcome.order); 39 | this.richText.text.load(welcome.text); 40 | this.richText.formatting.load(welcome.formatting); 41 | 42 | // Sync initial state to Quill. 43 | this.editor.updateContents(deltaFromSlices(this.richText.formattedChars())); 44 | // Delete Quill's own initial "\n" - the server's state already contains one. 45 | this.editor.updateContents( 46 | new Delta().retain(this.richText.text.length).delete(1) 47 | ); 48 | 49 | // Sync Quill changes to our local state and to the server. 50 | let ourChange = false; 51 | this.editor.on("text-change", (delta) => { 52 | // Filter our own programmatic changes. 53 | if (ourChange) return; 54 | 55 | for (const op of getRelevantDeltaOperations(delta)) { 56 | // Insertion 57 | if (op.insert) { 58 | if (typeof op.insert === "string") { 59 | const quillAttrs = op.attributes ?? {}; 60 | const formattingAttrs = Object.fromEntries( 61 | [...Object.entries(quillAttrs)].map(quillAttrToFormatting) 62 | ); 63 | const [startPos, createdBunch, createdMarks] = 64 | this.richText.insertWithFormat( 65 | op.index, 66 | formattingAttrs, 67 | op.insert 68 | ); 69 | this.send({ 70 | type: "set", 71 | startPos, 72 | chars: op.insert, 73 | meta: createdBunch ?? undefined, 74 | }); 75 | for (const mark of createdMarks) { 76 | this.send({ 77 | type: "mark", 78 | mark, 79 | }); 80 | } 81 | } else { 82 | // Embed of object 83 | throw new Error("Embeds not supported"); 84 | } 85 | } 86 | // Deletion 87 | else if (op.delete) { 88 | const toDelete = [ 89 | ...this.richText.text.positions(op.index, op.index + op.delete), 90 | ]; 91 | for (const pos of toDelete) { 92 | this.richText.text.delete(pos); 93 | this.send({ 94 | type: "delete", 95 | pos, 96 | }); 97 | } 98 | } 99 | // Formatting 100 | else if (op.attributes && op.retain) { 101 | for (const [quillKey, quillValue] of Object.entries(op.attributes)) { 102 | const [key, value] = quillAttrToFormatting([quillKey, quillValue]); 103 | const [mark] = this.richText.format( 104 | op.index, 105 | op.index + op.retain, 106 | key, 107 | value 108 | ); 109 | this.send({ 110 | type: "mark", 111 | mark, 112 | }); 113 | } 114 | } 115 | } 116 | }); 117 | 118 | // Sync server changes to our local state and to Quill. 119 | this.ws.addEventListener("message", (e) => { 120 | ourChange = true; 121 | try { 122 | const msg = JSON.parse(e.data) as Message; 123 | switch (msg.type) { 124 | case "set": 125 | if (msg.meta) { 126 | this.richText.order.addMetas([msg.meta]); 127 | } 128 | // Sets are always nontrivial. 129 | // Because the server enforces causal ordering, bunched values 130 | // are always still contiguous and have a single format. 131 | this.richText.text.set(msg.startPos, msg.chars); 132 | const startIndex = this.richText.text.indexOfPosition(msg.startPos); 133 | const format = this.richText.formatting.getFormat(msg.startPos); 134 | this.editor.updateContents( 135 | new Delta() 136 | .retain(startIndex) 137 | .insert(msg.chars, formattingToQuillAttr(format)) 138 | ); 139 | break; 140 | case "delete": 141 | if (this.richText.text.has(msg.pos)) { 142 | const index = this.richText.text.indexOfPosition(msg.pos); 143 | this.richText.text.delete(msg.pos); 144 | this.editor.updateContents(new Delta().retain(index).delete(1)); 145 | } 146 | break; 147 | case "mark": 148 | const changes = this.richText.formatting.addMark(msg.mark); 149 | for (const change of changes) { 150 | const { startIndex, endIndex } = sliceFromSpan( 151 | this.richText.text, 152 | change.start, 153 | change.end 154 | ); 155 | this.editor.updateContents( 156 | new Delta() 157 | .retain(startIndex) 158 | .retain( 159 | endIndex - startIndex, 160 | formattingToQuillAttr({ [change.key]: change.value }) 161 | ) 162 | ); 163 | } 164 | break; 165 | default: 166 | throw new Error("Unknown message type: " + msg.type); 167 | } 168 | } finally { 169 | ourChange = false; 170 | } 171 | }); 172 | } 173 | 174 | private send(msg: Message) { 175 | if (this.ws.readyState === WebSocket.OPEN) { 176 | this.ws.send(JSON.stringify(msg)); 177 | } 178 | } 179 | } 180 | 181 | /** 182 | * Expand arg for the given format key's mark/unmark op. 183 | * 184 | * Default for inline formatting is "after"/"after". 185 | * 186 | * For links, instead use "none"/"both" (Peritext example 9). 187 | * 188 | * We also set all block formats to "none"/"none" for a Quill-specific reason: 189 | * Quill doesn't let a block format apply to a non-"\n", so a block format 190 | * shouldn't expand to neighboring non-"\n" chars (otherwise, we have to do 191 | * extra unmark ops). 192 | */ 193 | function expandRules( 194 | key: string, 195 | value: any 196 | ): "none" | "before" | "after" | "both" { 197 | switch (key) { 198 | case "block": 199 | case "indent": 200 | case "align": 201 | case "direction": 202 | return "none"; 203 | case "link": 204 | return value !== null ? "none" : "both"; 205 | default: 206 | return "after"; 207 | } 208 | } 209 | 210 | type ModifiedDeltaOperation = { 211 | index: number; 212 | insert?: string | object; 213 | delete?: number; 214 | attributes?: Record; 215 | retain?: number; 216 | }; 217 | 218 | /** 219 | * Convert delta.ops into an array of modified DeltaOperations 220 | * having the form { index: first char index, ...DeltaOperation }, 221 | * leaving out ops that do nothing. 222 | */ 223 | function getRelevantDeltaOperations(delta: Delta): ModifiedDeltaOperation[] { 224 | if (delta.ops === undefined) return []; 225 | const relevantOps: ModifiedDeltaOperation[] = []; 226 | let index = 0; 227 | for (const op of delta.ops) { 228 | if (op.retain === undefined || op.attributes) { 229 | relevantOps.push({ 230 | index, 231 | insert: op.insert, 232 | delete: op.delete, 233 | attributes: op.attributes, 234 | retain: typeof op.retain === "number" ? op.retain : undefined, 235 | }); 236 | } 237 | // Adjust index for the next op. 238 | if (op.insert !== undefined) { 239 | if (typeof op.insert === "string") index += op.insert.length; 240 | else index += 1; // Embed 241 | } else if (op.retain !== undefined) { 242 | if (typeof op.retain === "number") index += op.retain; 243 | // Embed, do not increment index 244 | } 245 | // Deletes don't add to the index because we'll do the 246 | // next operation after them, hence the text will already 247 | // be shifted left. 248 | } 249 | return relevantOps; 250 | } 251 | 252 | function deltaFromSlices(slices: FormattedChars[]) { 253 | let delta = new Delta(); 254 | for (const slice of slices) { 255 | delta = delta.insert(slice.chars, formattingToQuillAttr(slice.format)); 256 | } 257 | return delta; 258 | } 259 | 260 | /** 261 | * These formats are exclusive; we need to pass only one at a time to Quill or 262 | * the result is inconsistent. 263 | * So, we wrap them in our own "block" formatting attribute: 264 | * { block: [key, value] }. 265 | */ 266 | const exclusiveBlocks = new Set(["blockquote", "header", "list", "code-block"]); 267 | 268 | /** 269 | * Converts a Quill formatting attr (key/value pair) to the format 270 | * we store in Formatting. 271 | */ 272 | function quillAttrToFormatting( 273 | attr: [key: string, value: any] 274 | ): [key: string, value: any] { 275 | const [key, value] = attr; 276 | if (exclusiveBlocks.has(key)) { 277 | // Wrap it in our own "block" formatting attribute. 278 | // See the comment above exclusiveBlocks. 279 | if (value === null) return ["block", null]; 280 | else return ["block", JSON.stringify([key, value])]; 281 | } else { 282 | return [key, value]; 283 | } 284 | } 285 | 286 | /** 287 | * Inverse of quillAttrToFormatting, except acting on a whole object at a time. 288 | */ 289 | function formattingToQuillAttr( 290 | attrs: Record 291 | ): Record { 292 | const ret: Record = {}; 293 | for (const [key, value] of Object.entries(attrs)) { 294 | if (key === "block") { 295 | if (value === null) { 296 | // Instead of figuring out which block key is being unmarked, 297 | // just ask Quill to unmark all of them. 298 | for (const blockKey of exclusiveBlocks) ret[blockKey] = null; 299 | } else { 300 | const [quillKey, quillValue] = JSON.parse(value) as [string, any]; 301 | ret[quillKey] = quillValue; 302 | } 303 | } else ret[key] = value; 304 | } 305 | return ret; 306 | } 307 | -------------------------------------------------------------------------------- /websocket-quill/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | /* Recommend by Webpack. */ 5 | "module": "ES6", 6 | /* Needed with module: ES6 or else compilation breaks. */ 7 | "moduleResolution": "node", 8 | /* Enable strict type checking. */ 9 | "strict": true, 10 | /* Prevent errors caused by other libraries. */ 11 | "skipLibCheck": true, 12 | /* Enable interop with dependencies using different module systems. */ 13 | "esModuleInterop": true, 14 | /* We don't need to emit declarations. */ 15 | "declaration": false, 16 | /* Emit sourcemap files. */ 17 | "sourceMap": true, 18 | "rootDir": "src" 19 | }, 20 | "include": ["src/site", "src/common", "src/site/old.ts"] 21 | } 22 | -------------------------------------------------------------------------------- /websocket-quill/tsconfig.server.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "ES6", 5 | /* Enable strict type checking. */ 6 | "strict": true, 7 | /* Prevent errors caused by other libraries. */ 8 | "skipLibCheck": true, 9 | /* Enable interop with dependencies using different module systems. */ 10 | "esModuleInterop": true, 11 | /* Use tsc (npm run build) to typecheck only. */ 12 | "noEmit": true, 13 | "rootDir": "src" 14 | }, 15 | "include": ["src/server", "src/common"] 16 | } 17 | -------------------------------------------------------------------------------- /websocket-quill/tsconfig.webpack-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "ES6", 5 | /* Enable strict type checking. */ 6 | "strict": true, 7 | /* Prevent errors caused by other libraries. */ 8 | "skipLibCheck": true, 9 | /* Enable interop with dependencies using different module systems. */ 10 | "esModuleInterop": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /websocket-quill/webpack.config.ts: -------------------------------------------------------------------------------- 1 | import HtmlWebpackPlugin from "html-webpack-plugin"; 2 | import * as path from "path"; 3 | import * as webpack from "webpack"; 4 | 5 | // Basic Webpack config for TypeScript, based on 6 | // https://webpack.js.org/guides/typescript/ . 7 | const config: webpack.Configuration = { 8 | // mode and devtool are overridden by `npm run build` for production mode. 9 | mode: "development", 10 | devtool: "eval-source-map", 11 | entry: "./src/site/main.ts", 12 | module: { 13 | rules: [ 14 | { 15 | test: /\.tsx?$/, 16 | use: "ts-loader", 17 | exclude: /node_modules/, 18 | }, 19 | { 20 | test: /\.js$/, 21 | enforce: "pre", 22 | use: ["source-map-loader"], 23 | }, 24 | { 25 | test: /\.css$/, 26 | use: ["style-loader", "css-loader"], 27 | }, 28 | ], 29 | }, 30 | resolve: { 31 | extensions: [".tsx", ".ts", ".js"], 32 | }, 33 | output: { 34 | filename: "[name].bundle.js", 35 | path: path.resolve(__dirname, "dist"), 36 | clean: true, 37 | }, 38 | plugins: [ 39 | // Use src/index.html as the entry point. 40 | new HtmlWebpackPlugin({ 41 | template: "./src/site/index.html", 42 | }), 43 | ], 44 | }; 45 | 46 | export default config; 47 | --------------------------------------------------------------------------------