├── .gitignore
├── .markdownlint.json
├── test.html
├── test.js
├── rollup.config.js
├── auth.js
├── LICENSE
├── tsconfig.json
├── awareness.test.js
├── package.json
├── README.md
├── sync.js
├── PROTOCOL.md
└── awareness.js
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | dist
3 | *.d.ts
4 | *.d.ts.map
--------------------------------------------------------------------------------
/.markdownlint.json:
--------------------------------------------------------------------------------
1 | {
2 | "default": true,
3 | "no-inline-html": false
4 | }
5 |
--------------------------------------------------------------------------------
/test.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Testing y-protocols
5 |
6 |
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/test.js:
--------------------------------------------------------------------------------
1 | import { runTests } from 'lib0/testing'
2 | import * as log from 'lib0/logging'
3 | import * as awareness from './awareness.test.js'
4 |
5 | import { isBrowser, isNode } from 'lib0/environment'
6 |
7 | /* istanbul ignore if */
8 | if (isBrowser) {
9 | log.createVConsole(document.body)
10 | }
11 |
12 | runTests({
13 | awareness
14 | }).then(success => {
15 | /* istanbul ignore next */
16 | if (isNode) {
17 | process.exit(success ? 0 : 1)
18 | }
19 | })
20 |
--------------------------------------------------------------------------------
/rollup.config.js:
--------------------------------------------------------------------------------
1 | import resolve from '@rollup/plugin-node-resolve'
2 | import commonjs from '@rollup/plugin-commonjs'
3 |
4 | const files = ['awareness.js', 'auth.js', 'sync.js', 'test.js']
5 |
6 | export default [{
7 | input: './test.js',
8 | output: {
9 | file: './dist/test.js',
10 | format: 'iife',
11 | sourcemap: true
12 | },
13 | plugins: [
14 | resolve({ mainFields: ['module', 'browser', 'main'] }),
15 | commonjs()
16 | ]
17 | }, {
18 | input: files,
19 | output: {
20 | dir: './dist',
21 | format: 'cjs',
22 | sourcemap: true,
23 | entryFileNames: '[name].cjs',
24 | chunkFileNames: '[name]-[hash].cjs',
25 | paths: /** @param {any} path */ path => {
26 | if (/^lib0\//.test(path)) {
27 | return `lib0/dist/${path.slice(5) + '.cjs'}`
28 | }
29 | return path
30 | }
31 | },
32 | external: /** @param {any} id */ id => /^lib0\/|yjs/.test(id)
33 | }]
34 |
--------------------------------------------------------------------------------
/auth.js:
--------------------------------------------------------------------------------
1 |
2 | import * as Y from '@y/y' // eslint-disable-line
3 | import * as encoding from 'lib0/encoding'
4 | import * as decoding from 'lib0/decoding'
5 |
6 | export const messagePermissionDenied = 0
7 |
8 | /**
9 | * @param {encoding.Encoder} encoder
10 | * @param {string} reason
11 | */
12 | export const writePermissionDenied = (encoder, reason) => {
13 | encoding.writeVarUint(encoder, messagePermissionDenied)
14 | encoding.writeVarString(encoder, reason)
15 | }
16 |
17 | /**
18 | * @callback PermissionDeniedHandler
19 | * @param {any} y
20 | * @param {string} reason
21 | */
22 |
23 | /**
24 | *
25 | * @param {decoding.Decoder} decoder
26 | * @param {Y.Doc} y
27 | * @param {PermissionDeniedHandler} permissionDeniedHandler
28 | */
29 | export const readAuthMessage = (decoder, y, permissionDeniedHandler) => {
30 | switch (decoding.readVarUint(decoder)) {
31 | case messagePermissionDenied: permissionDeniedHandler(y, decoding.readVarString(decoder))
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2019 Kevin Jahns .
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | /* Basic Options */
4 | "target": "es2018",
5 | "lib": ["es2018", "dom"], /* Specify library files to be included in the compilation. */
6 | "allowJs": true, /* Allow javascript files to be compiled. */
7 | "checkJs": true, /* Report errors in .js files. */
8 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
9 | "declaration": true, /* Generates corresponding '.d.ts' file. */
10 | "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
11 | // "outFile": "./index.js", /* Concatenate and emit output to single file. */
12 | "outDir": "dist", // this is overritten by `npm run types`
13 | "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
14 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
15 | "emitDeclarationOnly": true,
16 | "strict": true,
17 | "noImplicitAny": true,
18 | "moduleResolution": "node",
19 | "allowSyntheticDefaultImports": true
20 | },
21 | "include": ["./*.js"],
22 | "exclude": ["./dist", "./node_modules"]
23 | }
24 |
--------------------------------------------------------------------------------
/awareness.test.js:
--------------------------------------------------------------------------------
1 |
2 | import * as Y from '@y/y'
3 | import * as t from 'lib0/testing'
4 | import * as awareness from './awareness'
5 |
6 | /**
7 | * @param {t.TestCase} tc
8 | */
9 | export const testAwareness = tc => {
10 | const doc1 = new Y.Doc()
11 | doc1.clientID = 0
12 | const doc2 = new Y.Doc()
13 | doc2.clientID = 1
14 | const aw1 = new awareness.Awareness(doc1)
15 | const aw2 = new awareness.Awareness(doc2)
16 | aw1.on('update', /** @param {any} p */ ({ added, updated, removed }) => {
17 | const enc = awareness.encodeAwarenessUpdate(aw1, added.concat(updated).concat(removed))
18 | awareness.applyAwarenessUpdate(aw2, enc, 'custom')
19 | })
20 | let lastChangeLocal = /** @type {any} */ (null)
21 | aw1.on('change', /** @param {any} change */ change => {
22 | lastChangeLocal = change
23 | })
24 | let lastChange = /** @type {any} */ (null)
25 | aw2.on('change', /** @param {any} change */ change => {
26 | lastChange = change
27 | })
28 | aw1.setLocalState({ x: 3 })
29 | t.compare(aw2.getStates().get(0), { x: 3 })
30 | t.assert(/** @type {any} */ (aw2.meta.get(0)).clock === 1)
31 | t.compare(lastChange.added, [0])
32 | // When creating an Awareness instance, the the local client is already marked as available, so it is not updated.
33 | t.compare(lastChangeLocal, { added: [], updated: [0], removed: [] })
34 |
35 | // update state
36 | lastChange = null
37 | lastChangeLocal = null
38 | aw1.setLocalState({ x: 4 })
39 | t.compare(aw2.getStates().get(0), { x: 4 })
40 | t.compare(lastChangeLocal, { added: [], updated: [0], removed: [] })
41 | t.compare(lastChangeLocal, lastChange)
42 |
43 | lastChange = null
44 | lastChangeLocal = null
45 | aw1.setLocalState({ x: 4 })
46 | t.assert(lastChange === null)
47 | t.assert(/** @type {any} */ (aw2.meta.get(0)).clock === 3)
48 | t.compare(lastChangeLocal, lastChange)
49 | aw1.setLocalState(null)
50 | t.assert(lastChange.removed.length === 1)
51 | t.compare(aw1.getStates().get(0), undefined)
52 | t.compare(lastChangeLocal, lastChange)
53 | }
54 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@y/protocols",
3 | "version": "1.0.6-3",
4 | "description": "Yjs encoding protocols",
5 | "type": "module",
6 | "funding": {
7 | "type": "GitHub Sponsors ❤",
8 | "url": "https://github.com/sponsors/dmonad"
9 | },
10 | "files": [
11 | "dist/*",
12 | "auth.*",
13 | "sync.*",
14 | "awareness.*"
15 | ],
16 | "scripts": {
17 | "clean": "rm -rf dist *.d.ts */*.d.ts *.d.ts.map */*.d.ts.map",
18 | "dist": "npm run clean && rollup -c",
19 | "test": "npm run lint && npm run dist && node dist/test.cjs",
20 | "lint": "standard && tsc --skipLibCheck",
21 | "types": "tsc --outDir . --skipLibCheck",
22 | "debug": "rollup -c && concurrently 'rollup -wc' 'http-server -o test.html'",
23 | "preversion": "npm run dist && npm run types",
24 | "postpublish": "npm run clean"
25 | },
26 | "repository": {
27 | "type": "git",
28 | "url": "git+https://github.com/yjs/y-protocols.git"
29 | },
30 | "keywords": [
31 | "Yjs"
32 | ],
33 | "author": "Kevin Jahns ",
34 | "license": "MIT",
35 | "standard": {
36 | "ignore": [
37 | "/dist",
38 | "/node_modules"
39 | ]
40 | },
41 | "bugs": {
42 | "url": "https://github.com/yjs/y-protocols/issues"
43 | },
44 | "homepage": "https://github.com/yjs/y-protocols#readme",
45 | "exports": {
46 | "./package.json": "./package.json",
47 | "./sync.js": "./sync.js",
48 | "./dist/sync.cjs": "./dist/sync.cjs",
49 | "./sync": {
50 | "types": "./sync.d.ts",
51 | "module": "./sync.js",
52 | "import": "./sync.js",
53 | "require": "./dist/sync.cjs"
54 | },
55 | "./awareness.js": "./awareness.js",
56 | "./dist/awareness.cjs": "./dist/awareness.cjs",
57 | "./awareness": {
58 | "types": "./awareness.d.ts",
59 | "module": "./awareness.js",
60 | "import": "./awareness.js",
61 | "require": "./dist/awareness.cjs"
62 | },
63 | "./auth.js": "./auth.js",
64 | "./dist/auth.cjs": "./dist/auth.cjs",
65 | "./auth": {
66 | "types": "./auth.d.ts",
67 | "module": "./auth.js",
68 | "import": "./auth.js",
69 | "require": "./dist/auth.cjs"
70 | }
71 | },
72 | "dependencies": {
73 | "lib0": "^0.2.85"
74 | },
75 | "devDependencies": {
76 | "@rollup/plugin-commonjs": "^25.0.4",
77 | "@rollup/plugin-node-resolve": "^15.2.1",
78 | "@types/node": "^20.6.2",
79 | "concurrently": "^5.3.0",
80 | "rollup": "^3.29.2",
81 | "standard": "^12.0.1",
82 | "typescript": "^5.8.3",
83 | "@y/y": "^14.0.0-16"
84 | },
85 | "peerDependencies": {
86 | "@y/y": "^14.0.0-16 || ^14"
87 | },
88 | "engines": {
89 | "npm": ">=8.0.0",
90 | "node": ">=16.0.0"
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Yjs Protocols
2 | > Binary encoding protocols for *syncing*, *awareness*, and *history information*
3 |
4 | This API is unstable and subject to change.
5 |
6 | ## API
7 |
8 | ### Awareness Protocol
9 |
10 | ```js
11 | import * as awarenessProtocol from 'y-protocols/awareness.js'
12 | ```
13 |
14 | The Awareness protocol implements a simple network agnostic algorithm that
15 | manages user status (who is online?) and propagate awareness information like
16 | cursor location, username, or email address. Each client can update its own
17 | local state and listen to state changes of remote clients.
18 |
19 | Each client has an awareness state. Remote awareness are stored in a Map that
20 | maps from remote client id to remote awareness state. An *awareness state* is an
21 | increasing clock attached to a schemaless json object.
22 |
23 | Whenever the client changes its local state, it increases the clock and
24 | propagates its own awareness state to all peers. When a client receives a remote
25 | awareness state, and overwrites the clients state if the received state is newer
26 | than the local awareness state for that client. If the state is `null`, the
27 | client is marked as offline. If a client doesn't receive updates from a remote
28 | peer for 30 seconds, it marks the remote client as offline. Hence each client
29 | must broadcast its own awareness state in a regular interval to make sure that
30 | remote clients don't mark it as offline.
31 |
32 | #### awarenessProtocol.Awareness Class
33 |
34 | ```js
35 | const ydoc = new Y.Doc()
36 | const awareness = new awarenessProtocol.Awareness(ydoc)
37 | ```
38 |
39 |
40 | clientID:number
41 | - A unique identifier that identifies this client.
42 | getLocalState():Object<string,any>|null
43 | - Get the local awareness state.
44 | setLocalState(Object<string,any>|null)
45 | -
46 | Set/Update the local awareness state. Set `null` to mark the local client as
47 | offline.
48 |
49 | setLocalStateField(string, any)
50 | -
51 | Only update a single field on the local awareness object. Does not do
52 | anything if the local state is not set.
53 |
54 | getStates():Map<number,Object<string,any>>
55 | -
56 | Get all client awareness states (remote and local). Maps from clientID to
57 | awareness state.
58 |
59 |
60 | on('change', ({ added: Array<number>, updated: Array<number>
61 | removed: Array<number> }, [transactionOrigin:any]) => ..)
62 |
63 | -
64 | Listen to remote and local state changes on the awareness instance.
65 |
66 |
67 | on('update', ({ added: Array<number>, updated: Array<number>
68 | removed: Array<number> }, [transactionOrigin:any]) => ..)
69 |
70 | -
71 | Listen to remote and local awareness changes on the awareness instance.
72 | This event is called even when the awarenes state does not change.
73 |
74 |
75 |
76 | ### License
77 |
78 | [The MIT License](./LICENSE) © Kevin Jahns
79 |
--------------------------------------------------------------------------------
/sync.js:
--------------------------------------------------------------------------------
1 | /**
2 | * @module sync-protocol
3 | */
4 |
5 | import * as encoding from 'lib0/encoding'
6 | import * as decoding from 'lib0/decoding'
7 | import * as Y from '@y/y'
8 |
9 | /**
10 | * @typedef {Map} StateMap
11 | */
12 |
13 | /**
14 | * Core Yjs defines two message types:
15 | * • YjsSyncStep1: Includes the State Set of the sending client. When received, the client should reply with YjsSyncStep2.
16 | * • YjsSyncStep2: Includes all missing structs and the complete delete set. When received, the client is assured that it
17 | * received all information from the remote client.
18 | *
19 | * In a peer-to-peer network, you may want to introduce a SyncDone message type. Both parties should initiate the connection
20 | * with SyncStep1. When a client received SyncStep2, it should reply with SyncDone. When the local client received both
21 | * SyncStep2 and SyncDone, it is assured that it is synced to the remote client.
22 | *
23 | * In a client-server model, you want to handle this differently: The client should initiate the connection with SyncStep1.
24 | * When the server receives SyncStep1, it should reply with SyncStep2 immediately followed by SyncStep1. The client replies
25 | * with SyncStep2 when it receives SyncStep1. Optionally the server may send a SyncDone after it received SyncStep2, so the
26 | * client knows that the sync is finished. There are two reasons for this more elaborated sync model: 1. This protocol can
27 | * easily be implemented on top of http and websockets. 2. The server should only reply to requests, and not initiate them.
28 | * Therefore it is necessary that the client initiates the sync.
29 | *
30 | * Construction of a message:
31 | * [messageType : varUint, message definition..]
32 | *
33 | * Note: A message does not include information about the room name. This must to be handled by the upper layer protocol!
34 | *
35 | * stringify[messageType] stringifies a message definition (messageType is already read from the bufffer)
36 | */
37 |
38 | export const messageYjsSyncStep1 = 0
39 | export const messageYjsSyncStep2 = 1
40 | export const messageYjsUpdate = 2
41 |
42 | /**
43 | * Create a sync step 1 message based on the state of the current shared document.
44 | *
45 | * @param {encoding.Encoder} encoder
46 | * @param {Y.Doc} doc
47 | */
48 | export const writeSyncStep1 = (encoder, doc) => {
49 | encoding.writeVarUint(encoder, messageYjsSyncStep1)
50 | const sv = Y.encodeStateVector(doc)
51 | encoding.writeVarUint8Array(encoder, sv)
52 | }
53 |
54 | /**
55 | * @param {encoding.Encoder} encoder
56 | * @param {Y.Doc} doc
57 | * @param {Uint8Array} [encodedStateVector]
58 | */
59 | export const writeSyncStep2 = (encoder, doc, encodedStateVector) => {
60 | encoding.writeVarUint(encoder, messageYjsSyncStep2)
61 | encoding.writeVarUint8Array(encoder, Y.encodeStateAsUpdate(doc, encodedStateVector))
62 | }
63 |
64 | /**
65 | * Read SyncStep1 message and reply with SyncStep2.
66 | *
67 | * @param {decoding.Decoder} decoder The reply to the received message
68 | * @param {encoding.Encoder} encoder The received message
69 | * @param {Y.Doc} doc
70 | */
71 | export const readSyncStep1 = (decoder, encoder, doc) =>
72 | writeSyncStep2(encoder, doc, decoding.readVarUint8Array(decoder))
73 |
74 | /**
75 | * Read and apply Structs and then DeleteStore to a y instance.
76 | *
77 | * @param {decoding.Decoder} decoder
78 | * @param {Y.Doc} doc
79 | * @param {any} transactionOrigin
80 | * @param {(error:Error)=>any} [errorHandler]
81 | */
82 | export const readSyncStep2 = (decoder, doc, transactionOrigin, errorHandler) => {
83 | try {
84 | Y.applyUpdate(doc, decoding.readVarUint8Array(decoder), transactionOrigin)
85 | } catch (error) {
86 | if (errorHandler != null) errorHandler(/** @type {Error} */ (error))
87 | // This catches errors that are thrown by event handlers
88 | console.error('Caught error while handling a Yjs update', error)
89 | }
90 | }
91 |
92 | /**
93 | * @param {encoding.Encoder} encoder
94 | * @param {Uint8Array} update
95 | */
96 | export const writeUpdate = (encoder, update) => {
97 | encoding.writeVarUint(encoder, messageYjsUpdate)
98 | encoding.writeVarUint8Array(encoder, update)
99 | }
100 |
101 | /**
102 | * Read and apply Structs and then DeleteStore to a y instance.
103 | *
104 | * @param {decoding.Decoder} decoder
105 | * @param {Y.Doc} doc
106 | * @param {any} transactionOrigin
107 | * @param {(error:Error)=>any} [errorHandler]
108 | */
109 | export const readUpdate = readSyncStep2
110 |
111 | /**
112 | * @param {decoding.Decoder} decoder A message received from another client
113 | * @param {encoding.Encoder} encoder The reply message. Does not need to be sent if empty.
114 | * @param {Y.Doc} doc
115 | * @param {any} transactionOrigin
116 | * @param {(error:Error)=>any} [errorHandler] Optional error handler that catches errors when reading Yjs messages.
117 | */
118 | export const readSyncMessage = (decoder, encoder, doc, transactionOrigin, errorHandler) => {
119 | const messageType = decoding.readVarUint(decoder)
120 | switch (messageType) {
121 | case messageYjsSyncStep1:
122 | readSyncStep1(decoder, encoder, doc)
123 | break
124 | case messageYjsSyncStep2:
125 | readSyncStep2(decoder, doc, transactionOrigin, errorHandler)
126 | break
127 | case messageYjsUpdate:
128 | readUpdate(decoder, doc, transactionOrigin, errorHandler)
129 | break
130 | default:
131 | throw new Error('Unknown message type')
132 | }
133 | return messageType
134 | }
135 |
--------------------------------------------------------------------------------
/PROTOCOL.md:
--------------------------------------------------------------------------------
1 |
2 | # Protocol Spec
3 |
4 | y-protocol implements different binary communication protocols for efficient exchange of information.
5 |
6 | This is the recommended approach to exchange Awareness updates, and syncing Yjs documents & incremental updates.
7 |
8 | ## Base encoding approaches
9 |
10 | We use efficient variable-length encoding where possible.
11 |
12 | The protocol operates on byte arrays. We use the `•` operator to define concatenations on array buffers (i.e. `[1, 2] = [1] • [2]`). A "buffer" shall refer to any byte array.
13 |
14 | * `varUint(number)` - encodes a 53bit unsigned integer to 1-8 bytes.
15 | - unsigned integers are serialized 7 bits at a time, starting with the
16 | least significant bits.
17 | - the most significant bit (msb) in each output byte indicates if there
18 | is a continuation byte (msb = 1).
19 | - A reference implementation can be found in lib0: [encoding.writeVarUint](https://github.com/dmonad/lib0/blob/1ca4b11f355c5ccec25f20ac8d3e2382e6c4303c/encoding.js#L243) [decoding.readVarUint](https://github.com/dmonad/lib0/blob/1ca4b11f355c5ccec25f20ac8d3e2382e6c4303c/decoding.js#L235)
20 | * `varByteArray(buffer) := varUint(length(buffer)) • buffer` - allows us to read any buffer by prepending the size of the buffer
21 | * `utf8(string)` - transforms a string to a utf8-encoded byte array.
22 | * `varString(string) := varByteArray(utf8(string))`
23 | * `json(object) := varString(JSON.stringify(object))` - Write a JavaScript object as a JSON string.
24 |
25 | ### Sync protocol (v1 encoding)
26 |
27 | The Sync protocol defines a few message types that allow two peers to efficiently sync Yjs documents with each other. For more information about Yjs updates and sync messages, please look at [Yjs Docs / Document Updates](https://docs.yjs.dev/api/document-updates).
28 |
29 | We initially sync using the state-vector approach. First, each client sends a `SyncStep1` message to the other peer that contains a `state-vector` (see Yjs docs). When receiving `SyncStep1`, one should reply with `SyncStep2` which contains the missing document updates (`Y.encodeStateAsUpdate(remoteYdoc, sv)`). Once a client receives `SyncStep2`, it knows that it is now synced with the other peer. From now on all changes on the Yjs document should be send to the remote client using an `Update` message containing the update messages generated by Yjs.
30 |
31 | #### Message types
32 |
33 | * `SyncStep1MessageType := 0`
34 | * `SyncStep2MessageType := 1`
35 | * `UpdateMessageType:= 2`
36 |
37 | #### Encodings
38 |
39 | * `syncStep1(sv) := varUint(SyncStep1MessageType) • varByteArray(sv)` - Initial sync request. The state vector can be received by calling `Y.encodeStateVector(ydoc)`.
40 | * `syncStep2(documentState) := varUint(SyncStep2MessageType) • varByteArray(documentState)` - As a reply to `SyncStep1`. The document state can be received by calling `Y.encodeStateAsUpdate(ydoc, sv)`.
41 | * `documentUpdate(update) := varUint(UpdateMessageType) • varByteArray(update)` - Incremental updates the Yjs event handler `Y.on('update', update => sendUpdate(update))`. The receiving part should apply incremental updates to the Yjs document `Y.applyUpdate(ydoc, update)`.
42 |
43 | ### Awareness protocol
44 |
45 | The Awareness protocol synchronizes the pure state-based Awareness CRDT between peers. This can be useful to exchange ephemeral data like presence, cursor positions, etc..
46 |
47 | Each peer is allocated a unique entry in the Awareness CRDT that only they can modify. Eventually, this state is going to be removed by either a timeout or by the owner setting the state to `null`.
48 |
49 | Since the Awareness CRDT is purely state-based, we always exchange the whole state of all locally known clients. Eventually, all Awareness CRDT instances will synchronize. The Awareness CRDT must remove clients from clients that haven't been updated for longer than `30` seconds. With each generated update, the `clock: uint` counter must increase. The Awareness CRDT must only apply updates if the received `clock` is newer / larger than the currently known `clock` for that client.
50 |
51 | `awarenessUpdate(clients) := varUint(clients.length) • clients.map(client => varUint(client.clientid) • varUint(client.clock) • json(client.state))`
52 |
53 |
54 | ## Combining protocols
55 |
56 | The base protocols can be mixed with your own protocols. The y-protocol package only defines the "base" protocol layers that can be reused across communication providers.
57 |
58 | * `SyncProtocolMessageType := 0`
59 | * `AwarenessProtocolMessageType := 1`
60 |
61 | A message should start with the message-type (e.g. `SyncProtocolMessageType`) and be appended with a specify protocol message (e.g. `SyncStep1MessageType`)
62 |
63 | * E.g. encoding a SyncStep1 message over the communication protocol: `varUint(SyncProtocolMessage) • syncStep1(sv)`
64 | * E.g. encoding an awareness update over the communication protocol: `varUint(AwarenessProtocolMessageType) • awarenessUpdate(clients)`
65 |
66 | A communication provider could parse protocols as follows:
67 |
68 | ```js
69 |
70 | import * as decoding from 'lib0/decoding'
71 | import * as encoding from 'lib0/encoding'
72 | import * as sync from 'y-protocols/sync'
73 | import * as awareness from 'y-protocols/awareness'
74 |
75 | const messageTypes = {
76 | [SyncProtocolMessageType]: sync.readSyncMessage,
77 | [SyncProtocolMessageType]: awareness.readAwarenessMessage,
78 | [YourCustomMessageType]: readCustomMessage
79 | }
80 |
81 | function readMessage (buffer) {
82 | const decoder = decoding.createDecoder(buffer)
83 | const messageType = decoding.readVarUint(decoder)
84 | const replyMessage = encoding.createEncoder()
85 |
86 | const messageHandler = messageTypes[messageType]
87 | if (messageHandler) {
88 | messageHandler(decoder, encoder, ydoc)
89 | if (encoding.length(encoder) > 0) {
90 | // the message handler wants to send a reply (e.g. after receiving SyncStep1 the client should respond with SyncStep2)
91 | provider.sendMessage(encoding.toUint8Array(encoder))
92 | }
93 | } else {
94 | throw new Error('Unknown message type')
95 | }
96 | }
97 |
98 | ```
99 |
100 | ### Handling read-only users
101 |
102 | Yjs itself doesn't distinguish between read-only and read-write users. However, you can enforce that no modifying operations are accepted by the server/peer if the client doesn't have write-access.
103 |
104 | It suffices to read the first two bytes in order to determine whether a message should be accepted from a read-only user.
105 |
106 | * `[0, 0, ..]` is a SyncStep1. It is request to receive the missing state (it contains a state-vector that the server uses to compute the missing updates)
107 | * `[0, 1, ..]` is SyncStep2, which is the reply to a SyncStep1. It contains the missing updates. You want to ignore this message as it contains document updates.
108 | * `[0, 2, ..]` is a regular document update message.
109 | * `[1, ..]` Awareness message. This information is only used to represent shared cursors and the name of each user. However, with enough malice intention you could assign other users temporarily false identities.
110 |
111 | It suffices to block messages that start with `[0, 1, ..]` or `[0, 2, ..]`. Optionally, awareness can be disabled by blocking `[1, ..]`.
112 |
--------------------------------------------------------------------------------
/awareness.js:
--------------------------------------------------------------------------------
1 | /**
2 | * @module awareness-protocol
3 | */
4 |
5 | import * as encoding from 'lib0/encoding'
6 | import * as decoding from 'lib0/decoding'
7 | import * as time from 'lib0/time'
8 | import * as math from 'lib0/math'
9 | import { ObservableV2 } from 'lib0/observable'
10 | import * as f from 'lib0/function'
11 | import * as Y from '@y/y' // eslint-disable-line
12 |
13 | export const outdatedTimeout = 30000
14 |
15 | /**
16 | * @typedef {Object} MetaClientState
17 | * @property {number} MetaClientState.clock
18 | * @property {number} MetaClientState.lastUpdated unix timestamp
19 | */
20 |
21 | /**
22 | * The Awareness class implements a simple shared state protocol that can be used for non-persistent data like awareness information
23 | * (cursor, username, status, ..). Each client can update its own local state and listen to state changes of
24 | * remote clients. Every client may set a state of a remote peer to `null` to mark the client as offline.
25 | *
26 | * Each client is identified by a unique client id (something we borrow from `doc.clientID`). A client can override
27 | * its own state by propagating a message with an increasing timestamp (`clock`). If such a message is received, it is
28 | * applied if the known state of that client is older than the new state (`clock < newClock`). If a client thinks that
29 | * a remote client is offline, it may propagate a message with
30 | * `{ clock: currentClientClock, state: null, client: remoteClient }`. If such a
31 | * message is received, and the known clock of that client equals the received clock, it will override the state with `null`.
32 | *
33 | * Before a client disconnects, it should propagate a `null` state with an updated clock.
34 | *
35 | * Awareness states must be updated every 30 seconds. Otherwise the Awareness instance will delete the client state.
36 | *
37 | * @todo migrate to observable v2
38 | * @extends {ObservableV2<{
39 | * destroy: (awareness:Awareness)=>any,
40 | * update: (updates: { added: number[], updated: number[], removed: number[] }, origin: any) => any,
41 | * change: (changes: { added: number[], updated: number[], removed: number[] }, origin: any) => any
42 | * }>}
43 | */
44 | export class Awareness extends ObservableV2 {
45 | /**
46 | * @param {Y.Doc} doc
47 | */
48 | constructor (doc) {
49 | super()
50 | this.doc = doc
51 | /**
52 | * @type {number}
53 | */
54 | this.clientID = doc.clientID
55 | /**
56 | * Maps from client id to client state
57 | * @type {Map>}
58 | */
59 | this.states = new Map()
60 | /**
61 | * @type {Map}
62 | */
63 | this.meta = new Map()
64 | this._checkInterval = /** @type {any} */ (setInterval(() => {
65 | const now = time.getUnixTime()
66 | if (this.getLocalState() !== null && (outdatedTimeout / 2 <= now - /** @type {{lastUpdated:number}} */ (this.meta.get(this.clientID)).lastUpdated)) {
67 | // renew local clock
68 | this.setLocalState(this.getLocalState())
69 | }
70 | /**
71 | * @type {Array}
72 | */
73 | const remove = []
74 | this.meta.forEach((meta, clientid) => {
75 | if (clientid !== this.clientID && outdatedTimeout <= now - meta.lastUpdated && this.states.has(clientid)) {
76 | remove.push(clientid)
77 | }
78 | })
79 | if (remove.length > 0) {
80 | removeAwarenessStates(this, remove, 'timeout')
81 | }
82 | }, math.floor(outdatedTimeout / 10)))
83 | doc.on('destroy', () => {
84 | this.destroy()
85 | })
86 | this.setLocalState({})
87 | }
88 |
89 | destroy () {
90 | this.emit('destroy', [this])
91 | this.setLocalState(null)
92 | super.destroy()
93 | clearInterval(this._checkInterval)
94 | }
95 |
96 | /**
97 | * @return {Object|null}
98 | */
99 | getLocalState () {
100 | return this.states.get(this.clientID) || null
101 | }
102 |
103 | /**
104 | * @param {Object|null} state
105 | */
106 | setLocalState (state) {
107 | const clientID = this.clientID
108 | const currLocalMeta = this.meta.get(clientID)
109 | const clock = currLocalMeta === undefined ? 0 : currLocalMeta.clock + 1
110 | const prevState = this.states.get(clientID)
111 | if (state === null) {
112 | this.states.delete(clientID)
113 | } else {
114 | this.states.set(clientID, state)
115 | }
116 | this.meta.set(clientID, {
117 | clock,
118 | lastUpdated: time.getUnixTime()
119 | })
120 | const added = []
121 | const updated = []
122 | const filteredUpdated = []
123 | const removed = []
124 | if (state === null) {
125 | removed.push(clientID)
126 | } else if (prevState == null) {
127 | if (state != null) {
128 | added.push(clientID)
129 | }
130 | } else {
131 | updated.push(clientID)
132 | if (!f.equalityDeep(prevState, state)) {
133 | filteredUpdated.push(clientID)
134 | }
135 | }
136 | if (added.length > 0 || filteredUpdated.length > 0 || removed.length > 0) {
137 | this.emit('change', [{ added, updated: filteredUpdated, removed }, 'local'])
138 | }
139 | this.emit('update', [{ added, updated, removed }, 'local'])
140 | }
141 |
142 | /**
143 | * @param {string} field
144 | * @param {any} value
145 | */
146 | setLocalStateField (field, value) {
147 | const state = this.getLocalState()
148 | if (state !== null) {
149 | this.setLocalState({
150 | ...state,
151 | [field]: value
152 | })
153 | }
154 | }
155 |
156 | /**
157 | * @return {Map>}
158 | */
159 | getStates () {
160 | return this.states
161 | }
162 | }
163 |
164 | /**
165 | * Mark (remote) clients as inactive and remove them from the list of active peers.
166 | * This change will be propagated to remote clients.
167 | *
168 | * @param {Awareness} awareness
169 | * @param {Array} clients
170 | * @param {any} origin
171 | */
172 | export const removeAwarenessStates = (awareness, clients, origin) => {
173 | const removed = []
174 | for (let i = 0; i < clients.length; i++) {
175 | const clientID = clients[i]
176 | if (awareness.states.has(clientID)) {
177 | awareness.states.delete(clientID)
178 | if (clientID === awareness.clientID) {
179 | const curMeta = /** @type {MetaClientState} */ (awareness.meta.get(clientID))
180 | awareness.meta.set(clientID, {
181 | clock: curMeta.clock + 1,
182 | lastUpdated: time.getUnixTime()
183 | })
184 | }
185 | removed.push(clientID)
186 | }
187 | }
188 | if (removed.length > 0) {
189 | awareness.emit('change', [{ added: [], updated: [], removed }, origin])
190 | awareness.emit('update', [{ added: [], updated: [], removed }, origin])
191 | }
192 | }
193 |
194 | /**
195 | * @param {Awareness} awareness
196 | * @param {Array} clients
197 | * @return {Uint8Array}
198 | */
199 | export const encodeAwarenessUpdate = (awareness, clients, states = awareness.states) => {
200 | const len = clients.length
201 | const encoder = encoding.createEncoder()
202 | encoding.writeVarUint(encoder, len)
203 | for (let i = 0; i < len; i++) {
204 | const clientID = clients[i]
205 | const state = states.get(clientID) || null
206 | const clock = /** @type {MetaClientState} */ (awareness.meta.get(clientID)).clock
207 | encoding.writeVarUint(encoder, clientID)
208 | encoding.writeVarUint(encoder, clock)
209 | encoding.writeVarString(encoder, JSON.stringify(state))
210 | }
211 | return encoding.toUint8Array(encoder)
212 | }
213 |
214 | /**
215 | * Modify the content of an awareness update before re-encoding it to an awareness update.
216 | *
217 | * This might be useful when you have a central server that wants to ensure that clients
218 | * cant hijack somebody elses identity.
219 | *
220 | * @param {Uint8Array} update
221 | * @param {function(any):any} modify
222 | * @return {Uint8Array}
223 | */
224 | export const modifyAwarenessUpdate = (update, modify) => {
225 | const decoder = decoding.createDecoder(update)
226 | const encoder = encoding.createEncoder()
227 | const len = decoding.readVarUint(decoder)
228 | encoding.writeVarUint(encoder, len)
229 | for (let i = 0; i < len; i++) {
230 | const clientID = decoding.readVarUint(decoder)
231 | const clock = decoding.readVarUint(decoder)
232 | const state = JSON.parse(decoding.readVarString(decoder))
233 | const modifiedState = modify(state)
234 | encoding.writeVarUint(encoder, clientID)
235 | encoding.writeVarUint(encoder, clock)
236 | encoding.writeVarString(encoder, JSON.stringify(modifiedState))
237 | }
238 | return encoding.toUint8Array(encoder)
239 | }
240 |
241 | /**
242 | * @param {Awareness} awareness
243 | * @param {Uint8Array} update
244 | * @param {any} origin This will be added to the emitted change event
245 | */
246 | export const applyAwarenessUpdate = (awareness, update, origin) => {
247 | const decoder = decoding.createDecoder(update)
248 | const timestamp = time.getUnixTime()
249 | const added = []
250 | const updated = []
251 | const filteredUpdated = []
252 | const removed = []
253 | const len = decoding.readVarUint(decoder)
254 | for (let i = 0; i < len; i++) {
255 | const clientID = decoding.readVarUint(decoder)
256 | let clock = decoding.readVarUint(decoder)
257 | const state = JSON.parse(decoding.readVarString(decoder))
258 | const clientMeta = awareness.meta.get(clientID)
259 | const prevState = awareness.states.get(clientID)
260 | const currClock = clientMeta === undefined ? 0 : clientMeta.clock
261 | if (currClock < clock || (currClock === clock && state === null && awareness.states.has(clientID))) {
262 | if (state === null) {
263 | // never let a remote client remove this local state
264 | if (clientID === awareness.clientID && awareness.getLocalState() != null) {
265 | // remote client removed the local state. Do not remote state. Broadcast a message indicating
266 | // that this client still exists by increasing the clock
267 | clock++
268 | } else {
269 | awareness.states.delete(clientID)
270 | }
271 | } else {
272 | awareness.states.set(clientID, state)
273 | }
274 | awareness.meta.set(clientID, {
275 | clock,
276 | lastUpdated: timestamp
277 | })
278 | if (clientMeta === undefined && state !== null) {
279 | added.push(clientID)
280 | } else if (clientMeta !== undefined && state === null) {
281 | removed.push(clientID)
282 | } else if (state !== null) {
283 | if (!f.equalityDeep(state, prevState)) {
284 | filteredUpdated.push(clientID)
285 | }
286 | updated.push(clientID)
287 | }
288 | }
289 | }
290 | if (added.length > 0 || filteredUpdated.length > 0 || removed.length > 0) {
291 | awareness.emit('change', [{
292 | added, updated: filteredUpdated, removed
293 | }, origin])
294 | }
295 | if (added.length > 0 || updated.length > 0 || removed.length > 0) {
296 | awareness.emit('update', [{
297 | added, updated, removed
298 | }, origin])
299 | }
300 | }
301 |
--------------------------------------------------------------------------------