14 |
15 | {page.meta.title || page.frontMatter?.title || page.name}
16 |
17 |
19 | ) })}
20 |
--------------------------------------------------------------------------------
/documentation/pages/blog/test1.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | description: "This is a test description of this post."
3 | ---
4 |
5 | # Test 1
6 |
--------------------------------------------------------------------------------
/documentation/pages/blog/test2.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | description: "This is a test 2 description of this post."
3 | ---
4 |
5 | # Test 2
6 |
--------------------------------------------------------------------------------
/documentation/pages/docs/_meta.json:
--------------------------------------------------------------------------------
1 | {
2 | "getting-started": "Getting started",
3 | "integration-examples": "Integration Examples",
4 | "other-examples": "Other Examples",
5 | "server": "Server",
6 | "api": "API",
7 | "error-handling": "Error Handling",
8 | "security_and_privacy": "Security & Privacy",
9 | "specification": "Specification",
10 | "faq": "FAQ",
11 | "benchmarks": "Benchmarks",
12 | "architecture-design": "Architecture Design",
13 | "future-ideas": "Future Ideas"
14 | }
15 |
--------------------------------------------------------------------------------
/documentation/pages/docs/api/_meta.json:
--------------------------------------------------------------------------------
1 | {
2 | "client": "Client",
3 | "server": "Server"
4 | }
5 |
--------------------------------------------------------------------------------
/documentation/pages/docs/api/server.mdx:
--------------------------------------------------------------------------------
1 | # Server API
2 |
3 | #### createSnapshot
4 |
5 | A callback that receives an object containing the `snapshot` structure sent by clients. The Snapshot should be persistet to a database. In case persisting fails an error should be thrown.
6 |
7 | ```ts
8 | export type CreateSnapshotParams = {
9 | snapshot: SnapshotWithClientData;
10 | };
11 | ```
12 |
13 | #### createUpdate
14 |
15 | A callback that receives an object containing the `update` structure sent by clients. The Update should be persistet to a database. In case persisting fails an error should be thrown.
16 |
17 | ```ts
18 | export type CreateUpdateParams = {
19 | update: Update;
20 | };
21 | ```
22 |
23 | #### getDocument
24 |
25 | A callback that should return the necessary document information. The simples implementation would just return the latest snapshot, the proofs for snapshot ancestor chain and all related updates.
26 |
27 | A more advanced implementation should take into account if the full document or just a delta was requested and idenitfy if and if so which updates are necessary to send and only return those.
28 |
29 | ```ts
30 | export type GetDocumentMode = "complete" | "delta";
31 |
32 | export type GetDocumentParams = {
33 | documentId: string;
34 | knownSnapshotId?: string;
35 | knownSnapshotUpdateClocks?: SnapshotUpdateClocks;
36 | mode: GetDocumentMode;
37 | };
38 | ```
39 |
40 | Note: Depending on the application the `getDocument` can be a `getOrCreateDocument` and it creates a document if the ID has not seen before.
41 |
42 | #### hasAccess
43 |
44 | A callback to verify if the client has read or write access to the requested document. Should return `true` if the client has access and `false` in case not.
45 |
46 | The callback is invoked with the following argument:
47 |
48 | ```tsx
49 | type HasAccessParams =
50 | | {
51 | action: "read";
52 | documentId: string;
53 | websocketSessionKey: string | undefined;
54 | }
55 | | {
56 | action: "write-snapshot" | "write-update" | "send-ephemeral-message";
57 | documentId: string;
58 | publicKey: string;
59 | websocketSessionKey: string | undefined;
60 | };
61 | ```
62 |
63 | #### hasBroadcastAccess
64 |
65 | A callback to verify if clients are allowed to receive a new message. Should an array of `true` or `false` values matching the index of the `websocketSessionKeys` to indicate which client has access.
66 |
67 | The callback is invoked with the following argument:
68 |
69 | ```ts
70 | export type HasBroadcastAccessParams = {
71 | documentId: string;
72 | websocketSessionKeys: string[];
73 | };
74 | ```
75 |
76 | ## additionalAuthenticationDataValidations
77 |
78 | Must be identical to the supported `additionalAuthenticationDataValidations` on the client for data parsing to function correctly.
79 |
80 | ### Documentation Example Server
81 |
82 | A fully functional server can be found here [https://github.com/serenity-kit/secsync/tree/main/examples/backend](https://github.com/serenity-kit/secsync/tree/main/examples/backend). Keep in mind that it accepts any client and does not verify the Websocket session key/token.
83 |
--------------------------------------------------------------------------------
/documentation/pages/docs/architecture-design.mdx:
--------------------------------------------------------------------------------
1 | # Architecture Design
2 |
3 | ## Goal
4 |
5 | The goal is to develop an architecture and with it a protocol to allow multiple users to collaborate on a CRDT based data structure in an end-to-end encrypted way.
6 |
7 | ## Requirements
8 |
9 | ### Actors
10 |
11 | - `User` represents a person interacting with the content.
12 | - `Client` represents the actual instance connecting to the service. A user can have one or multiple clients at the same time.
13 | - `Service` represents the server responsible to receive, persist and deliver information to the clients.
14 |
15 | ### Business Requirements
16 |
17 | - The content must be end-to-end encrypted.
18 | - The same user must be able to interact on the same document with multiple clients.
19 | - Clients must not see each others IP addresses.
20 | - When activated it must be possible to identify who wrote which content.
21 | - The user must be able to start or stop sending and/or receiving updates and be able to send updates batched later.
22 |
23 | ### System level Requirements
24 |
25 | #### Data exchange
26 |
27 | - Must support asynchronous exchange of data. This means participants don't have to be online at the same time, but still can exchange data.
28 | - Must support real-time exchange incl. awareness features e.g. cursor position.
29 | - The architecture must support clients that have to rebuild the CRDT based data structure from ground up.
30 | - The architecture must support local-first clients. These clients can be offline for a while and only sync later once they are connected again.
31 | - The architecture must support multiple CRDT implementations. In detail this means Secsync is a layer on top of a data type, where the operations are commutative. In particular Yjs and automerge should be supported.
32 | - The architecture can, but must not be decentralized. Leveraging a centralized service is a viable option.
33 |
34 | #### Security
35 |
36 | - The content of a document must only be accessible to the participants.
37 | - There are no limitations on meta data e.g. who created how many changes.
38 |
39 | #### Authorization
40 |
41 | The architecture should support two main use-cases:
42 |
43 | - Every client is verifiable through a private-public keypair. The keypairs could come from any kind of Public-Key Infrastructure or Web of Trust system. The scenario here is close groups where the public keys are verified.
44 | - Every client with access to the document ID can retrieve data and only with the shared secret can decrypt it e.g. `www.example.com/doc/{id}#{pake of the shared key}` would allow multiple anonymous participants to collaborate.
45 |
46 | #### Eventual consistency
47 |
48 | All clients receive the same set of content updates (possibly in different orders), and all clients converge to the same view of the document state as they receive the same set of control content updates.
49 |
50 | ## Design Decisions
51 |
52 | ### EphemeralMessages
53 |
54 | The session ID + session counter are stored in the encrypted data.
55 |
56 | The benefit of this design is that the id as well as the counter are not exposed to any MITM (man-in-the-middle) attacker nor the server.
57 |
58 | To make this design work it's important that the sessionId is stored per client and not in one denormalized store per sessionId. Otherwise one client could increase the counter of another making their session basically invalid.
59 |
60 | #### Process
61 |
62 | - initialize -> proof and ask for proof
63 | - validate proof -> respond with a proof
64 | - validate proof
65 | - message
66 |
67 | ### Handling of missing messages and retry strategies
68 |
69 | The snapshots are stored in order - old snapshots can be cleaned out (or at least the updates)
70 |
71 | #### Usecases
72 |
73 | - missing a update (e.g. update comes in not in order)
74 | -> the solution: have an incoming queue
75 | -> challenge: when to abort it
76 | - getting a snapshot that doesn't apply to a known one
77 | -> precondition: what if it is an old snapshot?
78 | -> the solution: reconnect and ask the server for a new version of the document
79 |
80 | #### Incoming queue
81 |
82 | ```tsx
83 | {
84 | [snaphotId]: {
85 | lastUpdatesPerAuthor: {
86 | [authorId]: {
87 | lastUpdate,
88 | updatesReceived
89 | }
90 | },
91 | }
92 | }
93 | ```
94 |
--------------------------------------------------------------------------------
/documentation/pages/docs/faq.mdx:
--------------------------------------------------------------------------------
1 | # FAQ
2 |
3 | ## Is it possible to convert from Yjs e.g. to Automerge?
4 |
5 | Secsync does not provide any conversion functionality. Even if you have the raw changes/updates from one library and apply them in the other the outcome would be different, because they are based on different algorithms.
6 |
7 | A lossy conversion is probably possible but not in scope for this project.
8 |
--------------------------------------------------------------------------------
/documentation/pages/docs/future-ideas.mdx:
--------------------------------------------------------------------------------
1 | ## Throttling updates
2 |
3 | Currently updates are created and sent as soon as possible after adding one or multiple changes. This can lead to a lot of updates being sent in a short period of time and especially when a user is not collaborating in real-time with someone else this is not necessary. In fact bundling more changes together makes it even more efficient as you need download and process less updates.
4 |
5 | Ideally in the future there would be a mode where updates are bundled together based on a timeout if no other active collaborator is present (can be checked with the ephemeralMessage sessions).
6 |
7 | To keep a good real-time experience sending every change as soon as possible is still a good practice. To even improve the experience it might make sense to apply changes one by one instead all at once when multiple active collaborators are present. (can be checked with the ephemeralMessage sessions).
8 |
9 | ## Visualize which user made which change
10 |
11 | - In Automerge the user or device public should be used as an ID.
12 | - One idea was to leverage the clientID for it and assign it the publicKey. This approach would not work.
13 | - The main issue for the userId is that it's only a uin32 which results in only 4 bytes. The publicKey itself is way longer and also some randomness would be good in case users have multiple tabs open on the same web-device using the same publicKey. In the long run it's probably better to build a parallel structure allowing users to sign a transaction and make it verifyable instead of manipulating the `clientID` directly.
14 | - Note: For EphemeralMessages we already have validation. In the user state the publicKey is injected and verified within the `useYjsSync` hook. Details can be found here: https://github.com/serenity-kit/secsync/pull/47
15 |
16 | ### Possible Meta data issues
17 |
18 | - Should users be able to identify which device of a user made a certain change?
19 |
--------------------------------------------------------------------------------
/documentation/pages/docs/integration-examples/_meta.json:
--------------------------------------------------------------------------------
1 | {
2 | "yjs-tiptap": "Tiptap Editor (yjs)",
3 | "yjs-prosemirror": "Prosemirror Editor (yjs)",
4 | "yjs-tldraw": {
5 | "title": "Tldraw Whiteboard (yjs)",
6 | "display": "hidden"
7 | },
8 | "automerge-todos": "Todo List (automerge)"
9 | }
10 |
--------------------------------------------------------------------------------
/documentation/pages/docs/integration-examples/automerge-todos.mdx:
--------------------------------------------------------------------------------
1 | # Automerge Todos Example
2 |
3 | ## Instructions
4 |
5 | - Any change that you make will be encrypted and uploaded to the
6 | server.
7 | - You can refresh the page and the current state will be
8 | reconstructed.
9 | - You can share the current URL and collaborate real-time with others.
10 |
11 | ## Example
12 |
13 | import AutomergeTodosExample from "../../../components/AutomergeTodosExample";
14 | import SimpleExampleWrapper from "../../../components/SimpleExampleWrapper";
15 |
16 | {page.frontMatter?.description}
18 |
21 |
23 | ),
24 | project: {
25 | link: "https://github.com/serenity-kit/secsync",
26 | },
27 | // chat: {
28 | // link: "https://discord.com",
29 | // },
30 | docsRepositoryBase: "https://github.com/serenity-kit/secsync",
31 | footer: {
32 | component: Footer,
33 | },
34 | primaryHue: 232,
35 | components: {
36 | // https://mdxjs.com/table-of-components/
37 | pre: Pre,
38 | code: Code,
39 | p: (props) => ,
40 | table: Table,
41 | th: Th,
42 | tr: Tr,
43 | td: Td,
44 | },
45 | };
46 |
47 | export default config;
48 |
--------------------------------------------------------------------------------
/documentation/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "es5",
4 | "lib": ["dom", "dom.iterable", "esnext"],
5 | "allowJs": true,
6 | "skipLibCheck": true,
7 | "strict": true,
8 | "forceConsistentCasingInFileNames": true,
9 | "noEmit": true,
10 | "incremental": true,
11 | "esModuleInterop": true,
12 | "module": "esnext",
13 | "moduleResolution": "node",
14 | "resolveJsonModule": true,
15 | "isolatedModules": true,
16 | "jsx": "preserve"
17 | },
18 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
19 | "exclude": ["node_modules"]
20 | }
21 |
--------------------------------------------------------------------------------
/examples/backend/.dockerignore:
--------------------------------------------------------------------------------
1 | /node_modules
2 | npm-debug.log
--------------------------------------------------------------------------------
/examples/backend/.env.example:
--------------------------------------------------------------------------------
1 | DATABASE_URL="postgresql://prisma:prisma@localhost:5432/secsync"
--------------------------------------------------------------------------------
/examples/backend/.vscode/extensions.json:
--------------------------------------------------------------------------------
1 | {
2 | "recommendations": ["Prisma.prisma"]
3 | }
4 |
--------------------------------------------------------------------------------
/examples/backend/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM node:20-alpine
2 |
3 | # Create app directory
4 | WORKDIR /usr/src/app
5 |
6 | COPY . .
7 |
8 | EXPOSE $PORT
9 | # necessary for small machines on fly.io to avoid crashing during npm install
10 | ENV NODE_OPTIONS=--max_old_space_size=4096
11 | CMD ["npm", "run", "start:prod" ]
--------------------------------------------------------------------------------
/examples/backend/README.md:
--------------------------------------------------------------------------------
1 | ## Dev
2 |
3 | ```
4 | npx prisma generate
5 | ```
6 |
7 | ## Idea: get inspired by event sourcing
8 |
9 | ## Resources
10 |
11 | - https://dev.to/kspeakman/event-storage-in-postgres-4dk2
12 | - https://softwaremill.com/implementing-event-sourcing-using-a-relational-database/
13 | - https://github.com/evgeniy-khist/postgresql-event-sourcing
14 |
--------------------------------------------------------------------------------
/examples/backend/fly.toml:
--------------------------------------------------------------------------------
1 | # fly.toml app configuration file generated for secsync on 2023-06-27T11:49:39+02:00
2 | #
3 | # See https://fly.io/docs/reference/configuration/ for information about how to use this file.
4 | #
5 |
6 | app = "secsync"
7 | primary_region = "ams"
8 | # necessary for small machines on fly.io to avoid running out of memory
9 | swap_size_mb = 2048
10 |
11 | [deploy]
12 | release_command = "npm run prisma:prod:migrate"
13 |
14 | [env]
15 | PORT = "8080"
16 |
17 | [http_service]
18 | internal_port = 8080
19 | force_https = true
20 | auto_stop_machines = true
21 | auto_start_machines = true
22 | min_machines_running = 0
23 | processes = ["app"]
24 |
--------------------------------------------------------------------------------
/examples/backend/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "backend",
3 | "version": "1.0.0",
4 | "private": true,
5 | "devDependencies": {
6 | "@types/node": "^20.13.0",
7 | "@types/uuid": "^9.0.8",
8 | "@types/ws": "^8.5.10",
9 | "@vercel/ncc": "^0.38.1",
10 | "prettier": "^3.2.5",
11 | "prisma": "^5.14.0",
12 | "ts-node": "10.9.2",
13 | "ts-node-dev": "^2.0.0"
14 | },
15 | "scripts": {
16 | "ts:check": "pnpm tsc --noEmit",
17 | "test": "echo \"No tests yet\"",
18 | "lint": "echo \"No linting setup\"",
19 | "dev": "ts-node-dev --transpile-only --no-notify ./src/index.ts",
20 | "clean": "rm -rf build",
21 | "build": "pnpm install && pnpm clean && pnpm prisma:prod:generate && pnpm ncc build ./src/index.ts -o build",
22 | "deploy": "pnpm build && fly launch",
23 | "prisma:prod:migrate": "npm install -g prisma@5 && DATABASE_URL=$DATABASE_URL prisma migrate deploy",
24 | "prisma:prod:generate": "DATABASE_URL=$DATABASE_URL prisma generate",
25 | "prisma:prod:studio": "DATABASE_URL=$DATABASE_URL prisma studio",
26 | "start:prod": "PORT=$PORT DATABASE_URL=$DATABASE_URL NODE_ENV=production node ./build"
27 | },
28 | "dependencies": {
29 | "@prisma/client": "^5.14.0",
30 | "cors": "^2.8.5",
31 | "express": "^4.19.2",
32 | "libsodium-wrappers": "^0.7.13",
33 | "make-promises-safe": "^5.1.0",
34 | "secsync": "workspace:^",
35 | "secsync-server": "workspace:^",
36 | "uuid": "^9.0.1",
37 | "ws": "^8.17.0"
38 | },
39 | "engines": {
40 | "node": ">=12.2.0"
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/examples/backend/prisma/migrations/20230915120302_initial_schema/migration.sql:
--------------------------------------------------------------------------------
1 | -- CreateTable
2 | CREATE TABLE "Document" (
3 | "id" TEXT NOT NULL,
4 | "activeSnapshotId" TEXT,
5 | "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
6 |
7 | CONSTRAINT "Document_pkey" PRIMARY KEY ("id")
8 | );
9 |
10 | -- CreateTable
11 | CREATE TABLE "Snapshot" (
12 | "id" TEXT NOT NULL,
13 | "latestVersion" INTEGER NOT NULL,
14 | "data" TEXT NOT NULL,
15 | "ciphertextHash" TEXT NOT NULL,
16 | "documentId" TEXT NOT NULL,
17 | "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
18 | "clocks" JSONB NOT NULL,
19 | "parentSnapshotUpdateClocks" JSONB NOT NULL,
20 | "parentSnapshotProof" TEXT NOT NULL,
21 |
22 | CONSTRAINT "Snapshot_pkey" PRIMARY KEY ("id")
23 | );
24 |
25 | -- CreateTable
26 | CREATE TABLE "Update" (
27 | "id" TEXT NOT NULL,
28 | "version" INTEGER NOT NULL,
29 | "data" TEXT NOT NULL,
30 | "snapshotId" TEXT NOT NULL,
31 | "clock" INTEGER NOT NULL,
32 | "pubKey" TEXT NOT NULL
33 | );
34 |
35 | -- CreateIndex
36 | CREATE UNIQUE INDEX "Document_activeSnapshotId_key" ON "Document"("activeSnapshotId");
37 |
38 | -- CreateIndex
39 | CREATE UNIQUE INDEX "Update_id_key" ON "Update"("id");
40 |
41 | -- CreateIndex
42 | CREATE INDEX "Update_id_version_idx" ON "Update"("id", "version");
43 |
44 | -- CreateIndex
45 | CREATE UNIQUE INDEX "Update_snapshotId_version_key" ON "Update"("snapshotId", "version");
46 |
47 | -- CreateIndex
48 | CREATE UNIQUE INDEX "Update_snapshotId_pubKey_clock_key" ON "Update"("snapshotId", "pubKey", "clock");
49 |
50 | -- AddForeignKey
51 | ALTER TABLE "Document" ADD CONSTRAINT "Document_activeSnapshotId_fkey" FOREIGN KEY ("activeSnapshotId") REFERENCES "Snapshot"("id") ON DELETE SET NULL ON UPDATE CASCADE;
52 |
53 | -- AddForeignKey
54 | ALTER TABLE "Snapshot" ADD CONSTRAINT "Snapshot_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "Document"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
55 |
56 | -- AddForeignKey
57 | ALTER TABLE "Update" ADD CONSTRAINT "Update_snapshotId_fkey" FOREIGN KEY ("snapshotId") REFERENCES "Snapshot"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
58 |
--------------------------------------------------------------------------------
/examples/backend/prisma/migrations/migration_lock.toml:
--------------------------------------------------------------------------------
1 | # Please do not edit this file manually
2 | # It should be added in your version-control system (i.e. Git)
3 | provider = "postgresql"
--------------------------------------------------------------------------------
/examples/backend/prisma/schema.prisma:
--------------------------------------------------------------------------------
1 | datasource db {
2 | provider = "postgresql"
3 | url = env("DATABASE_URL")
4 | }
5 |
6 | generator client {
7 | provider = "prisma-client-js"
8 | binaryTargets = ["linux-musl-openssl-3.0.x", "native"]
9 | output = "./generated/output"
10 | }
11 |
12 | model Document {
13 | id String @id
14 | activeSnapshot Snapshot? @relation(name: "activeSnapshot", fields: [activeSnapshotId], references: [id])
15 | activeSnapshotId String? @unique
16 | snapshots Snapshot[]
17 | createdAt DateTime @default(now())
18 | }
19 |
20 | model Snapshot {
21 | id String @id
22 | latestVersion Int
23 | data String
24 | ciphertextHash String
25 | document Document @relation(fields: [documentId], references: [id])
26 | documentId String
27 | updates Update[]
28 | activeSnapshotDocument Document? @relation("activeSnapshot")
29 | createdAt DateTime @default(now())
30 | clocks Json
31 | parentSnapshotUpdateClocks Json
32 | parentSnapshotProof String
33 | }
34 |
35 | model Update {
36 | id String @unique // composed out of snapshotId, pubKey, clock
37 | version Int
38 | data String
39 | snapshot Snapshot @relation(fields: [snapshotId], references: [id])
40 | snapshotId String
41 | clock Int
42 | pubKey String
43 |
44 | @@unique([snapshotId, version])
45 | @@unique([snapshotId, pubKey, clock]) // matches the id
46 | @@index([id, version])
47 | }
48 |
--------------------------------------------------------------------------------
/examples/backend/src/database/createSnapshot.ts:
--------------------------------------------------------------------------------
1 | import sodium from "libsodium-wrappers";
2 | import {
3 | CreateSnapshotParams,
4 | SecsyncSnapshotBasedOnOutdatedSnapshotError,
5 | SecsyncSnapshotMissesUpdatesError,
6 | Snapshot,
7 | compareUpdateClocks,
8 | hash,
9 | } from "secsync";
10 | import { serializeSnapshot } from "../utils/serialize";
11 | import { Prisma, prisma } from "./prisma";
12 |
13 | export async function createSnapshot({ snapshot }: CreateSnapshotParams) {
14 | const MAX_RETRIES = 5;
15 | let retries = 0;
16 | let result: Snapshot;
17 |
18 | // use retries approach as described here: https://www.prisma.io/docs/concepts/components/prisma-client/transactions#transaction-timing-issues
19 | while (retries < MAX_RETRIES) {
20 | try {
21 | result = await prisma.$transaction(
22 | async (prisma) => {
23 | const document = await prisma.document.findUniqueOrThrow({
24 | where: { id: snapshot.publicData.docId },
25 | select: {
26 | activeSnapshot: true,
27 | },
28 | });
29 |
30 | // function sleep(ms) {
31 | // return new Promise((resolve) => setTimeout(resolve, ms));
32 | // }
33 | // await sleep(3000);
34 |
35 | // const random = Math.floor(Math.random() * 10);
36 | // if (random < 8) {
37 | // throw new SecsyncSnapshotBasedOnOutdatedSnapshotError(
38 | // "Snapshot is out of date."
39 | // );
40 | // }
41 |
42 | // const random = Math.floor(Math.random() * 10);
43 | // if (random < 8) {
44 | // throw new SecsyncSnapshotMissesUpdatesError(
45 | // "Snapshot does not include the latest changes."
46 | // );
47 | // }
48 |
49 | if (document.activeSnapshot) {
50 | if (
51 | snapshot.publicData.parentSnapshotId !== undefined &&
52 | snapshot.publicData.parentSnapshotId !==
53 | document.activeSnapshot.id
54 | ) {
55 | throw new SecsyncSnapshotBasedOnOutdatedSnapshotError(
56 | "Snapshot is out of date."
57 | );
58 | }
59 |
60 | const compareUpdateClocksResult = compareUpdateClocks(
61 | // @ts-expect-error the values are parsed by the function
62 | document.activeSnapshot.clocks,
63 | snapshot.publicData.parentSnapshotUpdateClocks
64 | );
65 |
66 | if (!compareUpdateClocksResult.equal) {
67 | throw new SecsyncSnapshotMissesUpdatesError(
68 | "Snapshot does not include the latest changes."
69 | );
70 | }
71 | }
72 |
73 | const newSnapshot = await prisma.snapshot.create({
74 | data: {
75 | id: snapshot.publicData.snapshotId,
76 | latestVersion: 0,
77 | data: JSON.stringify(snapshot),
78 | ciphertextHash: hash(snapshot.ciphertext, sodium),
79 | activeSnapshotDocument: {
80 | connect: { id: snapshot.publicData.docId },
81 | },
82 | document: { connect: { id: snapshot.publicData.docId } },
83 | clocks: {},
84 | parentSnapshotProof: snapshot.publicData.parentSnapshotProof,
85 | parentSnapshotUpdateClocks:
86 | snapshot.publicData.parentSnapshotUpdateClocks,
87 | },
88 | });
89 |
90 | return serializeSnapshot(newSnapshot);
91 | },
92 | {
93 | isolationLevel: Prisma.TransactionIsolationLevel.Serializable,
94 | }
95 | );
96 | break;
97 | } catch (error) {
98 | if (error.code === "P2034") {
99 | retries++;
100 | continue;
101 | }
102 | throw error;
103 | }
104 | }
105 |
106 | return result;
107 | }
108 |
--------------------------------------------------------------------------------
/examples/backend/src/database/createUpdate.ts:
--------------------------------------------------------------------------------
1 | import { CreateUpdateParams, Update } from "secsync";
2 | import { Prisma } from "../../prisma/generated/output";
3 | import { serializeUpdate } from "../utils/serialize";
4 | import { prisma } from "./prisma";
5 |
6 | export async function createUpdate({ update }: CreateUpdateParams) {
7 | const MAX_RETRIES = 5;
8 | let retries = 0;
9 | let result: Update;
10 |
11 | // use retries approach as described here: https://www.prisma.io/docs/concepts/components/prisma-client/transactions#transaction-timing-issues
12 | while (retries < MAX_RETRIES) {
13 | try {
14 | result = await prisma.$transaction(
15 | async (prisma) => {
16 | const snapshot = await prisma.snapshot.findUniqueOrThrow({
17 | where: { id: update.publicData.refSnapshotId },
18 | select: {
19 | latestVersion: true,
20 | clocks: true,
21 | document: { select: { activeSnapshotId: true } },
22 | },
23 | });
24 | if (
25 | snapshot.document.activeSnapshotId !==
26 | update.publicData.refSnapshotId
27 | ) {
28 | throw new Error("Update referencing an out of date snapshot.");
29 | }
30 |
31 | if (
32 | snapshot.clocks &&
33 | typeof snapshot.clocks === "object" &&
34 | !Array.isArray(snapshot.clocks)
35 | ) {
36 | if (snapshot.clocks[update.publicData.pubKey] === undefined) {
37 | if (update.publicData.clock !== 0) {
38 | throw new Error(
39 | `Update clock incorrect. Clock: ${update.publicData.clock}, but should be 0`
40 | );
41 | }
42 | // update the clock for the public key
43 | snapshot.clocks[update.publicData.pubKey] =
44 | update.publicData.clock;
45 | } else {
46 | const expectedClockValue =
47 | // @ts-expect-error
48 | snapshot.clocks[update.publicData.pubKey] + 1;
49 | if (expectedClockValue !== update.publicData.clock) {
50 | throw new Error(
51 | `Update clock incorrect. Clock: ${update.publicData.clock}, but should be ${expectedClockValue}`
52 | );
53 | }
54 | // update the clock for the public key
55 | snapshot.clocks[update.publicData.pubKey] =
56 | update.publicData.clock;
57 | }
58 | }
59 |
60 | await prisma.snapshot.update({
61 | where: { id: update.publicData.refSnapshotId },
62 | data: {
63 | latestVersion: snapshot.latestVersion + 1,
64 | clocks: snapshot.clocks as Prisma.JsonObject,
65 | },
66 | });
67 |
68 | return serializeUpdate(
69 | await prisma.update.create({
70 | data: {
71 | id: `${update.publicData.refSnapshotId}-${update.publicData.pubKey}-${update.publicData.clock}`,
72 | data: JSON.stringify(update),
73 | version: snapshot.latestVersion + 1,
74 | snapshot: {
75 | connect: {
76 | id: update.publicData.refSnapshotId,
77 | },
78 | },
79 | clock: update.publicData.clock,
80 | pubKey: update.publicData.pubKey,
81 | },
82 | })
83 | );
84 | },
85 | {
86 | isolationLevel: Prisma.TransactionIsolationLevel.Serializable,
87 | }
88 | );
89 | break;
90 | } catch (error) {
91 | if (error.code === "P2034") {
92 | retries++;
93 | continue;
94 | }
95 | throw error;
96 | }
97 | }
98 |
99 | return result;
100 | }
101 |
--------------------------------------------------------------------------------
/examples/backend/src/database/getOrCreateDocument.ts:
--------------------------------------------------------------------------------
1 | import { GetDocumentParams } from "packages/secsync/src";
2 | import { serializeSnapshot, serializeUpdates } from "../utils/serialize";
3 | import { prisma } from "./prisma";
4 |
5 | export async function getOrCreateDocument({
6 | documentId,
7 | knownSnapshotId,
8 | knownSnapshotUpdateClocks,
9 | mode,
10 | }: GetDocumentParams) {
11 | return prisma.$transaction(async (prisma) => {
12 | const doc = await prisma.document.findUnique({
13 | where: { id: documentId },
14 | include: { activeSnapshot: { select: { id: true } } },
15 | });
16 |
17 | if (!doc) {
18 | await prisma.document.create({
19 | data: { id: documentId },
20 | });
21 | return {
22 | updates: [],
23 | snapshotProofChain: [],
24 | };
25 | }
26 | if (!doc.activeSnapshot) {
27 | return {
28 | updates: [],
29 | snapshotProofChain: [],
30 | };
31 | }
32 |
33 | let snapshotProofChain: {
34 | id: string;
35 | parentSnapshotProof: string;
36 | ciphertextHash: string;
37 | }[] = [];
38 |
39 | if (knownSnapshotId && knownSnapshotId !== doc.activeSnapshot.id) {
40 | snapshotProofChain = await prisma.snapshot.findMany({
41 | where: { documentId },
42 | cursor: { id: knownSnapshotId },
43 | skip: 1,
44 | select: {
45 | id: true,
46 | parentSnapshotProof: true,
47 | ciphertextHash: true,
48 | createdAt: true,
49 | },
50 | orderBy: { createdAt: "asc" },
51 | });
52 | }
53 |
54 | let lastKnownVersion: number | undefined = undefined;
55 | // in case the last known snapshot is the current one, try to find the lastKnownVersion number
56 | if (knownSnapshotId === doc.activeSnapshot.id) {
57 | const updateIds = Object.entries(knownSnapshotUpdateClocks).map(
58 | ([pubKey, clock]) => {
59 | return `${knownSnapshotId}-${pubKey}-${clock}`;
60 | }
61 | );
62 | const lastUpdate = await prisma.update.findFirst({
63 | where: {
64 | id: { in: updateIds },
65 | },
66 | orderBy: { version: "desc" },
67 | });
68 | if (lastUpdate) {
69 | lastKnownVersion = lastUpdate.version;
70 | }
71 | }
72 |
73 | // fetch the active snapshot with
74 | // - all updates after the last known version if there is one and
75 | // - all updates if there is none
76 | const activeSnapshot = await prisma.snapshot.findUnique({
77 | where: { id: doc.activeSnapshot.id },
78 | include: {
79 | updates:
80 | lastKnownVersion !== undefined
81 | ? {
82 | orderBy: { version: "asc" },
83 | where: {
84 | version: { gt: lastKnownVersion },
85 | },
86 | }
87 | : {
88 | orderBy: { version: "asc" },
89 | },
90 | },
91 | });
92 |
93 | if (mode === "delta" && knownSnapshotId === activeSnapshot.id) {
94 | return {
95 | updates: serializeUpdates(activeSnapshot.updates),
96 | };
97 | }
98 |
99 | return {
100 | snapshot: serializeSnapshot(activeSnapshot),
101 | updates: serializeUpdates(activeSnapshot.updates),
102 | snapshotProofChain: snapshotProofChain.map((snapshotProofChainEntry) => {
103 | return {
104 | snapshotId: snapshotProofChainEntry.id,
105 | parentSnapshotProof: snapshotProofChainEntry.parentSnapshotProof,
106 | snapshotCiphertextHash: snapshotProofChainEntry.ciphertextHash,
107 | };
108 | }),
109 | };
110 | });
111 | }
112 |
--------------------------------------------------------------------------------
/examples/backend/src/database/prisma.ts:
--------------------------------------------------------------------------------
1 | import { PrismaClient } from "../../prisma/generated/output";
2 | export { Prisma } from "../../prisma/generated/output";
3 |
4 | export const prisma = new PrismaClient();
5 |
--------------------------------------------------------------------------------
/examples/backend/src/index.ts:
--------------------------------------------------------------------------------
1 | require("make-promises-safe"); // installs an 'unhandledRejection' handler
2 | import cors from "cors";
3 | import express from "express";
4 | import { createServer } from "http";
5 | import { createWebSocketConnection } from "secsync-server";
6 | import { WebSocketServer } from "ws";
7 | import { createSnapshot as createSnapshotDb } from "./database/createSnapshot";
8 | import { createUpdate as createUpdateDb } from "./database/createUpdate";
9 | import { getOrCreateDocument as getOrCreateDocumentDb } from "./database/getOrCreateDocument";
10 |
11 | async function main() {
12 | // const allowedOrigin =
13 | // process.env.NODE_ENV === "development" || process.env.NODE_ENV === "test"
14 | // ? "http://localhost:3000"
15 | // : "https://www.secsync.com";
16 | const allowedOrigin = "*";
17 | const corsOptions = { credentials: true, origin: allowedOrigin };
18 |
19 | const app = express();
20 | app.use(cors(corsOptions));
21 |
22 | const server = createServer(app);
23 |
24 | const webSocketServer = new WebSocketServer({ noServer: true });
25 | webSocketServer.on(
26 | "connection",
27 | createWebSocketConnection({
28 | getDocument: getOrCreateDocumentDb,
29 | createSnapshot: createSnapshotDb,
30 | createUpdate: createUpdateDb,
31 | hasAccess: async () => true,
32 | hasBroadcastAccess: async ({ websocketSessionKeys }) =>
33 | websocketSessionKeys.map(() => true),
34 | logging: "error",
35 | })
36 | );
37 |
38 | server.on("upgrade", (request, socket, head) => {
39 | // @ts-ignore
40 | webSocketServer.handleUpgrade(request, socket, head, (ws) => {
41 | webSocketServer.emit("connection", ws, request);
42 | });
43 | });
44 |
45 | const port = process.env.PORT ? parseInt(process.env.PORT) : 4000;
46 | server.listen(port, () => {
47 | console.log(`🚀 App ready at http://localhost:${port}/`);
48 | console.log(`🚀 Websocket service ready at ws://localhost:${port}`);
49 | });
50 | }
51 |
52 | main();
53 |
--------------------------------------------------------------------------------
/examples/backend/src/utils/serialize.ts:
--------------------------------------------------------------------------------
1 | import { Snapshot, Update } from "secsync";
2 | import {
3 | Snapshot as DbSnapshot,
4 | Update as DbUpdate,
5 | } from "../../prisma/generated/output";
6 |
7 | export function serializeSnapshot(snapshot: DbSnapshot): Snapshot {
8 | return {
9 | ...JSON.parse(snapshot.data),
10 | };
11 | }
12 |
13 | export function serializeUpdate(update: DbUpdate): Update {
14 | return {
15 | ...JSON.parse(update.data),
16 | };
17 | }
18 |
19 | export function serializeUpdates(updates: DbUpdate[]): Update[] {
20 | return updates.map((update) => {
21 | return serializeUpdate(update);
22 | });
23 | }
24 |
--------------------------------------------------------------------------------
/examples/backend/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "../../tsconfig.json",
3 |
4 | "compilerOptions": {
5 | "sourceMap": true,
6 | "outDir": "dist",
7 | "strict": false,
8 | "lib": ["esnext", "dom"],
9 | "esModuleInterop": true
10 | },
11 |
12 | "include": ["src/**/*"]
13 | }
14 |
--------------------------------------------------------------------------------
/ngrok.yml:
--------------------------------------------------------------------------------
1 | version: 2
2 | tunnels:
3 | backend:
4 | addr: 4000
5 | proto: http
6 | frontend:
7 | addr: 3000
8 | proto: http
9 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "secsync-monorepo",
3 | "private": true,
4 | "workspaces": [
5 | "examples/*",
6 | "packages/*",
7 | "benchmarks/*"
8 | ],
9 | "devDependencies": {
10 | "ts-node": "^10.9.2",
11 | "tsup": "^8.0.2",
12 | "typescript": "^5.4.5"
13 | },
14 | "scripts": {
15 | "ts:check": "pnpm install && pnpm -r ts:check",
16 | "test": "pnpm install && pnpm -r test",
17 | "lint": "pnpm install && pnpm -r lint"
18 | },
19 | "pnpm": {
20 | "overrides": {
21 | "yjs": "^13.6.15",
22 | "@automerge/automerge": "^2.2.2",
23 | "prosemirror-model": "^1.21.0",
24 | "prosemirror-state": "^1.4.3",
25 | "prosemirror-view": "^1.33.7",
26 | "prosemirror-transform": "^1.9.0",
27 | "y-prosemirror": "^1.2.5",
28 | "@tiptap/core": "^2.4.0"
29 | }
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/packages/secsync-react-automerge/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # Changelog
2 |
3 | All notable changes to this project will be documented in this file.
4 |
5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7 |
8 | ## [Unreleased]
9 |
10 | ## [0.5.0] - 2024-06-04
11 |
12 | - Version bump due secsync core version bump
13 |
14 | ## [0.4.0] - 2024-06-01
15 |
16 | ### Changed
17 |
18 | - dependency updates
19 |
20 | ## [0.3.0] - 2024-05-29
21 |
22 | ### Changed
23 |
24 | - Upgrade secsync dependency
25 |
26 | ## [0.2.0] - 2023-09-29
27 |
28 | ### Added
29 |
30 | - Initial version
31 |
--------------------------------------------------------------------------------
/packages/secsync-react-automerge/babel.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | presets: [
3 | ["@babel/preset-env", { targets: { node: "current" } }],
4 | "@babel/preset-typescript",
5 | ],
6 | };
7 |
--------------------------------------------------------------------------------
/packages/secsync-react-automerge/package-json-build-script.js:
--------------------------------------------------------------------------------
1 | const fs = require("fs");
2 | const data = fs.readFileSync("./package.json", { encoding: "utf8", flag: "r" });
3 |
4 | // Display the file data
5 | const dataJson = JSON.parse(data);
6 |
7 | dataJson.module = "index.mjs";
8 | dataJson.types = "index.d.ts";
9 | dataJson.main = "index.js";
10 | dataJson.browser = "index.mjs";
11 |
12 | fs.writeFileSync("./dist/package.json", JSON.stringify(dataJson, null, 2));
13 |
--------------------------------------------------------------------------------
/packages/secsync-react-automerge/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "secsync-react-automerge",
3 | "version": "0.5.0",
4 | "main": "src/index",
5 | "types": "src/index",
6 | "scripts": {
7 | "build": "pnpm tsup && pnpm build:package-json",
8 | "build:package-json": "node package-json-build-script.js",
9 | "prepublishOnly": "pnpm run build",
10 | "test": "echo \"No tests\"",
11 | "ts:check": "pnpm tsc --noEmit",
12 | "lint": "echo \"No linting configured\""
13 | },
14 | "dependencies": {
15 | "@xstate/react": "^4.1.1"
16 | },
17 | "peerDependencies": {
18 | "@automerge/automerge": "^2.2.0",
19 | "react": "*",
20 | "secsync": "*"
21 | },
22 | "devDependencies": {
23 | "@babel/core": "^7.24.6",
24 | "@babel/preset-env": "^7.24.6",
25 | "@babel/preset-typescript": "^7.24.6",
26 | "@types/jest": "^29.5.12",
27 | "@types/react": "^18.3.3",
28 | "jest": "^29.7.0",
29 | "secsync": "workspace:^"
30 | },
31 | "jest": {
32 | "setupFilesAfterEnv": [
33 | "