├── .gitignore ├── jest.config.js ├── src ├── index.ts ├── __tests__ │ └── Clock.test.ts ├── TreeNode.ts ├── LogOpMove.ts ├── Clock.ts ├── OpMove.ts ├── Tree.ts ├── TreeReplica.ts └── State.ts ├── .github └── workflows │ ├── size.yml │ ├── main.yml │ └── release.yml ├── README.md ├── package.json ├── tsconfig.json ├── test └── tree.test.ts └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | .DS_Store 3 | node_modules 4 | dist 5 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | }; -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./Clock"; 2 | export * from "./LogOpMove"; 3 | export * from "./OpMove"; 4 | export * from "./State"; 5 | export * from "./Tree"; 6 | export * from "./TreeNode"; 7 | export * from "./TreeReplica"; 8 | -------------------------------------------------------------------------------- /src/__tests__/Clock.test.ts: -------------------------------------------------------------------------------- 1 | import { Clock } from ".."; 2 | 3 | test("valueOf comparison", () => { 4 | const clock1 = new Clock("a"); 5 | const clock2 = new Clock("b"); 6 | 7 | expect(clock1 < clock2).toBe(true); 8 | 9 | clock1.tick(); 10 | 11 | expect(clock1 < clock2).toBe(false); 12 | }); 13 | -------------------------------------------------------------------------------- /.github/workflows/size.yml: -------------------------------------------------------------------------------- 1 | name: Size 2 | on: [pull_request] 3 | jobs: 4 | size: 5 | runs-on: ubuntu-latest 6 | env: 7 | CI_JOB_NUMBER: 1 8 | steps: 9 | - uses: actions/checkout@v1 10 | - uses: andresz1/size-limit-action@v1 11 | with: 12 | github_token: ${{ secrets.GITHUB_TOKEN }} 13 | -------------------------------------------------------------------------------- /src/TreeNode.ts: -------------------------------------------------------------------------------- 1 | /** A node that is stored in a `Tree` */ 2 | // Logically, each `TreeNode` consists of a triple `(parentId, metadata, childId)`. 3 | // However, in this implementation, the `childId` is stored as the 4 | // keys in `Tree.children` 5 | export class TreeNode { 6 | parentId: Id; 7 | metadata: Metadata; 8 | 9 | constructor(parentId: Id, metadata: Metadata) { 10 | this.parentId = parentId; 11 | this.metadata = metadata; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [push] 3 | jobs: 4 | build: 5 | name: Build, lint, and test on Node ${{ matrix.node }} and ${{ matrix.os }} 6 | 7 | runs-on: ${{ matrix.os }} 8 | strategy: 9 | matrix: 10 | node: ['14.x', '16.x'] 11 | os: [ubuntu-latest, windows-latest, macOS-latest] 12 | 13 | steps: 14 | - name: Checkout repo 15 | uses: actions/checkout@v2 16 | 17 | - name: Use Node ${{ matrix.node }} 18 | uses: actions/setup-node@v1 19 | with: 20 | node-version: ${{ matrix.node }} 21 | 22 | - name: Install deps and build (with cache) 23 | uses: bahmutov/npm-install@v1 24 | 25 | - name: Lint 26 | run: yarn lint 27 | 28 | - name: Test 29 | run: yarn test --ci --coverage --maxWorkers=2 30 | 31 | - name: Build 32 | run: yarn build 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

crdt-tree

2 |

An implementation of a tree Conflict-Free Replicated Data Type (CRDT).

3 | 4 | --- 5 | 6 | This crate aims to be an accurate implementation of the tree crdt algorithm described in the paper: 7 | 8 | [A highly-available move operation for replicated trees and distributed filesystems](https://martin.kleppmann.com/papers/move-op.pdf) by M. Kleppmann, et al. 9 | 10 | Please refer to the paper for a description of the algorithm's properties. 11 | 12 | For clarity, data structures in this implementation are named the same as in the paper (State, Tree) or close to (OpMove --> Move, LogOpMove --> LogOp). Some are not explicitly named in the paper, such as TreeId,TreeMeta, TreeNode, Clock. 13 | 14 | ### Additional References 15 | 16 | - [CRDT: The Hard Parts](https://martin.kleppmann.com/2020/07/06/crdt-hard-parts-hydra.html) 17 | - [Youtube Video: CRDT: The Hard Parts](https://youtu.be/x7drE24geUw) 18 | 19 | ## Usage 20 | 21 | See [test/tree.test.ts](test/tree.test.ts). 22 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | branches: 5 | - main 6 | - beta 7 | jobs: 8 | release: 9 | name: Release 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v2 14 | 15 | - name: Setup Node.js 16 | uses: actions/setup-node@v1 17 | with: 18 | node-version: 12 19 | 20 | - name: Install deps and build (with cache) 21 | uses: bahmutov/npm-install@v1 22 | 23 | - name: Setup semantic-release 24 | run: | 25 | yarn global add semantic-release 26 | echo "$(yarn global bin)" >> $GITHUB_PATH 27 | 28 | - name: Release (NPM) 29 | env: 30 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 31 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 32 | run: semantic-release 33 | 34 | - name: Release (GitHub Package Registry) 35 | NPM_TOKEN: ${{ secrets.GITHUB_TOKEN }} 36 | run: | 37 | echo "{\"publishConfig\":{\"registry\":\"https://npm.pkg.github.com\"}}" >> .npmrc 38 | semantic-release --plugins=@semantic-release/npm 39 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@codesandbox/crdt-tree", 3 | "description": "An accurate implementation of the tree CRDT algorithm described in \"A highly-available move operation for replicated trees and distributed filesystems\"", 4 | "author": "Matan Kushner", 5 | "version": "0.0.0-development", 6 | "license": "MIT", 7 | "main": "dist/index.js", 8 | "typings": "dist/index.d.ts", 9 | "module": "dist/crdt-tree.esm.js", 10 | "files": [ 11 | "dist", 12 | "src" 13 | ], 14 | "engines": { 15 | "node": ">=10" 16 | }, 17 | "scripts": { 18 | "start": "tsdx watch", 19 | "build": "tsdx build", 20 | "test": "tsdx test", 21 | "lint": "tsdx lint", 22 | "prepare": "tsdx build", 23 | "size": "size-limit", 24 | "analyze": "size-limit --why" 25 | }, 26 | "devDependencies": { 27 | "@size-limit/preset-small-lib": "^4.10.2", 28 | "husky": "^6.0.0", 29 | "size-limit": "^4.10.2", 30 | "@weiran.zsd/tsdx": "^0.15.0", 31 | "tslib": "^2.2.0", 32 | "typescript": "^4.2.4" 33 | }, 34 | "size-limit": [ 35 | { 36 | "path": "dist/crdt-tree.cjs.production.min.js", 37 | "limit": "10 KB" 38 | }, 39 | { 40 | "path": "dist/crdt-tree.esm.js", 41 | "limit": "10 KB" 42 | } 43 | ], 44 | "release": { 45 | "branches": [ 46 | "main" 47 | ] 48 | }, 49 | "husky": { 50 | "hooks": { 51 | "pre-commit": "tsdx lint --fix" 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | // see https://www.typescriptlang.org/tsconfig to better understand tsconfigs 3 | "include": ["src", "types"], 4 | "compilerOptions": { 5 | "module": "esnext", 6 | "lib": ["esnext", "dom"], 7 | "importHelpers": true, 8 | // output .d.ts declaration files for consumers 9 | "declaration": true, 10 | // output .js.map sourcemap files for consumers 11 | "sourceMap": true, 12 | // match output dir to input dir. e.g. dist/index instead of dist/src/index 13 | "rootDir": "./src", 14 | // stricter type-checking for stronger correctness. Recommended by TS 15 | "strict": true, 16 | // linter checks for common issues 17 | "noImplicitReturns": true, 18 | "noFallthroughCasesInSwitch": true, 19 | // noUnused* overlap with @typescript-eslint/no-unused-vars, can disable if duplicative 20 | "noUnusedLocals": true, 21 | "noUnusedParameters": true, 22 | // use Node's module resolution algorithm, instead of the legacy TS one 23 | "moduleResolution": "node", 24 | // transpile JSX to React.createElement 25 | "jsx": "react", 26 | // interop between ESM and CJS modules. Recommended by TS 27 | "esModuleInterop": true, 28 | // significant perf increase by skipping checking .d.ts files, particularly those in node_modules. Recommended by TS 29 | "skipLibCheck": true, 30 | // error out if import and file system have a casing mismatch. Recommended by TS 31 | "forceConsistentCasingInFileNames": true, 32 | // `tsdx build` ignores this option, but it is commonly used when type-checking separately with `tsc` 33 | "noEmit": true, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/LogOpMove.ts: -------------------------------------------------------------------------------- 1 | // Implements `LogOpMove`, a log entry used by `State`. 2 | // 3 | // From the paper[1]: 4 | // ---- 5 | // In order to correctly apply move operations, a replica needs 6 | // to maintain not only the current state of the tree, but also 7 | // an operation log. The log is a list of `LogMove` records in 8 | // descending timestamp order. `LogMove` t oldp p m c is similar 9 | // to Move t p m c; the difference is that `LogMove` has an additional 10 | // field oldp of type ('n x 'm) option. This option type means 11 | // the field can either take the value None or a pair of a node ID 12 | // and a metadata field. 13 | // 14 | // When a replica applies a `Move` operation to its tree it 15 | // also records a corresponding LogMove operation in its log. 16 | // The t, p, m, and c fields are taken directly from the Move 17 | // record while the oldp field is filled in based on the 18 | // state of the tree before the move. If c did not exist 19 | // in the tree, oldp is set to None. Else oldp records the 20 | // previous parent metadata of c: if there exist p' and m' 21 | // such that (p', m', c') E tree, then `oldp` is set to `Some(p', m')`. 22 | // The `get_parent()` function implements this. 23 | // ---- 24 | // [1] 25 | 26 | import { OpMove } from "./OpMove"; 27 | import { TreeNode } from "./TreeNode"; 28 | 29 | export interface LogOpMove { 30 | /** The operation being logged */ 31 | op: OpMove; 32 | /** 33 | * Parent and metadata prior to the application of the operation. 34 | * Is `undefined` if the node previously didn't exist in the tree. 35 | * */ 36 | oldNode?: TreeNode; 37 | } 38 | -------------------------------------------------------------------------------- /src/Clock.ts: -------------------------------------------------------------------------------- 1 | export enum Ordering { 2 | Equal, 3 | Greater, 4 | Less 5 | } 6 | 7 | /** Implements a Lamport Clock */ 8 | export class Clock { 9 | actorId: Id; 10 | counter: number; 11 | 12 | constructor(actorId: Id, counter: number = 0) { 13 | this.actorId = actorId; 14 | this.counter = counter; 15 | } 16 | 17 | /** Returns a new Clock with same actor but counter incremented by 1 */ 18 | inc(): Clock { 19 | return new Clock(this.actorId, this.counter + 1); 20 | } 21 | 22 | /** Increments the clock counter and returns a new Clock */ 23 | tick(): Clock { 24 | this.counter += 1; 25 | return new Clock(this.actorId, this.counter); 26 | } 27 | 28 | /** Returns a new clock with the same actor but the counter is the larger of the two */ 29 | merge(clock: Clock): Clock { 30 | return new Clock(this.actorId, Math.max(this.counter, clock.counter)); 31 | } 32 | 33 | /** Compare the ordering of the current Clock with another */ 34 | compare(other: Clock): Ordering { 35 | // Compare Clock's counter with another 36 | if (this.counter > other.counter) return Ordering.Greater; 37 | if (this.counter < other.counter) return Ordering.Less; 38 | 39 | // If counters are equal, order is determined based on actorId 40 | // (this is arbitrary, but deterministic) 41 | if (this.actorId > other.actorId) return Ordering.Greater; 42 | if (this.actorId < other.actorId) return Ordering.Less; 43 | return Ordering.Equal; 44 | } 45 | 46 | /** 47 | * Used to retreive a value's primitive, used in comparisons 48 | * @example 49 | * const clock1 = new Clock('a'); 50 | * const clock2 = new Clock('b'); 51 | * clock1.tick(); 52 | * 53 | * // returns true 54 | * console.log(clock1 > clock2); 55 | */ 56 | valueOf(): string { 57 | return this.toString(); 58 | } 59 | 60 | /** Stringify the current clock into a comparable string */ 61 | toString(): string { 62 | const paddedCounter = String(this.counter).padStart(10, "0"); 63 | return `${paddedCounter}:${this.actorId}`; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/OpMove.ts: -------------------------------------------------------------------------------- 1 | // Implements `OpMove`, the only way to manipulate tree data. 2 | // 3 | // `OpMove` are applied via `State.applyOp` or at a higher 4 | // level via `TreeReplica.applyOp` 5 | // 6 | // From the paper[1]: 7 | // ---- 8 | // We allow the tree to be updated in three ways: by creating 9 | // a new child of any parent node, by deleting a node, or by 10 | // moving a node to be a child of a new parent. However all 11 | // three types of update can be represented by a move operation. 12 | // To create a node, we generate a fresh ID for that node, and 13 | // issue an operation to move this new ID to be created. We 14 | // also designate as "trash" some node ID that does not exist 15 | // in the tree; then we can delete a node by moving it to be 16 | // a child of the trash. 17 | // 18 | // Thus, we define one kind of operation: Move t p m c. A move 19 | // operation is a 4-tuple consisting of a timestamp t of type 't, 20 | // a parent node ID p of type 'n, a metadata field m of type 'm, 21 | // and a child node ID c of type 'n. Here, 't, 'n, and 'm are 22 | // type variables that can be replaced with arbitrary types; 23 | // we only require that node identifiers 'n are globally unique 24 | // (eg UUIDs); timestamps 't need to be globally unique and 25 | // totally ordered (eg Lamport timestamps [11]). 26 | // 27 | // The meaning of an operation Move t p m c is that at time t, 28 | // the node with ID c is moved to be a child of the parent node 29 | // with ID p. The operation does not specify the old location 30 | // of c; the algorithm simply removes c from wherever it is 31 | // currently located in the tree, and moves it to p. If c 32 | // does not currently exist in the tree, it is created as a child 33 | // of p. 34 | // 35 | // The metadata field m in a move operation allows additional 36 | // information to be associated with the parent-child relationship 37 | // of p and c. For example, in a filesystem, the parent and 38 | // child are the inodes of a directory and a file within it 39 | // respectively, and the metadata contains the filename of the 40 | // child. Thus, a file with inode c can be renamed by performing 41 | // a Move t p m c, where the new parent directory p is the inode 42 | // of the existing parent (unchanged), but the metadata m contains 43 | // the new filename. 44 | // 45 | // When users want to make changes to the tree on their local 46 | // replica they generate new Move t p m c operations for these 47 | // changes, and apply these operations using the algorithm 48 | // described... 49 | // ---- 50 | // [1] https://martin.kleppmann.com/papers/move-op.pdf 51 | 52 | import { Clock } from "./Clock"; 53 | 54 | export interface OpMove { 55 | id: Id; 56 | timestamp: Clock; 57 | metadata: Metadata; 58 | parentId: Id; 59 | } 60 | -------------------------------------------------------------------------------- /src/Tree.ts: -------------------------------------------------------------------------------- 1 | // Implements `Tree`, a map of nodes representing the current tree structure. 2 | // 3 | // Normally `Tree` should not be instantiated directly. 4 | // Instead instantiate `State` (lower-level) or `TreeReplica` (higher-level) 5 | // and invoke operations on them. 6 | // 7 | // From the paper[1]: 8 | // ---- 9 | // We can represent the tree as a set of (parent, meta, child) 10 | // triples, denoted in Isabelle/HOL as (’n × ’m × ’n) set. When 11 | // we have (p, m, c) ∈ tree, that means c is a child of p in the tree, 12 | // with associated metadata m. Given a tree, we can construct 13 | // a new tree’ in which the child c is moved to a new parent p, 14 | // with associated metadata m, as follows: 15 | // 16 | // tree’ = {(p’, m’, c’) ∈ tree. c’ != c} ∪ {(p, m, c)} 17 | // 18 | // That is, we remove any existing parent-child relationship 19 | // for c from the set tree, and then add {(p, m, c)} to represent 20 | // the new parent-child relationship. 21 | // ---- 22 | // [1] https://martin.kleppmann.com/papers/move-op.pdf 23 | 24 | import { TreeNode } from "./TreeNode"; 25 | 26 | /** Create a new Tree instance */ 27 | export class Tree { 28 | /** Tree nodes indexed by id */ 29 | nodes: Map> = new Map(); 30 | /** Parent id to child id index */ 31 | children: Map> = new Map(); 32 | 33 | size = this.nodes.size; 34 | 35 | /** Remove a node based on its id */ 36 | remove(id: Id): void { 37 | const entry = this.nodes.get(id); 38 | if (!entry) return; 39 | 40 | let parent = this.children.get(entry.parentId); 41 | if (!parent) return; 42 | parent.delete(id); 43 | // Clean up parent entry if empty 44 | if (parent.size === 0) this.children.delete(entry.parentId); 45 | 46 | this.nodes.delete(id); 47 | } 48 | 49 | /** Add a node to the tree */ 50 | addNode(id: Id, node: TreeNode): void { 51 | let childrenSet = this.children.get(node.parentId); 52 | if (!childrenSet) { 53 | childrenSet = new Set(); 54 | this.children.set(node.parentId, childrenSet); 55 | } 56 | 57 | childrenSet.add(id); 58 | this.nodes.set(id, node); 59 | } 60 | 61 | /** Get a node by its id */ 62 | get(id: Id): TreeNode | undefined { 63 | return this.nodes.get(id); 64 | } 65 | 66 | /** Returns true if the given `ancestorId` is an ancestor of `id` in the tree */ 67 | // 68 | // parent | child 69 | // -------------- 70 | // 1 2 71 | // 1 3 72 | // 3 5 73 | // 2 6 74 | // 6 8 75 | // 76 | // 1 77 | // 2 3 78 | // 6 5 79 | // 8 80 | // 81 | // is 2 ancestor of 8? yes. 82 | // is 2 ancestor of 5? no. 83 | isAncestor(id: Id, ancestorId: Id): boolean { 84 | let targetId = id; 85 | 86 | let node; 87 | while ((node = this.get(targetId))) { 88 | if (node.parentId === ancestorId) return true; 89 | targetId = node.parentId; 90 | } 91 | 92 | return false; 93 | } 94 | 95 | /** Print a tree node recursively */ 96 | printNode(id: Id, depth: number = 0) { 97 | const node = this.get(id); 98 | const line = `${id} ${node ? `${JSON.stringify(node.metadata)}` : ""}`; 99 | const indentation = " ".repeat(depth * 2); 100 | console.log(indentation + line); 101 | 102 | let children = Array.from(this.children.get(id) ?? []); 103 | for (let childId of children) { 104 | this.printNode(childId, depth + 1); 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /test/tree.test.ts: -------------------------------------------------------------------------------- 1 | import { OpMove, Tree, TreeReplica } from "../src"; 2 | 3 | let id = 1; 4 | const newId = () => String(++id); 5 | 6 | test("concurrent moves converge to a common location", () => { 7 | const r1 = new TreeReplica("a"); 8 | const r2 = new TreeReplica("b"); 9 | 10 | const ids = { 11 | root: newId(), 12 | a: newId(), 13 | b: newId(), 14 | c: newId() 15 | }; 16 | 17 | const ops = r1.opMoves([ 18 | [ids.root, "root", "0"], 19 | [ids.a, "a", ids.root], 20 | [ids.b, "b", ids.root], 21 | [ids.c, "c", ids.root] 22 | ]); 23 | 24 | r1.applyOps(ops); 25 | r2.applyOps(ops); 26 | 27 | // Replica 1 moves /root/a to /root/b 28 | let repl1Ops = [r1.opMove(ids.a, "a", ids.b)]; 29 | // Replica 2 moves /root/a to /root/c 30 | let repl2Ops = [r2.opMove(ids.a, "a", ids.c)]; 31 | 32 | r1.applyOps(repl1Ops); 33 | r1.applyOps(repl2Ops); 34 | 35 | r2.applyOps(repl2Ops); 36 | r2.applyOps(repl1Ops); 37 | 38 | // The state is the same on both replicas, converging to /root/c/a 39 | // because last-write-wins and replica2's op has a later timestamp 40 | expect(r1.state.toString()).toEqual(r2.state.toString()); 41 | expect(r1.state.tree.nodes.get(ids.a)?.parentId).toBe(ids.c); 42 | }); 43 | 44 | test("concurrent moves avoid cycles, converging to a common location", () => { 45 | const r1 = new TreeReplica("a"); 46 | const r2 = new TreeReplica("b"); 47 | 48 | const ids = { 49 | root: newId(), 50 | a: newId(), 51 | b: newId(), 52 | c: newId() 53 | }; 54 | 55 | const ops = r1.opMoves([ 56 | [ids.root, "root", "0"], 57 | [ids.a, "a", ids.root], 58 | [ids.b, "b", ids.root], 59 | [ids.c, "c", ids.a] 60 | ]); 61 | 62 | r1.applyOps(ops); 63 | r2.applyOps(ops); 64 | 65 | // Replica 1 moves /root/b to /root/a, creating /root/a/b 66 | let repl1Ops = [r1.opMove(ids.b, "b", ids.a)]; 67 | // Replica 2 "simultaneously" moves /root/a to /root/b, creating /root/b/a 68 | let repl2Ops = [r2.opMove(ids.a, "a", ids.b)]; 69 | 70 | r1.applyOps(repl1Ops); 71 | r1.applyOps(repl2Ops); 72 | 73 | r2.applyOps(repl2Ops); 74 | r2.applyOps(repl1Ops); 75 | 76 | // The state is the same on both replicas, converging to /root/a/b 77 | // because last-write-wins and replica2's op has a later timestamp 78 | expect(r1.state.toString()).toEqual(r2.state.toString()); 79 | expect(r1.state.tree.nodes.get(ids.b)?.parentId).toBe(ids.a); 80 | expect(r1.state.tree.nodes.get(ids.a)?.parentId).toBe(ids.root); 81 | }); 82 | 83 | test("custom conflict handler supports metadata-based custom conflicts", () => { 84 | type Id = string; 85 | type FileName = string; 86 | 87 | // A custom handler that rejects if a sibling exists with the same name 88 | function conflictHandler(op: OpMove, tree: Tree) { 89 | const siblings = tree.children.get(op.parentId) ?? []; 90 | return [...siblings].some(id => { 91 | const isSibling = id !== op.id; 92 | const hasSameName = tree.get(id)?.metadata === op.metadata; 93 | return isSibling && hasSameName; 94 | }); 95 | } 96 | 97 | const r1 = new TreeReplica("a", { conflictHandler }); 98 | const r2 = new TreeReplica("b", { conflictHandler }); 99 | 100 | const ids = { 101 | root: newId(), 102 | a: newId(), 103 | b: newId() 104 | }; 105 | 106 | const ops = r1.opMoves([ 107 | [ids.root, "root", "0"], 108 | [ids.a, "a", ids.root], 109 | [ids.b, "b", ids.root] 110 | ]); 111 | 112 | r1.applyOps(ops); 113 | r2.applyOps(ops); 114 | 115 | // Replica 1 renames /root/a to /root/b, producing a conflict 116 | let repl1Ops = [r1.opMove(ids.a, "b", ids.root)]; 117 | 118 | r1.applyOps(repl1Ops); 119 | r2.applyOps(repl1Ops); 120 | 121 | // The state is the same on both replicas, ignoring the operation that 122 | // produced conflicting metadata state 123 | expect(r1.state.toString()).toEqual(r2.state.toString()); 124 | expect(r1.state.tree.nodes.get(ids.a)?.metadata).toBe("a"); 125 | }); 126 | -------------------------------------------------------------------------------- /src/TreeReplica.ts: -------------------------------------------------------------------------------- 1 | // `TreeReplica` holds tree `State` plus lamport timestamp (actor + counter) 2 | // 3 | // It can optionally keep track of the latest timestamp for each 4 | // replica which is needed for calculating the causally stable threshold which 5 | // is in turn needed for log truncation. 6 | // 7 | // `TreeReplica` is a higher-level interface to the Tree CRDT and is tied to a 8 | // particular actor/peer. 9 | // 10 | // `State` is a lower-level interface to the Tree CRDT and is not tied to any 11 | // actor/peer. 12 | 13 | import { Clock } from "./Clock"; 14 | import { OpMove } from "./OpMove"; 15 | import { State } from "./State"; 16 | import { Tree } from "./Tree"; 17 | import { TreeNode } from "./TreeNode"; 18 | 19 | interface ReplicaOptions { 20 | /** 21 | * An function to provide domain-specific conflict handling logic. 22 | * The resulting boolean value determines whether the operation conflicts. 23 | * 24 | * This is useful if metadata collision can produce conflicts in your business 25 | * logic. For example, making name collisions impossible in a filesystem. 26 | */ 27 | conflictHandler?: ( 28 | operation: OpMove, 29 | tree: Tree 30 | ) => boolean; 31 | } 32 | 33 | export class TreeReplica { 34 | /** The Tree state */ 35 | state: State; 36 | /** The logical clock for this replica/tree */ 37 | time: Clock; 38 | /** Mapping of replicas and their latest time */ 39 | latestTimeByReplica: Map> = new Map(); 40 | /** A tree structure that represents the current state of the tree */ 41 | tree: Tree; 42 | 43 | constructor(authorId: Id, options: ReplicaOptions = {}) { 44 | this.time = new Clock(authorId); 45 | this.state = new State(options); 46 | this.tree = this.state.tree; 47 | } 48 | 49 | /** Get a node by its id */ 50 | get(id: Id): TreeNode | undefined { 51 | return this.tree.get(id); 52 | } 53 | 54 | /** 55 | * Generates an OpMove 56 | * Note that `this.time` is not updated until `applyOp` is called. 57 | * 58 | * Therefore, multiple ops generate3d with this method may share the same 59 | * timestamp, and only one can be successfully applied. 60 | * 61 | * To generate multiple ops before calling `applyOp`, use `opMoves` instead. 62 | */ 63 | opMove(id: Id, metadata: Metadata, parentId: Id): OpMove { 64 | return { timestamp: this.time.inc(), metadata, id, parentId }; 65 | } 66 | 67 | /** 68 | * Generates a list of OpMove from a list of tuples (id, metadata, parentId) 69 | * 70 | * Each timestamp will be greater than the previous op in the returned list. 71 | * Therefore, these operations can be successfully applied via `applyOp()` without 72 | * timestamp collision. 73 | */ 74 | opMoves(ops: Array<[Id, Metadata, Id]>): OpMove[] { 75 | const opMoves: OpMove[] = []; 76 | for (const op of ops) { 77 | opMoves.push({ 78 | timestamp: this.time.tick(), 79 | id: op[0], 80 | metadata: op[1], 81 | parentId: op[2] 82 | }); 83 | } 84 | return opMoves; 85 | } 86 | 87 | /** 88 | * Applies a single operation to `State` and updates our clock 89 | * Also records the latest timestamp for each replica. 90 | */ 91 | applyOp(op: OpMove) { 92 | this.time = this.time.merge(op.timestamp); 93 | 94 | const id = op.timestamp.actorId; 95 | const latestTimeOfActor = this.latestTimeByReplica.get(id) ?? 0; 96 | if (op.timestamp <= latestTimeOfActor) { 97 | console.log( 98 | `Clock not increased, current timestamp ${latestTimeOfActor.toString()}, provided is ${op.timestamp.toString()}.` 99 | ); 100 | console.log("Dropping operation."); 101 | } else { 102 | this.latestTimeByReplica.set(id, op.timestamp); 103 | } 104 | 105 | this.state.applyOp(op); 106 | } 107 | 108 | applyOps(ops: OpMove[]) { 109 | for (const op of ops) { 110 | this.applyOp(op); 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/State.ts: -------------------------------------------------------------------------------- 1 | // Holds Tree CRDT state and implements the core algorithm. 2 | // 3 | // `State` is not tied to any actor/peer and should be equal on any 4 | // two replicas where each has applied the same operations. 5 | // 6 | // `State` may be instantiated to manipulate a CRDT Tree or 7 | // alternatively the higher level `TreeReplica` may be used. 8 | // 9 | // This code aims to be an accurate implementation of the 10 | // tree crdt algorithm described in: 11 | // 12 | // "A highly-available move operation for replicated trees 13 | // and distributed filesystems" [1] by Martin Klepmann, et al. 14 | // 15 | // [1] https://martin.kleppmann.com/papers/move-op.pdf 16 | 17 | import { LogOpMove } from "./LogOpMove"; 18 | import { OpMove } from "./OpMove"; 19 | import { Tree } from "./Tree"; 20 | import { TreeNode } from "./TreeNode"; 21 | 22 | interface StateOptions { 23 | /** 24 | * An function to provide domain-specific conflict handling logic. 25 | * The resulting boolean value determines whether the operation conflicts. 26 | * 27 | * This is useful if metadata collision can produce conflicts in your business 28 | * logic. For example, making name collisions impossible in a filesystem. 29 | */ 30 | conflictHandler?: ( 31 | operation: OpMove, 32 | tree: Tree 33 | ) => boolean; 34 | } 35 | 36 | export class State { 37 | /** A list of `LogOpMove` in descending timestamp order */ 38 | readonly operationLog: LogOpMove[] = []; 39 | /** A tree structure that represents the current state of the tree */ 40 | tree: Tree = new Tree(); 41 | /** Returns true if the given operation should be discarded */ 42 | conflictHandler: ( 43 | operation: OpMove, 44 | tree: Tree 45 | ) => boolean; 46 | 47 | constructor(options: StateOptions = {}) { 48 | // Default to not handling conflict 49 | this.conflictHandler = options.conflictHandler ?? (() => false); 50 | } 51 | 52 | /** Insert a log entry to the top of the log */ 53 | addLogEntry(entry: LogOpMove) { 54 | this.operationLog.unshift(entry); 55 | } 56 | 57 | /** 58 | * Applies the given operation at the correct point in the order of events, 59 | * determined by the operation's logical timestamp. 60 | */ 61 | applyOp(op: OpMove) { 62 | if (this.operationLog.length === 0) { 63 | let logEntry = this.doOperation(op); 64 | this.addLogEntry(logEntry); 65 | } else { 66 | const lastOp = this.operationLog[0].op; 67 | if (op.timestamp === lastOp.timestamp) { 68 | // This case should never happen in normal operation 69 | // because it is a requirement that all timestamps are unique. 70 | // However, uniqueness is not strictly enforced in this impl. 71 | // The crdt paper does not even check for this case. 72 | // We just treat it as a no-op. 73 | console.log( 74 | "op with timestamp equal to previous op ignored. (not applied). Every op must have a unique timestamp." 75 | ); 76 | } 77 | if (op.timestamp < lastOp.timestamp) { 78 | const logEntry = this.operationLog.shift()!; 79 | this.undoOp(logEntry); 80 | this.applyOp(op); 81 | this.redoOp(logEntry); 82 | } 83 | if (op.timestamp > lastOp.timestamp) { 84 | let logEntry = this.doOperation(op); 85 | this.addLogEntry(logEntry); 86 | } 87 | } 88 | } 89 | 90 | /** Apply a list of operations */ 91 | applyOps(ops: OpMove[]) { 92 | for (const op of ops) { 93 | this.applyOp(op); 94 | } 95 | } 96 | 97 | /** Perform the provided move operation, outputting the operation's log entry */ 98 | private doOperation(op: OpMove): LogOpMove { 99 | // When a replica applies a `Move` op to its tree, it also records 100 | // a corresponding `LogMove` op in its log. The t, p, m, and c 101 | // fields are taken directly from the `Move` record, while the `oldNode` 102 | // field is filled in based on the state of the tree before the move. 103 | // If c did not exist in the tree, `oldNode` is set to None. Otherwise 104 | // `oldNode` records the previous parent and metadata of c. 105 | const oldNode = this.tree.get(op.id); 106 | 107 | // ensures no cycles are introduced. If the node c 108 | // is being moved, and c is an ancestor of the new parent 109 | // newp, then the tree is returned unmodified, ie the operation 110 | // is ignored. 111 | // Similarly, the operation is also ignored if c == newp 112 | if (op.id === op.parentId || this.tree.isAncestor(op.parentId, op.id)) { 113 | return { op, oldNode }; 114 | } 115 | 116 | // ignores operations that produce conflicts according to the 117 | // custom conflict handler. 118 | if (this.conflictHandler(op, this.tree)) { 119 | return { op, oldNode }; 120 | } 121 | 122 | // Otherwise, the tree is updated by removing c from 123 | // its existing parent, if any, and adding the new 124 | // parent-child relationship (newp, m, c) to the tree. 125 | this.tree.remove(op.id); 126 | let node = new TreeNode(op.parentId, op.metadata); 127 | this.tree.addNode(op.id, node); 128 | return { op, oldNode }; 129 | } 130 | 131 | /** Undo a previously made operation */ 132 | private undoOp(log: LogOpMove): void { 133 | this.tree.remove(log.op.id); 134 | if (!log.oldNode) return; 135 | 136 | let node = new TreeNode(log.oldNode.parentId, log.oldNode.metadata); 137 | this.tree.addNode(log.op.id, node); 138 | } 139 | 140 | /** 141 | * Reperforms an operation, recomputing the `LogOpMove` record due to the 142 | * effect of the new operation 143 | */ 144 | private redoOp(log: LogOpMove): void { 145 | let op = log.op; 146 | let redoLog = this.doOperation(op); 147 | this.addLogEntry(redoLog); 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------