├── .editorconfig ├── .gitignore ├── .vscode └── settings.json ├── Cargo.toml ├── LICENSE ├── README.md ├── benches ├── automerge-paper.json.gz ├── automerge.rs ├── bench.rs └── rich-text.rs ├── crdt-richtext-wasm ├── .cargo │ └── config.toml ├── .gitignore ├── .npmignore ├── Cargo.toml ├── LICENSE ├── README.md ├── package.json ├── scripts │ └── build.mjs ├── src │ ├── lib.rs │ └── log.rs ├── tests │ └── rich_text.test.ts └── yarn.lock ├── deno.json ├── deno.lock ├── examples └── bench.rs ├── fuzz ├── .gitignore ├── Cargo.lock ├── Cargo.toml └── fuzz_targets │ ├── basic.rs │ ├── five-actors.rs │ ├── rich-text-apply.rs │ ├── rich-text-match.rs │ ├── rich-text-utf16.rs │ └── rich-text.rs ├── js ├── .gitignore ├── package.json ├── packages │ └── cr-quill │ │ ├── .gitignore │ │ ├── .vite │ │ └── deps_temp_cd4bf69d │ │ │ └── package.json │ │ ├── .vscode │ │ └── extensions.json │ │ ├── README.md │ │ ├── index.html │ │ ├── package.json │ │ ├── public │ │ └── Loro.svg │ │ ├── src │ │ ├── App.vue │ │ ├── assets │ │ │ └── Loro.svg │ │ ├── binding.ts │ │ ├── components │ │ │ └── HelloWorld.vue │ │ ├── main.ts │ │ ├── style.css │ │ └── vite-env.d.ts │ │ ├── tsconfig.json │ │ ├── tsconfig.node.json │ │ └── vite.config.ts ├── pnpm-lock.yaml └── pnpm-workspace.yaml └── src ├── legacy ├── README.md ├── mod.rs ├── range_map.rs ├── range_map │ ├── dumb_impl.rs │ └── tree_impl.rs ├── test.rs └── test_utils.rs ├── lib.rs ├── rich_text.rs ├── rich_text ├── ann.rs ├── cursor.rs ├── delta.rs ├── encoding.rs ├── error.rs ├── event.rs ├── id_map.rs ├── iter.rs ├── op.rs ├── rich_tree.rs ├── rich_tree │ ├── query.rs │ ├── rich_tree_btree_impl.rs │ └── utf16.rs ├── test.rs ├── test_utils.rs ├── test_utils │ └── fuzz_line_breaks.rs └── vv.rs ├── small_set.rs └── test_utils.rs /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.rs] 2 | indent_size = 4 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | *.log 4 | flamegraph*.svg 5 | dhat-heap.json 6 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "rust-analyzer.cargo.features": ["test"], 3 | "cSpell.words": ["flamegraph", "unbold"], 4 | "rust-analyzer.runnableEnv": { 5 | "DEBUG": "*" 6 | }, 7 | "deno.enable": false 8 | } 9 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "crdt-richtext" 3 | version = "0.1.1" 4 | edition = "2021" 5 | license = "MIT" 6 | readme = "README.md" 7 | homepage = "https://github.com/loro-dev/crdt-richtext" 8 | description = "Richtext CRDT, Rust implementation of Peritext and Fugue" 9 | repository = "https://github.com/loro-dev/crdt-richtext" 10 | authors = ["zxch3n "] 11 | keywords = ["crdt", "p2p", "richtext", "text-editing"] 12 | include = ["Cargo.toml", "src/**/*.rs", "benches/**/*.rs", "examples/**/*.rs"] 13 | documentation = "https://docs.rs/crdt-richtext" 14 | 15 | [workspace] 16 | members = ["./crdt-richtext-wasm"] 17 | 18 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 19 | 20 | [dependencies] 21 | crdt-list = { version = "0.4.1", optional = true, features = ["fuzzing"] } 22 | rand = { version = "0.8.5", optional = true } 23 | arbitrary = { version = "1.2.3", optional = true } 24 | enum-as-inner = "0.5.1" 25 | arref = "0.1.0" 26 | debug-log = "0.1.4" 27 | heapless = "0.7.16" 28 | fxhash = "0.2.1" 29 | generic-btree = { version = "0.3.1" } 30 | bitvec = "1.0.1" 31 | append-only-bytes = { version = "0.1.5", features = ["u32_range"] } 32 | string_cache = "0.8.6" 33 | smallvec = "1.10.0" 34 | serde_columnar = "0.2.5" 35 | serde = { version = "1.0.140", features = ["derive"] } 36 | flate2 = "1.0.25" 37 | serde_json = "1.0" 38 | thiserror = "1.0" 39 | 40 | [dev-dependencies] 41 | rand = { version = "0.8.5" } 42 | pprof = { version = "0.11.1", features = [ 43 | "flamegraph", 44 | "criterion", 45 | "frame-pointer", 46 | ] } 47 | criterion = "0.4.0" 48 | arbtest = "0.2.0" 49 | color-backtrace = "0.5.1" 50 | ctor = "0.1.26" 51 | dhat = "0.3.2" 52 | serde_json = "1.0.94" 53 | arbitrary = { version = "1.2.3", features = ["derive"] } 54 | flate2 = "1.0.25" 55 | 56 | [features] 57 | test = ["crdt-list", "rand", "arbitrary"] 58 | 59 | 60 | [[bench]] 61 | name = "bench" 62 | harness = false 63 | 64 | [[bench]] 65 | name = "rich-text" 66 | harness = false 67 | 68 | [patch.crates-io] 69 | generic-btree = { path = "../generic-btree" } 70 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Zixuan Chen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # crdt-richtext 2 | 3 | > Rust implementation of [Peritext](https://www.inkandswitch.com/peritext/) and 4 | > [Fugue](https://arxiv.org/abs/2305.00583) 5 | 6 | [**📚 See the blog post**](https://loro-dev.notion.site/crdt-richtext-Rust-implementation-of-Peritext-and-Fugue-c49ef2a411c0404196170ac8daf066c0) 7 | 8 | [**🎨 Try online Demo**](https://crdt-richtext-quill-demo.vercel.app/) 9 | 10 | _The interface is not yet stable and is subject to changes. Do not use it in 11 | production._ 12 | 13 | This CRDT lib combines [Peritext](https://inkandswitch.com/peritext) and 14 | [Fugue](https://arxiv.org/abs/2305.00583)'s power, delivering impressive 15 | performance specifically tailored for rich text. It leverages the 16 | [generic-btree](https://github.com/loro-dev/generic-btree) library to boost 17 | speed, and the [serde-columnar](https://github.com/loro-dev/columnar) simplifies 18 | the implementation of efficient columnar encoding. 19 | 20 | ## Benchmark 21 | 22 | The benchmark was conducted on a 2020 M1 MacBook Pro 13-inch on 2023-05-11. 23 | 24 | The complete benchmark result and code is available 25 | [here](https://github.com/zxch3n/fugue-bench/blob/main/results_table.md). 26 | 27 | | N=6000 | crdt-richtext-wasm | loro-wasm | automerge-wasm | tree-fugue | yjs | ywasm | 28 | | ---------------------------------------------------------------- | ---------------------- | ----------------------- | ------------------- | --------------------------- | ---------------------------- | ------------------- | 29 | | [B4] Apply real-world editing dataset (time) | 176 +/- 10 ms | 141 +/- 15 ms | 821 +/- 7 ms | 721 +/- 15 ms | 1,114 +/- 33 ms | 23,419 +/- 102 ms | 30 | | [B4] Apply real-world editing dataset (memUsed) | skipped | skipped | skipped | 2,373,909 +/- 13725 bytes | 3,480,708 +/- 168887 bytes | skipped | 31 | | [B4] Apply real-world editing dataset (encodeTime) | 8 +/- 1 ms | 8 +/- 1 ms | 115 +/- 2 ms | 12 +/- 0 ms | 12 +/- 1 ms | 6 +/- 1 ms | 32 | | [B4] Apply real-world editing dataset (docSize) | 127,639 +/- 0 bytes | 255,603 +/- 8 bytes | 129,093 +/- 0 bytes | 167,873 +/- 0 bytes | 159,929 +/- 0 bytes | 159,929 +/- 0 bytes | 33 | | [B4] Apply real-world editing dataset (parseTime) | 11 +/- 0 ms | 2 +/- 0 ms | 620 +/- 5 ms | 8 +/- 0 ms | 43 +/- 3 ms | 40 +/- 3 ms | 34 | | [B4x100] Apply real-world editing dataset 100 times (time) | 15,324 +/- 3188 ms | 12,436 +/- 444 ms | skipped | 91,902 +/- 863 ms | 112,563 +/- 3861 ms | skipped | 35 | | [B4x100] Apply real-world editing dataset 100 times (memUsed) | skipped | skipped | skipped | 224076566 +/- 2812359 bytes | 318807378 +/- 15737245 bytes | skipped | 36 | | [B4x100] Apply real-world editing dataset 100 times (encodeTime) | 769 +/- 37 ms | 780 +/- 32 ms | skipped | 943 +/- 52 ms | 297 +/- 16 ms | skipped | 37 | | [B4x100] Apply real-world editing dataset 100 times (docSize) | 12,667,753 +/- 0 bytes | 26,634,606 +/- 80 bytes | skipped | 17,844,936 +/- 0 bytes | 15,989,245 +/- 0 bytes | skipped | 38 | | [B4x100] Apply real-world editing dataset 100 times (parseTime) | 1,252 +/- 14 ms | 170 +/- 15 ms | skipped | 368 +/- 13 ms | 1,335 +/- 238 ms | skipped | 39 | 40 | - The benchmark for Automerge is based on `automerge-wasm`, which is not the 41 | latest version of Automerge 2.0. 42 | - `crdt-richtext` and `fugue` are special-purpose CRDTs that tend to be faster 43 | and have a smaller encoding size. 44 | - The encoding of `yjs`, `ywasm`, and `loro-wasm` still contains redundancy that 45 | can be compressed significantly. For more details, see 46 | [the full report](https://loro.dev/docs/performance/docsize). 47 | - loro-wasm and fugue only support plain text for now 48 | -------------------------------------------------------------------------------- /benches/automerge-paper.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loro-dev/crdt-richtext/26f9425ef74d45937e00d6c8ec2e8bb12889013d/benches/automerge-paper.json.gz -------------------------------------------------------------------------------- /benches/automerge.rs: -------------------------------------------------------------------------------- 1 | use std::io::Read; 2 | 3 | use flate2::read::GzDecoder; 4 | use serde_json::Value; 5 | 6 | pub struct TextAction { 7 | pub pos: usize, 8 | pub ins: String, 9 | pub del: usize, 10 | } 11 | 12 | pub fn get_automerge_actions() -> Vec { 13 | const RAW_DATA: &[u8; 901823] = include_bytes!("./automerge-paper.json.gz"); 14 | let mut actions = Vec::new(); 15 | let mut d = GzDecoder::new(&RAW_DATA[..]); 16 | let mut s = String::new(); 17 | d.read_to_string(&mut s).unwrap(); 18 | let json: Value = serde_json::from_str(&s).unwrap(); 19 | let txns = json.as_object().unwrap().get("txns"); 20 | for txn in txns.unwrap().as_array().unwrap() { 21 | let patches = txn 22 | .as_object() 23 | .unwrap() 24 | .get("patches") 25 | .unwrap() 26 | .as_array() 27 | .unwrap(); 28 | for patch in patches { 29 | let pos = patch[0].as_u64().unwrap() as usize; 30 | let del_here = patch[1].as_u64().unwrap() as usize; 31 | let ins_content = patch[2].as_str().unwrap(); 32 | actions.push(TextAction { 33 | pos, 34 | ins: ins_content.to_string(), 35 | del: del_here, 36 | }); 37 | } 38 | } 39 | actions 40 | } 41 | -------------------------------------------------------------------------------- /benches/bench.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | 3 | use crdt_richtext::{ 4 | legacy::RangeMap, legacy::TreeRangeMap, Anchor, AnchorRange, AnchorType, Annotation, Behavior, 5 | OpID, 6 | }; 7 | use criterion::{criterion_group, criterion_main, Criterion}; 8 | use pprof::flamegraph::{Direction, Options}; 9 | use rand::{Rng, SeedableRng}; 10 | use string_cache::DefaultAtom; 11 | 12 | struct PProfGuard { 13 | path: String, 14 | guard: pprof::ProfilerGuard<'static>, 15 | } 16 | 17 | impl PProfGuard { 18 | #[must_use] 19 | pub fn new(name: &str) -> Self { 20 | let guard = pprof::ProfilerGuard::new(100).unwrap(); 21 | Self { 22 | path: name.to_string(), 23 | guard, 24 | } 25 | } 26 | } 27 | 28 | impl Drop for PProfGuard { 29 | fn drop(&mut self) { 30 | if let Ok(report) = self.guard.report().build() { 31 | let file = File::create(self.path.as_str()).unwrap(); 32 | let mut options = Options::default(); 33 | options.direction = Direction::Inverted; 34 | report.flamegraph_with_options(file, &mut options).unwrap(); 35 | }; 36 | } 37 | } 38 | 39 | fn a(n: u64) -> Annotation { 40 | Annotation { 41 | id: OpID::new(n, 0), 42 | range_lamport: (0, OpID::new(n, 0)), 43 | range: AnchorRange { 44 | start: Anchor { 45 | id: Some(OpID::new(n, 0)), 46 | type_: AnchorType::Before, 47 | }, 48 | end: Anchor { 49 | id: Some(OpID::new(n, 0)), 50 | type_: AnchorType::Before, 51 | }, 52 | }, 53 | behavior: Behavior::Merge, 54 | type_: DefaultAtom::from(""), 55 | value: serde_json::Value::Null, 56 | } 57 | } 58 | 59 | #[cfg(feature = "test")] 60 | pub fn bench(c: &mut Criterion) { 61 | fuzz(c); 62 | real(c); 63 | } 64 | 65 | fn real(c: &mut Criterion) { 66 | let mut b = c.benchmark_group("real"); 67 | b.bench_function("annotate to 1000 annotations", |b| { 68 | let guard = PProfGuard::new("target/annotate_flamegraph.svg"); 69 | b.iter(|| { 70 | let mut gen = rand::rngs::StdRng::seed_from_u64(0); 71 | let mut map = TreeRangeMap::new(); 72 | map.insert_directly(0, 10000); 73 | for i in 0..1000 { 74 | let start = gen.gen_range(0..10000); 75 | let end = gen.gen_range(start..10000); 76 | map.annotate(start, end - start, a(i)); 77 | } 78 | }); 79 | drop(guard); 80 | }); 81 | 82 | b.bench_function("random inserts 10K", |b| { 83 | let guard = PProfGuard::new("target/insert_flamegraph.svg"); 84 | b.iter(|| { 85 | let mut gen = rand::rngs::StdRng::seed_from_u64(0); 86 | let mut map = TreeRangeMap::new(); 87 | map.insert_directly(0, 10000); 88 | for i in 0..1000 { 89 | let start = gen.gen_range(0..10000); 90 | let end = gen.gen_range(start..10000); 91 | map.annotate(start, end - start, a(i)); 92 | } 93 | for _ in 0..10_000 { 94 | let start = gen.gen_range(0..10000); 95 | let end = gen.gen_range(start..10000); 96 | map.insert_directly(start, end - start); 97 | } 98 | }); 99 | drop(guard); 100 | }); 101 | } 102 | 103 | #[cfg(feature = "test")] 104 | fn fuzz(c: &mut Criterion) { 105 | use arbitrary::Unstructured; 106 | use crdt_richtext::legacy::test_utils::{fuzzing, Action}; 107 | let mut b = c.benchmark_group("fuzz"); 108 | b.bench_function("5 actors 1000 actions", |b| { 109 | let mut data = rand::rngs::StdRng::seed_from_u64(0); 110 | let mut bytes: Vec = Vec::new(); 111 | for _ in 0..10000 { 112 | bytes.push(data.gen()); 113 | } 114 | 115 | let mut u = Unstructured::new(&bytes); 116 | let actions: [Action; 1000] = u.arbitrary().unwrap(); 117 | let actions = actions.to_vec(); 118 | 119 | let guard = PProfGuard::new("target/flamegraph.svg"); 120 | b.iter(|| fuzzing(5, actions.clone())); 121 | drop(guard); 122 | }); 123 | } 124 | 125 | #[cfg(not(feature = "test"))] 126 | pub fn bench(c: &mut Criterion) {} 127 | 128 | criterion_group!(benches, bench); 129 | 130 | criterion_main!(benches); 131 | -------------------------------------------------------------------------------- /benches/rich-text.rs: -------------------------------------------------------------------------------- 1 | use automerge::get_automerge_actions; 2 | use crdt_richtext::rich_text::RichText; 3 | use criterion::{criterion_group, criterion_main, Criterion}; 4 | mod automerge; 5 | 6 | pub fn bench(c: &mut Criterion) { 7 | c.bench_function("automerge", |b| { 8 | let actions = get_automerge_actions(); 9 | b.iter(|| { 10 | let mut text = RichText::new(1); 11 | for action in actions.iter() { 12 | if action.del > 0 { 13 | text.delete(action.pos..action.pos + action.del); 14 | } 15 | if !action.ins.is_empty() { 16 | text.insert(action.pos, &action.ins); 17 | } 18 | } 19 | }) 20 | }); 21 | 22 | c.bench_function("automerge apply", |bench| { 23 | let actions = get_automerge_actions(); 24 | let mut a = RichText::new(1); 25 | for action in actions.iter() { 26 | if action.del > 0 { 27 | a.delete(action.pos..action.pos + action.del); 28 | } 29 | if !action.ins.is_empty() { 30 | a.insert(action.pos, &action.ins); 31 | } 32 | } 33 | 34 | bench.iter(|| { 35 | let mut b = RichText::new(1); 36 | b.merge(&a); 37 | }); 38 | }); 39 | } 40 | 41 | criterion_group!(benches, bench); 42 | criterion_main!(benches); 43 | -------------------------------------------------------------------------------- /crdt-richtext-wasm/.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [profile.release] 2 | lto = true 3 | -------------------------------------------------------------------------------- /crdt-richtext-wasm/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | bundler/ 4 | nodejs/ 5 | node_modules/ 6 | -------------------------------------------------------------------------------- /crdt-richtext-wasm/.npmignore: -------------------------------------------------------------------------------- 1 | /target 2 | *.log 3 | *.ipynb 4 | flamegraph.svg 5 | target 6 | dhat-heap.json 7 | .DS_Store 8 | node_modules/ 9 | deno_test/ 10 | src/ 11 | wasm-size/ 12 | web-test/ 13 | pkg/ 14 | web/ 15 | tests/ 16 | .cargo/ 17 | -------------------------------------------------------------------------------- /crdt-richtext-wasm/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "crdt-richtext-wasm" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [lib] 9 | crate-type = ["cdylib", "rlib"] 10 | 11 | [dependencies] 12 | js-sys = "0.3.60" 13 | wasm-bindgen = "0.2.83" 14 | console_error_panic_hook = { version = "0.1.6" } 15 | crdt-richtext = { path = "../" } 16 | serde_json = "1.0" 17 | serde-wasm-bindgen = "0.5" 18 | serde = { version = "1.0.163", features = ["derive"] } 19 | -------------------------------------------------------------------------------- /crdt-richtext-wasm/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Zixuan Chen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /crdt-richtext-wasm/README.md: -------------------------------------------------------------------------------- 1 | # crdt-richtext-wasm 2 | 3 | ## Usage 4 | 5 | ```typescript 6 | const text = new RichText(BigInt(2)); 7 | text.insert(0, "123"); 8 | text.annotate(0, 1, "bold", AnnotateType.BoldLike); 9 | text.insert(1, "k"); 10 | { 11 | const spans = text.getAnnSpans(); 12 | expect(spans[0].text).toBe("1k"); 13 | expect(spans[0].annotations).toStrictEqual(new Set(["bold"])); 14 | } 15 | 16 | text.eraseAnn(0, 2, "bold", AnnotateType.BoldLike); 17 | { 18 | const spans = text.getAnnSpans(); 19 | expect(spans[0].text).toBe("1k23"); 20 | expect(spans[0].annotations.size).toBe(0); 21 | } 22 | ``` 23 | -------------------------------------------------------------------------------- /crdt-richtext-wasm/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "crdt-richtext-wasm", 3 | "version": "0.4.0", 4 | "description": "", 5 | "main": "nodejs/crdt_richtext_wasm.js", 6 | "module": "bundler/crdt_richtext_wasm.js", 7 | "scripts": { 8 | "build": "node ./scripts/build.mjs release", 9 | "dev": "node ./scripts/build.mjs dev", 10 | "test": "vitest run", 11 | "watch": "vitest" 12 | }, 13 | "keywords": [], 14 | "author": "", 15 | "license": "ISC", 16 | "devDependencies": { 17 | "execa": "^7.1.1", 18 | "vitest": "^0.31.0" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /crdt-richtext-wasm/scripts/build.mjs: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | import { execa, $ } from "execa"; 3 | import { resolve } from "path"; 4 | import { rmdir } from "fs/promises"; 5 | import { fileURLToPath } from "url"; 6 | import path from "path"; 7 | 8 | const __filename = fileURLToPath(import.meta.url); 9 | 10 | const __dirname = path.dirname(__filename); 11 | 12 | // node build.mjs debug 13 | // node build.mjs release 14 | // node build.mjs release web 15 | // node build.mjs release nodejs 16 | let profile = "dev"; 17 | let profileDir = "debug"; 18 | if (process.argv[2] == "release") { 19 | profile = "release"; 20 | profileDir = "release"; 21 | } 22 | const TARGETS = ["bundler", "nodejs"]; 23 | const startTime = performance.now(); 24 | const WasmDir = resolve(__dirname, ".."); 25 | const WasmFileName = "crdt_richtext_wasm_bg.wasm"; 26 | 27 | console.log(WasmDir); 28 | async function build() { 29 | await cargoBuild(); 30 | if (process.argv[3] != null) { 31 | if (!TARGETS.includes(process.argv[3])) { 32 | throw new Error(`Invalid target [${process.argv[3]}]`); 33 | } 34 | 35 | buildTarget(process.argv[3]); 36 | return; 37 | } 38 | 39 | await Promise.all( 40 | TARGETS.map((target) => { 41 | return buildTarget(target); 42 | }) 43 | ); 44 | 45 | if (profile !== "dev") { 46 | await Promise.all( 47 | TARGETS.map(async (target) => { 48 | const cmd = `wasm-opt -O4 ./${target}/${WasmFileName} -o ./${target}/${WasmFileName}`; 49 | console.log(">", cmd); 50 | await $`wasm-opt -O4 ./${target}/${WasmFileName} -o ./${target}/${WasmFileName}`; 51 | }) 52 | ); 53 | } 54 | 55 | console.log( 56 | "✅", 57 | "Build complete in", 58 | (performance.now() - startTime) / 1000, 59 | "s" 60 | ); 61 | } 62 | 63 | async function cargoBuild() { 64 | const cmd = `cargo build --target wasm32-unknown-unknown --profile ${profile}`; 65 | console.log(cmd); 66 | const status = await $({ 67 | stdio: "inherit", 68 | cwd: WasmDir, 69 | })`cargo build --target wasm32-unknown-unknown --profile ${profile}`; 70 | if (status.failed) { 71 | console.log( 72 | "❌", 73 | "Build failed in", 74 | (performance.now() - startTime) / 1000, 75 | "s" 76 | ); 77 | process.exit(1); 78 | } 79 | } 80 | 81 | async function buildTarget(target) { 82 | console.log("🏗️ Building target", `[${target}]`); 83 | const targetDirPath = resolve(WasmDir, target); 84 | try { 85 | await rmdir(targetDirPath, { recursive: true }); 86 | console.log("Clear directory " + targetDirPath); 87 | } catch (e) {} 88 | 89 | const cmd = `wasm-bindgen --weak-refs --target ${target} --out-dir ${target} ../target/wasm32-unknown-unknown/${profileDir}/crdt_richtext_wasm.wasm`; 90 | console.log(">", cmd); 91 | await $({ 92 | cwd: WasmDir, 93 | stdout: "inherit", 94 | })`wasm-bindgen --weak-refs --target ${target} --out-dir ${target} ../target/wasm32-unknown-unknown/${profileDir}/crdt_richtext_wasm.wasm`; 95 | } 96 | 97 | build(); 98 | -------------------------------------------------------------------------------- /crdt-richtext-wasm/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::{cell::RefCell, collections::HashMap, panic}; 2 | 3 | use crdt_richtext::{ 4 | rich_text::{DeltaItem, IndexType, RichText as RichTextInner}, 5 | Behavior, Expand, Style, 6 | }; 7 | use serde::{Deserialize, Serialize}; 8 | use wasm_bindgen::prelude::*; 9 | 10 | #[wasm_bindgen] 11 | pub struct RichText { 12 | inner: RefCell, 13 | } 14 | 15 | #[wasm_bindgen] 16 | pub enum AnnotateType { 17 | BoldLike, 18 | LinkLike, 19 | } 20 | 21 | #[derive(Serialize, Deserialize)] 22 | struct AnnRange { 23 | start: usize, 24 | end: usize, 25 | expand: Option, 26 | inclusive: Option, 27 | } 28 | 29 | #[wasm_bindgen] 30 | impl RichText { 31 | #[wasm_bindgen(constructor)] 32 | pub fn new(id: u64) -> Self { 33 | let mut text = RichTextInner::new(id); 34 | text.set_event_index_type(IndexType::Utf16); 35 | Self { 36 | inner: RefCell::new(text), 37 | } 38 | } 39 | 40 | pub fn id(&self) -> u64 { 41 | self.inner.borrow().id() 42 | } 43 | 44 | #[wasm_bindgen(skip_typescript)] 45 | pub fn observe(&self, f: js_sys::Function) { 46 | self.inner.borrow_mut().observe(Box::new(move |event| { 47 | let serializer = serde_wasm_bindgen::Serializer::json_compatible(); 48 | let _ = f.call1(&JsValue::NULL, &event.serialize(&serializer).unwrap()); 49 | })); 50 | } 51 | 52 | pub fn insert(&self, index: usize, text: &str) -> Result<(), JsError> { 53 | if index > self.length() { 54 | return Err(JsError::new("index out of range")); 55 | } 56 | 57 | self.inner.borrow_mut().insert_utf16(index, text); 58 | Ok(()) 59 | } 60 | 61 | pub fn delete(&self, index: usize, length: usize) -> Result<(), JsError> { 62 | if index + length > self.length() { 63 | return Err(JsError::new("index out of range")); 64 | } 65 | 66 | self.inner.borrow_mut().delete_utf16(index..index + length); 67 | Ok(()) 68 | } 69 | 70 | #[allow(clippy::inherent_to_string)] 71 | #[wasm_bindgen(js_name = "toString")] 72 | pub fn to_string(&self) -> String { 73 | self.inner.borrow().to_string() 74 | } 75 | 76 | #[wasm_bindgen(skip_typescript)] 77 | pub fn annotate(&self, range: JsValue, ann_name: &str, value: JsValue) -> Result<(), JsError> { 78 | let range: AnnRange = serde_wasm_bindgen::from_value(range)?; 79 | 80 | if range.end > self.length() { 81 | return Err(JsError::new("index out of range")); 82 | } 83 | 84 | let expand: Expand = range 85 | .expand 86 | .as_deref() 87 | .try_into() 88 | .map_err(|_| JsError::new("invalid expand value"))?; 89 | let inclusive = range.inclusive.unwrap_or(false); 90 | let value = serde_wasm_bindgen::from_value(value)?; 91 | 92 | let style = Style { 93 | expand, 94 | behavior: if inclusive { 95 | Behavior::AllowMultiple 96 | } else { 97 | Behavior::Merge 98 | }, 99 | type_: ann_name.into(), 100 | value, 101 | }; 102 | 103 | self.inner 104 | .borrow_mut() 105 | .annotate_utf16(range.start..range.end, style); 106 | Ok(()) 107 | } 108 | 109 | /// TODO: Doc the behavior of expand 110 | #[wasm_bindgen(js_name = "eraseAnn", skip_typescript)] 111 | pub fn erase_ann(&self, range: JsValue, ann_name: &str) -> Result<(), JsError> { 112 | let range: AnnRange = serde_wasm_bindgen::from_value(range)?; 113 | 114 | if range.end > self.length() { 115 | return Err(JsError::new("index out of range")); 116 | } 117 | 118 | let expand: Expand = range 119 | .expand 120 | .as_deref() 121 | .try_into() 122 | .map_err(|_| JsError::new("invalid expand value"))?; 123 | // We expect user use the expand type of insertion, as it's most intuitive. 124 | // So we need to toggle it to make it work for deletion 125 | let expand = expand.toggle(); 126 | 127 | let style = Style { 128 | expand, 129 | behavior: Behavior::Delete, 130 | type_: ann_name.into(), 131 | value: serde_json::Value::Null, 132 | }; 133 | 134 | self.inner 135 | .borrow_mut() 136 | .annotate_utf16(range.start..range.end, style); 137 | Ok(()) 138 | } 139 | 140 | #[wasm_bindgen(js_name = "getAnnSpans", skip_typescript)] 141 | pub fn get_ann_spans(&self) -> Vec { 142 | let mut ans = Vec::new(); 143 | for span in self.inner.borrow().iter() { 144 | let serializer = serde_wasm_bindgen::Serializer::json_compatible(); 145 | ans.push(span.serialize(&serializer).unwrap()); 146 | } 147 | 148 | ans 149 | } 150 | 151 | #[wasm_bindgen(js_name = "getLine", skip_typescript)] 152 | pub fn get_line(&self, line: usize) -> Vec { 153 | let mut ans = Vec::new(); 154 | for span in self.inner.borrow().get_line(line) { 155 | let serializer = serde_wasm_bindgen::Serializer::json_compatible(); 156 | ans.push(span.serialize(&serializer).unwrap()); 157 | } 158 | 159 | ans 160 | } 161 | 162 | #[wasm_bindgen(js_name = "sliceString")] 163 | pub fn slice_str(&self, start: usize, end: usize) -> String { 164 | self.inner.borrow().slice_str(start..end, IndexType::Utf16) 165 | } 166 | 167 | #[wasm_bindgen(js_name = "chatAt")] 168 | pub fn char_at(&self, index: usize) -> String { 169 | self.inner 170 | .borrow() 171 | .slice_str(index..index + 1, IndexType::Utf16) 172 | } 173 | 174 | pub fn lines(&self) -> usize { 175 | self.inner.borrow().lines() 176 | } 177 | 178 | #[wasm_bindgen(js_name = "applyDelta", skip_typescript)] 179 | pub fn apply_delta(&self, delta: JsValue) -> Result<(), JsError> { 180 | let delta: Vec = serde_wasm_bindgen::from_value(delta)?; 181 | 182 | if delta.is_empty() { 183 | return Ok(()); 184 | } 185 | 186 | self.inner 187 | .borrow_mut() 188 | .apply_delta(delta.into_iter(), IndexType::Utf16); 189 | Ok(()) 190 | } 191 | 192 | pub fn version(&self) -> Vec { 193 | self.inner.borrow().version().encode() 194 | } 195 | 196 | #[wasm_bindgen(js_name = "versionDebugMap")] 197 | pub fn version_map(&self) -> Result { 198 | let serializer = serde_wasm_bindgen::Serializer::json_compatible(); 199 | let v = self 200 | .inner 201 | .borrow() 202 | .version() 203 | .vv 204 | .into_iter() 205 | .map(|(key, value)| (key.to_string(), value)) 206 | .collect::>(); 207 | Ok(v.serialize(&serializer)?) 208 | } 209 | 210 | pub fn export(&self, version: &[u8]) -> Vec { 211 | if version.is_empty() { 212 | self.inner.borrow().export(&Default::default()) 213 | } else { 214 | let vv = crdt_richtext::VersionVector::decode(version); 215 | self.inner.borrow().export(&vv) 216 | } 217 | } 218 | 219 | pub fn import(&self, data: &[u8]) { 220 | self.inner.borrow_mut().import(data); 221 | } 222 | 223 | pub fn length(&self) -> usize { 224 | self.inner.borrow().len_utf16() 225 | } 226 | } 227 | 228 | #[wasm_bindgen(js_name = setPanicHook)] 229 | pub fn set_panic_hook() { 230 | // When the `console_error_panic_hook` feature is enabled, we can call the 231 | // `set_panic_hook` function at least once during initialization, and then 232 | // we will get better error messages if our code ever panics. 233 | // 234 | // For more details see 235 | // https://github.com/rustwasm/console_error_panic_hook#readme 236 | panic::set_hook(Box::new(console_error_panic_hook::hook)); 237 | } 238 | 239 | #[wasm_bindgen(typescript_custom_section)] 240 | const TS_APPEND_CONTENT: &'static str = r#" 241 | export type AnnRange = { 242 | expand?: 'before' | 'after' | 'both' | 'none' 243 | inclusive?: boolean, 244 | start: number, 245 | end: number, 246 | } 247 | 248 | export interface Span { 249 | insert: string, 250 | attributes: Record, 251 | } 252 | 253 | export type DeltaItem = { 254 | retain?: number, 255 | insert?: string, 256 | delete?: number, 257 | attributes?: Record, 258 | }; 259 | 260 | export interface Event { 261 | ops: DeltaItem[], 262 | is_local: boolean, 263 | index_type: "Utf8" | "Utf16", 264 | } 265 | 266 | export interface RichText { 267 | getAnnSpans(): Span[]; 268 | getLine(line: number): Span[]; 269 | annotate( 270 | range: AnnRange, 271 | ann_name: string, 272 | value: null|boolean|number|string|object, 273 | ); 274 | eraseAnn( 275 | range: AnnRange, 276 | ann_name: string, 277 | ); 278 | observe(cb: (event: Event) => void): void; 279 | applyDelta(delta: DeltaItem[]): void; 280 | } 281 | "#; 282 | -------------------------------------------------------------------------------- /crdt-richtext-wasm/src/log.rs: -------------------------------------------------------------------------------- 1 | use wasm_bindgen::prelude::wasm_bindgen; 2 | 3 | #[wasm_bindgen] 4 | extern "C" { 5 | // Use `js_namespace` here to bind `console.log(..)` instead of just 6 | // `log(..)` 7 | #[wasm_bindgen(js_namespace = console)] 8 | pub fn log(s: &str); 9 | } 10 | 11 | #[macro_export] 12 | macro_rules! console_log { 13 | // Note that this is using the `log` function imported above during 14 | // `bare_bones` 15 | ($($t:tt)*) => ($crate::log::log(&format_args!($($t)*).to_string())) 16 | } 17 | -------------------------------------------------------------------------------- /crdt-richtext-wasm/tests/rich_text.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, it } from "vitest"; 2 | import { 3 | AnnotateType, 4 | RichText, 5 | setPanicHook, 6 | } from "../nodejs/crdt_richtext_wasm"; 7 | 8 | setPanicHook(); 9 | describe("basic ops", () => { 10 | it("insert & merge", () => { 11 | const text = new RichText(BigInt(1)); 12 | text.insert(0, "123"); 13 | const b = new RichText(BigInt(2)); 14 | b.import(text.export(new Uint8Array())); 15 | expect(b.toString()).toBe("123"); 16 | }); 17 | 18 | it("bold", () => { 19 | const text = new RichText(BigInt(2)); 20 | text.insert(0, "123"); 21 | text.annotate({ start: 0, end: 1 }, "bold", null); 22 | text.insert(1, "k"); 23 | { 24 | const spans = text.getAnnSpans(); 25 | expect(spans[0].insert).toBe("1k"); 26 | expect(spans[0].attributes).toStrictEqual( 27 | { bold: null }, 28 | ); 29 | } 30 | 31 | text.eraseAnn({ start: 0, end: 2 }, "bold"); 32 | { 33 | const spans = text.getAnnSpans(); 34 | expect(spans[0].insert).toBe("1k23"); 35 | expect(Object.keys(spans[0].attributes).length).toBe(0); 36 | } 37 | }); 38 | }); 39 | 40 | describe("utf16", () => { 41 | it("insert", () => { 42 | const text = new RichText(BigInt(1)); 43 | text.insert(0, "你好，世界！"); 44 | expect(text.chatAt(0)).toBe("你"); 45 | expect(text.chatAt(1)).toBe("好"); 46 | expect(text.sliceString(0, 2)).toBe("你好"); 47 | expect(text.sliceString(3, 5)).toBe("世界"); 48 | text.insert(0, ""); 49 | text.insert(2, "呀"); 50 | expect(text.toString()).toBe("你好呀，世界！"); 51 | text.annotate({ start: 0, end: 3 }, "bold", null); 52 | const spans = text.getAnnSpans(); 53 | expect(spans.length).toBe(2); 54 | expect(spans[0].insert).toBe("你好呀"); 55 | expect(Object.keys(spans[0].attributes).length).toBe(1); 56 | expect("bold" in spans[0].attributes).toBeTruthy(); 57 | expect(spans[1].insert.length).toBe(4); 58 | 59 | expect(() => text.annotate({ start: 0, end: 100 }, "bold", null)).toThrow(); 60 | expect(() => text.annotate({} as any, "bold", null)).toThrow(); 61 | }); 62 | 63 | it("delete", () => { 64 | const text = new RichText(BigInt(1)); 65 | text.insert(0, "你好，世界！"); 66 | text.delete(0, 0); 67 | expect(text.toString()).toBe("你好，世界！"); 68 | text.delete(0, 1); 69 | expect(text.toString()).toBe("好，世界！"); 70 | text.insert(5, "x"); 71 | expect(text.toString()).toBe("好，世界！x"); 72 | }); 73 | }); 74 | 75 | describe("get line", () => { 76 | it("basic", () => { 77 | const text = new RichText(BigInt(1)); 78 | text.insert(0, "你好，\n世界！"); 79 | expect(text.getLine(0)[0].insert).toBe("你好，\n"); 80 | expect(text.getLine(1)[0].insert).toBe("世界！"); 81 | expect(text.getLine(2).length).toBe(0); 82 | expect(text.getLine(3).length).toBe(0); 83 | text.insert(0, "\n"); 84 | expect(text.getLine(0)[0].insert).toBe("\n"); 85 | expect(text.getLine(1)[0].insert).toBe("你好，\n"); 86 | expect(text.getLine(2)[0].insert).toBe("世界！"); 87 | expect(text.getLine(3).length).toBe(0); 88 | expect(text.getLine(4).length).toBe(0); 89 | }); 90 | }); 91 | 92 | describe("Observable", () => { 93 | it("basic", () => { 94 | const text = new RichText(BigInt(1)); 95 | let s = ""; 96 | text.observe((event) => { 97 | let index = 0; 98 | event.ops.forEach((op) => { 99 | if (op.insert != null) { 100 | s = s.slice(0, index) + op.insert + s.slice(index); 101 | index += op.insert.length; 102 | } else if (op.delete != null) { 103 | s = s.slice(0, index) + s.slice(index + op.delete); 104 | } else { 105 | index += op.retain!; 106 | } 107 | }); 108 | }); 109 | 110 | text.insert(0, "xxx"); 111 | const b = new RichText(BigInt(2)); 112 | b.insert(0, "你好，\n世界！"); 113 | b.insert(1, "你"); 114 | b.insert(0, "k"); 115 | text.import(b.export(new Uint8Array())); 116 | text.delete(1, 4); 117 | b.import(text.export(new Uint8Array())); 118 | expect(s).toBe(text.toString()); 119 | b.annotate({ start: 0, end: 5 }, "bold", 1); 120 | b.insert(0, "你好，\n世界！"); 121 | text.import(b.export(new Uint8Array())); 122 | expect(s).toBe(text.toString()); 123 | expect(s).toBe(b.toString()); 124 | }); 125 | }); 126 | -------------------------------------------------------------------------------- /deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "tasks": { 3 | "test": "RUST_BACKTRACE=full cargo nextest run --features test", 4 | "test_log": "RUST_BACKTRACE=full NEXTEST_FAILURE_OUTPUT=immediate cargo nextest run --features test", 5 | "fuzz": "cargo +nightly fuzz run rich-text-utf16", 6 | "fuzz-match": "cargo +nightly fuzz run rich-text-match", 7 | "flame": "CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph --root --example bench -- automerge", 8 | "example": "DEBUG=\"*\" cargo run --example bench -- automerge", 9 | "example-encode": "cargo run --example bench -- encode", 10 | "bench": "cargo bench" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /deno.lock: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2", 3 | "remote": { 4 | "https://deno.land/std@0.105.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58", 5 | "https://deno.land/std@0.105.0/_util/os.ts": "dfb186cc4e968c770ab6cc3288bd65f4871be03b93beecae57d657232ecffcac", 6 | "https://deno.land/std@0.105.0/fmt/colors.ts": "d2f8355f00a74404668fc5a1e4a92983ce1a9b0a6ac1d40efbd681cb8f519586", 7 | "https://deno.land/std@0.105.0/fmt/printf.ts": "7ec612e9b89958b8f7710129f74f502327aad285a9e48ee5297f5882fbc3a078", 8 | "https://deno.land/std@0.105.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853", 9 | "https://deno.land/std@0.105.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4", 10 | "https://deno.land/std@0.105.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b", 11 | "https://deno.land/std@0.105.0/path/common.ts": "eaf03d08b569e8a87e674e4e265e099f237472b6fd135b3cbeae5827035ea14a", 12 | "https://deno.land/std@0.105.0/path/glob.ts": "3b84af55c53febacf6afe214c095624b22a56b6f57d7312157479cc783a0de65", 13 | "https://deno.land/std@0.105.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12", 14 | "https://deno.land/std@0.105.0/path/posix.ts": "b81974c768d298f8dcd2c720229639b3803ca4a241fa9a355c762fa2bc5ef0c1", 15 | "https://deno.land/std@0.105.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c", 16 | "https://deno.land/std@0.105.0/path/win32.ts": "f4a3d4a3f2c9fe894da046d5eac48b5e789a0ebec5152b2c0985efe96a9f7ae1", 17 | "https://deno.land/std@0.105.0/testing/_diff.ts": "5d3693155f561d1a5443ac751ac70aab9f5d67b4819a621d4b96b8a1a1c89620", 18 | "https://deno.land/std@0.105.0/testing/asserts.ts": "e4311d45d956459d4423bc267208fe154b5294989da2ed93257b6a85cae0427e", 19 | "https://deno.land/std@0.106.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58", 20 | "https://deno.land/std@0.106.0/_util/os.ts": "dfb186cc4e968c770ab6cc3288bd65f4871be03b93beecae57d657232ecffcac", 21 | "https://deno.land/std@0.106.0/fs/_util.ts": "f2ce811350236ea8c28450ed822a5f42a0892316515b1cd61321dec13569c56b", 22 | "https://deno.land/std@0.106.0/fs/copy.ts": "631bbafbfe6cba282158abc8aeb7e8251cc69a7ec28ce12878ea1b75fec2add4", 23 | "https://deno.land/std@0.106.0/fs/empty_dir.ts": "5f08b263dd064dc7917c4bbeb13de0f5505a664b9cdfe312fa86e7518cfaeb84", 24 | "https://deno.land/std@0.106.0/fs/ensure_dir.ts": "b7c103dc41a3d1dbbb522bf183c519c37065fdc234831a4a0f7d671b1ed5fea7", 25 | "https://deno.land/std@0.106.0/fs/ensure_file.ts": "c06031af24368e80c330897e4b8e9109efc8602ffabc8f3e2306be07529e1d13", 26 | "https://deno.land/std@0.106.0/fs/ensure_link.ts": "26e54363508b822afd87a3f6e873bbbcd6b5993dd638f8170758c16262a75065", 27 | "https://deno.land/std@0.106.0/fs/ensure_symlink.ts": "c07b6d19ef58b6f5c671ffa942e7f9be50315f4f78e2f9f511626fd2e13beccc", 28 | "https://deno.land/std@0.106.0/fs/eol.ts": "afaebaaac36f48c423b920c836551997715672b80a0fee9aa7667c181a94f2df", 29 | "https://deno.land/std@0.106.0/fs/exists.ts": "b0d2e31654819cc2a8d37df45d6b14686c0cc1d802e9ff09e902a63e98b85a00", 30 | "https://deno.land/std@0.106.0/fs/expand_glob.ts": "73e7b13f01097b04ed782b3d63863379b718417417758ba622e282b1e5300b91", 31 | "https://deno.land/std@0.106.0/fs/mod.ts": "26eee4b52a8c516e37d464094b080ff6822883e7f01ff0ba0a72b8dcd54b9927", 32 | "https://deno.land/std@0.106.0/fs/move.ts": "4623058e39bbbeb3ad30aeff9c974c55d2d574ad7c480295c12b04c244686a99", 33 | "https://deno.land/std@0.106.0/fs/walk.ts": "b91c655c60d048035f9cae0e6177991ab3245e786e3ab7d20a5b60012edf2126", 34 | "https://deno.land/std@0.106.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853", 35 | "https://deno.land/std@0.106.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4", 36 | "https://deno.land/std@0.106.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b", 37 | "https://deno.land/std@0.106.0/path/common.ts": "eaf03d08b569e8a87e674e4e265e099f237472b6fd135b3cbeae5827035ea14a", 38 | "https://deno.land/std@0.106.0/path/glob.ts": "3b84af55c53febacf6afe214c095624b22a56b6f57d7312157479cc783a0de65", 39 | "https://deno.land/std@0.106.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12", 40 | "https://deno.land/std@0.106.0/path/posix.ts": "b81974c768d298f8dcd2c720229639b3803ca4a241fa9a355c762fa2bc5ef0c1", 41 | "https://deno.land/std@0.106.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c", 42 | "https://deno.land/std@0.106.0/path/win32.ts": "f4a3d4a3f2c9fe894da046d5eac48b5e789a0ebec5152b2c0985efe96a9f7ae1", 43 | "https://deno.land/x/cliui@v7.0.4-deno/build/lib/index.d.ts": "4f04923352ce24027ad6fe25c85249e0b0cc5c29fe18d62024d004c59d9e41ee", 44 | "https://deno.land/x/cliui@v7.0.4-deno/build/lib/index.js": "fb6030c7b12602a4fca4d81de3ddafa301ba84fd9df73c53de6f3bdda7b482d5", 45 | "https://deno.land/x/cliui@v7.0.4-deno/build/lib/string-utils.js": "b3eb9d2e054a43a3064af17332fb1839a7dadb205c5371af4789616afb1a117f", 46 | "https://deno.land/x/cliui@v7.0.4-deno/deno.ts": "d07bc3338661f8011e3a5fd215061d17a52107a5383c29f40ce0c1ecb8bb8cc3", 47 | "https://deno.land/x/escalade@v3.0.3/sync.ts": "493bc66563292c5c10c4a75a467a5933f24dad67d74b0f5a87e7b988fe97c104", 48 | "https://deno.land/x/y18n@v5.0.0-deno/build/lib/index.d.ts": "11f40d97041eb271cc1a1c7b296c6e7a068d4843759575e7416f0d14ebf8239c", 49 | "https://deno.land/x/y18n@v5.0.0-deno/build/lib/index.js": "92c4624714aa508d33c6d21c0b0ffa072369a8b306e5f8c7727662f570bbd026", 50 | "https://deno.land/x/y18n@v5.0.0-deno/deno.ts": "80997f0709a0b43d29931e2b33946f2bbc32b13fd82f80a5409628455427e28d", 51 | "https://deno.land/x/y18n@v5.0.0-deno/lib/platform-shims/deno.ts": "8fa2c96ac03734966260cfd2c5bc240e41725c913e5b64a0297aede09f52b39d", 52 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/argsert.js": "eb085555452eac3ff300935994a42f35d16e04cf698cb775cb5ad4f5653c0627", 53 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/command.js": "499c95cecd5e93f627e0b5ce66a193c9a595adc10fbafe0581a9725e38324dee", 54 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/completion-templates.js": "f84823b1daa0ed0189e4f823f6a4fd29ad58de6a05771004918368fd62bb2b3f", 55 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/completion.js": "c91772b89907ebf1a462804305d12d3b9deade75cd1b319c06831ac0bf5abd27", 56 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/middleware.js": "cef3f017d5ff61c340c65b8422f5ab9600ba381aa656df634d1a3edf0f967527", 57 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/parse-command.js": "327242c0afae207b7aefa13133439e3b321d7db4229febc5b7bd5285770ac7f7", 58 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/typings/common-types.js": "9618b81a86acb88a61fd9988e9bc3ec21c5250d94fc2231ba7d898e71500789d", 59 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/usage.js": "61071feb99ac220f1b27036406ae8e4f9ee606b373a5f3bcb60042c7bcfbd0d8", 60 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/apply-extends.js": "64640dce92669705abead3bdbe2c46c8318c8623843a55e4726fb3c55ff9dd1d", 61 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/is-promise.js": "be45baa3090c5106dd4e442cceef6b357a268783a2ee28ec10fe131a8cd8db72", 62 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/levenshtein.js": "d8638efc3376b5f794b1c8df6ef4f3d484b29d919127c7fdc242400e3cfded91", 63 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/maybe-async-result.js": "31cf4026279e14c87d16faa14ac758f35c8cc5795d29393c5ce07120f5a3caf6", 64 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/obj-filter.js": "5523fb2288d1e86ed48c460e176770b49587554df4ae2405b468c093786b040b", 65 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/set-blocking.js": "6fa8ffc3299f456e42902736bae35fbc1f2dc96b3905a02ba9629f5bd9f80af1", 66 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/which-module.js": "9267633b2c9f8990b2c699101b641e59ae59932e0dee5270613c0508bfa13c5d", 67 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/validation.js": "3dc366de2eb23bc9457ed3e120b69db9d801251bef3dc19f93e4c0380ac0198c", 68 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/yargs-factory.js": "a3e629d7d063b5ac007b18a0d8e9ad2ca72ca4d702c5c46822fbbdfdd6c512df", 69 | "https://deno.land/x/yargs@v17.1.1-deno/build/lib/yerror.js": "1d9dead374fe06c8f13f2e4adafc002b8a15682b7185abf29638f1be96fd9dfc", 70 | "https://deno.land/x/yargs@v17.1.1-deno/deno-types.ts": "62f5c61899c6da491890c8c84fd9580cfbfa2a83f5a70f6dc74727bbfb148623", 71 | "https://deno.land/x/yargs@v17.1.1-deno/deno.ts": "f3df0bfd08ba367ec36dc59ef6cab1a391ace49ad44387ec5fe5d76289af08af", 72 | "https://deno.land/x/yargs@v17.1.1-deno/lib/platform-shims/deno.ts": "b5a48b40d5c64fe66f5a77f87ebaf4413eea828ccd8159feeac370b3eef9a356", 73 | "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/string-utils.js": "12fc056b23703bc370aae5b179dc5abee53fca277abc30eaf76f78d2546d6413", 74 | "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/tokenize-arg-string.js": "7e0875b11795b8e217386e45f14b24a6e501ebbc62e15aa469aa8829d4d0ee61", 75 | "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/yargs-parser-types.d.ts": "434deb76c6632b3b6cbc4c6f153f8aca04e06055ae9c6b24b40218cbc42688d9", 76 | "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/yargs-parser.js": "453200a7dfbb002e605d8009b7dad30f2b1d93665e046ab89c073a4fe63dfd48", 77 | "https://deno.land/x/yargs_parser@v20.2.4-deno/deno.ts": "ad53c0c82c3982c4fc5be9472384b259e0a32ce1f7ae0f68de7b2445df5642fc" 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /examples/bench.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use arbitrary::{Arbitrary, Unstructured}; 4 | use crdt_richtext::rich_text::RichText; 5 | use generic_btree::HeapVec; 6 | use rand::{Rng, SeedableRng}; 7 | 8 | #[derive(Arbitrary, Debug, Clone, Copy)] 9 | enum RandomAction { 10 | Insert { pos: u8, content: u8 }, 11 | Delete { pos: u8, len: u8 }, 12 | } 13 | 14 | use std::io::Read; 15 | 16 | use flate2::read::GzDecoder; 17 | use serde_json::Value; 18 | 19 | #[derive(Arbitrary)] 20 | pub struct TextAction { 21 | pub pos: usize, 22 | pub ins: String, 23 | pub del: usize, 24 | } 25 | 26 | pub fn get_automerge_actions() -> Vec { 27 | const RAW_DATA: &[u8; 901823] = include_bytes!("../benches/automerge-paper.json.gz"); 28 | let mut actions = Vec::new(); 29 | let mut d = GzDecoder::new(&RAW_DATA[..]); 30 | let mut s = String::new(); 31 | d.read_to_string(&mut s).unwrap(); 32 | let json: Value = serde_json::from_str(&s).unwrap(); 33 | let txns = json.as_object().unwrap().get("txns"); 34 | for txn in txns.unwrap().as_array().unwrap() { 35 | let patches = txn 36 | .as_object() 37 | .unwrap() 38 | .get("patches") 39 | .unwrap() 40 | .as_array() 41 | .unwrap(); 42 | for patch in patches { 43 | let pos = patch[0].as_u64().unwrap() as usize; 44 | let del_here = patch[1].as_u64().unwrap() as usize; 45 | let ins_content = patch[2].as_str().unwrap(); 46 | actions.push(TextAction { 47 | pos, 48 | ins: ins_content.to_string(), 49 | del: del_here, 50 | }); 51 | } 52 | } 53 | actions 54 | } 55 | 56 | pub fn main() { 57 | let args: Vec = env::args().collect(); 58 | if args.len() > 1 && args[1].eq_ignore_ascii_case("automerge") { 59 | println!("Running on automerge dataset"); 60 | let actions = get_automerge_actions(); 61 | bench(actions); 62 | } else if args.len() > 1 && args[1].eq_ignore_ascii_case("encode") { 63 | println!("Running on automerge dataset"); 64 | let actions = get_automerge_actions(); 65 | let mut text = RichText::new(1); 66 | for action in actions.iter() { 67 | if action.del > 0 { 68 | text.delete(action.pos..action.pos + action.del); 69 | } 70 | if !action.ins.is_empty() { 71 | text.insert(action.pos, &action.ins) 72 | } 73 | } 74 | let data = text.export(&Default::default()); 75 | println!("Size = {}", data.len()); 76 | } else { 77 | println!("Running on random generated actions 10k"); 78 | let mut rng = rand::rngs::StdRng::seed_from_u64(123); 79 | let data: HeapVec = (0..1_000_000).map(|_| rng.gen()).collect(); 80 | let mut gen = Unstructured::new(&data); 81 | let actions: [RandomAction; 10_000] = gen.arbitrary().unwrap(); 82 | 83 | let mut rope = RichText::new(1); 84 | for _ in 0..10000 { 85 | for action in actions.iter() { 86 | match *action { 87 | RandomAction::Insert { pos, content } => { 88 | let pos = pos as usize % (rope.len() + 1); 89 | let s = content.to_string(); 90 | rope.insert(pos, &s); 91 | } 92 | RandomAction::Delete { pos, len } => { 93 | let pos = pos as usize % (rope.len() + 1); 94 | let mut len = len as usize % 10; 95 | len = len.min(rope.len() - pos); 96 | rope.delete(pos..(pos + len)); 97 | } 98 | } 99 | } 100 | } 101 | } 102 | } 103 | 104 | #[inline(never)] 105 | fn bench(actions: Vec) { 106 | // #[global_allocator] 107 | // static ALLOC: dhat::Alloc = dhat::Alloc; 108 | for _ in 0..30 { 109 | let mut text = RichText::new(1); 110 | // let profiler = dhat::Profiler::builder().trim_backtraces(None).build(); 111 | for action in actions.iter() { 112 | if action.del > 0 { 113 | text.delete(action.pos..action.pos + action.del); 114 | } 115 | if !action.ins.is_empty() { 116 | text.insert(action.pos, &action.ins) 117 | } 118 | } 119 | // drop(profiler); 120 | text.debug_log(false) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | corpus 3 | artifacts 4 | -------------------------------------------------------------------------------- /fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "peritext-rs-fuzz" 3 | version = "0.0.0" 4 | authors = ["Automatically generated"] 5 | publish = false 6 | edition = "2018" 7 | 8 | [package.metadata] 9 | cargo-fuzz = true 10 | 11 | [dependencies] 12 | libfuzzer-sys = "0.4" 13 | 14 | [dependencies.crdt-richtext] 15 | path = ".." 16 | features = ["test"] 17 | 18 | # Prevent this from interfering with workspaces 19 | [workspace] 20 | members = ["."] 21 | 22 | [[bin]] 23 | name = "basic" 24 | path = "fuzz_targets/basic.rs" 25 | test = false 26 | doc = false 27 | 28 | [[bin]] 29 | name = "five-actors" 30 | path = "fuzz_targets/five-actors.rs" 31 | test = false 32 | doc = false 33 | 34 | [[bin]] 35 | name = "rich-text-apply" 36 | path = "fuzz_targets/rich-text-apply.rs" 37 | test = false 38 | doc = false 39 | 40 | [[bin]] 41 | name = "rich-text" 42 | path = "fuzz_targets/rich-text.rs" 43 | test = false 44 | doc = false 45 | 46 | [[bin]] 47 | name = "rich-text-utf16" 48 | path = "fuzz_targets/rich-text-utf16.rs" 49 | test = false 50 | doc = false 51 | 52 | [[bin]] 53 | name = "rich-text-match" 54 | path = "fuzz_targets/rich-text-match.rs" 55 | test = false 56 | doc = false 57 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/basic.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | use crdt_richtext::legacy::test_utils::{fuzzing, Action}; 3 | use libfuzzer_sys::fuzz_target; 4 | 5 | fuzz_target!(|actions: Vec| { fuzzing(2, actions) }); 6 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/five-actors.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | use crdt_richtext::legacy::test_utils::{fuzzing, Action}; 3 | use libfuzzer_sys::fuzz_target; 4 | 5 | fuzz_target!(|actions: [Action; 100]| { fuzzing(5, actions.to_vec()) }); 6 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/rich-text-apply.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | use crdt_richtext::rich_text::test_utils::{fuzzing, Action}; 3 | use libfuzzer_sys::fuzz_target; 4 | 5 | fuzz_target!(|actions: Vec| { fuzzing(2, actions) }); 6 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/rich-text-match.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | use crdt_richtext::rich_text::test_utils::{fuzzing_line_break, LineBreakFuzzAction}; 3 | use libfuzzer_sys::fuzz_target; 4 | 5 | fuzz_target!(|actions: Vec| { fuzzing_line_break(actions) }); 6 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/rich-text-utf16.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | use crdt_richtext::rich_text::test_utils::{fuzzing_utf16, Action}; 3 | use libfuzzer_sys::fuzz_target; 4 | 5 | fuzz_target!(|actions: Vec| { fuzzing_utf16(5, actions) }); 6 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/rich-text.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | use crdt_richtext::rich_text::test_utils::{fuzzing, Action}; 3 | use libfuzzer_sys::fuzz_target; 4 | 5 | fuzz_target!(|actions: Vec| { fuzzing(5, actions) }); 6 | -------------------------------------------------------------------------------- /js/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | -------------------------------------------------------------------------------- /js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "js", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "devDependencies": { 13 | "vite": "^4.3.6", 14 | "vitest": "^0.31.0" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /js/packages/cr-quill/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /js/packages/cr-quill/.vite/deps_temp_cd4bf69d/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "module" 3 | } 4 | -------------------------------------------------------------------------------- /js/packages/cr-quill/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": ["Vue.volar", "Vue.vscode-typescript-vue-plugin"] 3 | } 4 | -------------------------------------------------------------------------------- /js/packages/cr-quill/README.md: -------------------------------------------------------------------------------- 1 | # Vue 3 + TypeScript + Vite 2 | 3 | This template should help get you started developing with Vue 3 and TypeScript in Vite. The template uses Vue 3 ` 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /js/packages/cr-quill/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cr-quill", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite --force", 8 | "build": "vue-tsc && vite build", 9 | "preview": "vite preview" 10 | }, 11 | "dependencies": { 12 | "crdt-richtext-wasm": "0.4.0", 13 | "is-equal": "^1.6.4", 14 | "quill": "^1.3.7", 15 | "vue": "^3.2.47" 16 | }, 17 | "devDependencies": { 18 | "@types/quill": "^1.3.7", 19 | "@vitejs/plugin-vue": "^4.1.0", 20 | "typescript": "^5.0.2", 21 | "vite": "^4.3.2", 22 | "vite-plugin-top-level-await": "^1.3.0", 23 | "vite-plugin-wasm": "^3.2.2", 24 | "vue-tsc": "^1.4.2" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /js/packages/cr-quill/public/Loro.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /js/packages/cr-quill/src/App.vue: -------------------------------------------------------------------------------- 1 | 87 | 88 | 151 | 152 | 195 | -------------------------------------------------------------------------------- /js/packages/cr-quill/src/assets/Loro.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /js/packages/cr-quill/src/binding.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The skeleton of this binding is learned from https://github.com/yjs/y-quill 3 | */ 4 | 5 | import { DeltaItem, RichText, Span } from "crdt-richtext-wasm"; 6 | import Quill, { DeltaStatic, Sources } from "quill"; 7 | // @ts-ignore 8 | import isEqual from "is-equal"; 9 | 10 | const Delta = Quill.import("delta"); 11 | 12 | export class QuillBinding { 13 | constructor(public richtext: RichText, public quill: Quill) { 14 | this.quill = quill; 15 | richtext.observe((event) => { 16 | // Promise.resolve().then(() => { 17 | // let delta: DeltaType = new Delta( 18 | // richtext.getAnnSpans(), 19 | // ); 20 | // quill.setContents( 21 | // delta, 22 | // "this" as any, 23 | // ); 24 | // }); 25 | Promise.resolve().then(() => { 26 | if (!event.is_local) { 27 | console.log( 28 | richtext.id(), 29 | "CRDT_EVENT", 30 | event, 31 | ); 32 | const eventDelta = event.ops; 33 | const delta = []; 34 | let index = 0; 35 | for (let i = 0; i < eventDelta.length; i++) { 36 | const d = eventDelta[i]; 37 | const length = d.delete || d.retain || d.insert!.length; 38 | // skip the last newline that quill automatically appends 39 | if ( 40 | d.insert && d.insert === "\n" && 41 | index === quill.getLength() - 1 && 42 | i === eventDelta.length - 1 && d.attributes != null && 43 | Object.keys(d.attributes).length > 0 44 | ) { 45 | delta.push({ 46 | retain: 1, 47 | attributes: d.attributes, 48 | }); 49 | index += length; 50 | continue; 51 | } 52 | 53 | delta.push(d); 54 | index += length; 55 | } 56 | quill.updateContents(new Delta(delta), "this" as any); 57 | const a = this.richtext.getAnnSpans(); 58 | const b = this.quill.getContents().ops; 59 | console.log(this.richtext.id(), "COMPARE AFTER CRDT_EVENT"); 60 | if (!assertEqual(a, b as Span[])) { 61 | quill.setContents(new Delta(a), "this" as any); 62 | } 63 | } 64 | }); 65 | }); 66 | quill.setContents( 67 | new Delta( 68 | richtext.getAnnSpans().map((x) => ({ 69 | insert: x.insert, 70 | attributions: x.attributes, 71 | })), 72 | ), 73 | "this" as any, 74 | ); 75 | quill.on("editor-change", this.quillObserver); 76 | } 77 | 78 | quillObserver: ( 79 | name: "text-change", 80 | delta: DeltaStatic, 81 | oldContents: DeltaStatic, 82 | source: Sources, 83 | ) => any = (_eventType, delta, _state, origin) => { 84 | if (delta && delta.ops) { 85 | // update content 86 | const ops = delta.ops; 87 | if (origin !== "this" as any) { 88 | this.richtext.applyDelta(ops); 89 | const a = this.richtext.getAnnSpans(); 90 | const b = this.quill.getContents().ops; 91 | console.log(this.richtext.id(), "COMPARE AFTER QUILL_EVENT"); 92 | assertEqual(a, b as Span[]); 93 | console.log( 94 | this.richtext.id(), 95 | "CHECK_MATCH", 96 | { delta }, 97 | a, 98 | b, 99 | ); 100 | console.log("SIZE", this.richtext.export(new Uint8Array()).length); 101 | } 102 | } 103 | }; 104 | destroy() { 105 | // TODO: unobserve 106 | this.quill.off("editor-change", this.quillObserver); 107 | } 108 | } 109 | 110 | function assertEqual(a: DeltaItem[], b: DeltaItem[]): boolean { 111 | a = normQuillDelta(a); 112 | b = normQuillDelta(b); 113 | const equal = isEqual(a, b); 114 | console.assert(equal, a, b); 115 | return equal; 116 | } 117 | 118 | /** 119 | * Removes the pending '\n's if it has no attributes. 120 | * 121 | * Normalize attributes field 122 | */ 123 | export const normQuillDelta = (delta: DeltaItem[]) => { 124 | for (const d of delta) { 125 | if (Object.keys(d.attributes || {}).length === 0) { 126 | delete d.attributes; 127 | } 128 | } 129 | 130 | if (delta.length > 0) { 131 | const d = delta[delta.length - 1]; 132 | const insert = d.insert; 133 | if ( 134 | d.attributes === undefined && insert !== undefined && 135 | insert.slice(-1) === "\n" 136 | ) { 137 | delta = delta.slice(); 138 | let ins = insert.slice(0, -1); 139 | while (ins.slice(-1) === "\n") { 140 | ins = ins.slice(0, -1); 141 | } 142 | delta[delta.length - 1] = { insert: ins }; 143 | if (ins.length === 0) { 144 | delta.pop(); 145 | } 146 | return delta; 147 | } 148 | } 149 | return delta; 150 | }; 151 | -------------------------------------------------------------------------------- /js/packages/cr-quill/src/components/HelloWorld.vue: -------------------------------------------------------------------------------- 1 | 8 | 9 | 33 | 34 | 39 | -------------------------------------------------------------------------------- /js/packages/cr-quill/src/main.ts: -------------------------------------------------------------------------------- 1 | import { createApp } from 'vue' 2 | import './style.css' 3 | import App from './App.vue' 4 | 5 | createApp(App).mount('#app') 6 | -------------------------------------------------------------------------------- /js/packages/cr-quill/src/style.css: -------------------------------------------------------------------------------- 1 | :root { 2 | font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif; 3 | line-height: 1.5; 4 | font-weight: 400; 5 | 6 | color-scheme: light dark; 7 | color: rgba(255, 255, 255, 0.87); 8 | background-color: #ddd; 9 | 10 | font-synthesis: none; 11 | text-rendering: optimizeLegibility; 12 | -webkit-font-smoothing: antialiased; 13 | -moz-osx-font-smoothing: grayscale; 14 | -webkit-text-size-adjust: 100%; 15 | } 16 | 17 | a { 18 | font-weight: 500; 19 | color: #646cff; 20 | text-decoration: inherit; 21 | } 22 | a:hover { 23 | color: #535bf2; 24 | } 25 | 26 | body { 27 | margin: 0; 28 | display: flex; 29 | place-items: center; 30 | min-width: 320px; 31 | min-height: 100vh; 32 | } 33 | 34 | h1 { 35 | font-size: 3.2em; 36 | line-height: 1.1; 37 | } 38 | 39 | button { 40 | border-radius: 8px; 41 | border: 1px solid transparent; 42 | padding: 0.6em 1.2em; 43 | font-size: 1em; 44 | font-weight: 500; 45 | font-family: inherit; 46 | background-color: #1a1a1a; 47 | cursor: pointer; 48 | transition: border-color 0.25s; 49 | } 50 | button:hover { 51 | border-color: #646cff; 52 | } 53 | button:focus, 54 | button:focus-visible { 55 | outline: 4px auto -webkit-focus-ring-color; 56 | } 57 | 58 | .card { 59 | padding: 2em; 60 | } 61 | 62 | #app { 63 | max-width: 1280px; 64 | margin: 0 auto; 65 | padding: 2rem; 66 | text-align: center; 67 | } 68 | 69 | :root { 70 | color: #213547; 71 | background-color: #ffffff; 72 | } 73 | a:hover { 74 | color: #747bff; 75 | } 76 | button { 77 | background-color: #f9f9f9; 78 | } 79 | -------------------------------------------------------------------------------- /js/packages/cr-quill/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /js/packages/cr-quill/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "useDefineForClassFields": true, 5 | "module": "ESNext", 6 | "lib": ["ES2020", "DOM", "DOM.Iterable"], 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "resolveJsonModule": true, 13 | "isolatedModules": true, 14 | "noEmit": true, 15 | "jsx": "preserve", 16 | 17 | /* Linting */ 18 | "strict": true, 19 | "noUnusedLocals": true, 20 | "noUnusedParameters": true, 21 | "noFallthroughCasesInSwitch": true 22 | }, 23 | "include": ["src/**/*.ts", "src/**/*.d.ts", "src/**/*.tsx", "src/**/*.vue"], 24 | "references": [{ "path": "./tsconfig.node.json" }] 25 | } 26 | -------------------------------------------------------------------------------- /js/packages/cr-quill/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | "skipLibCheck": true, 5 | "module": "ESNext", 6 | "moduleResolution": "bundler", 7 | "allowSyntheticDefaultImports": true 8 | }, 9 | "include": ["vite.config.ts"] 10 | } 11 | -------------------------------------------------------------------------------- /js/packages/cr-quill/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vite"; 2 | import vue from "@vitejs/plugin-vue"; 3 | import wasm from "vite-plugin-wasm"; 4 | import topLevelAwait from "vite-plugin-top-level-await"; 5 | 6 | // https://vitejs.dev/config/ 7 | export default defineConfig({ 8 | plugins: [vue(), wasm(), topLevelAwait()], 9 | }); 10 | -------------------------------------------------------------------------------- /js/pnpm-workspace.yaml: -------------------------------------------------------------------------------- 1 | packages: 2 | - packages/* 3 | -------------------------------------------------------------------------------- /src/legacy/README.md: -------------------------------------------------------------------------------- 1 | # This module contains a failed idea of improving Peritext 2 | 3 | [Peritext is not agnostic to the underlying plain text CRDT](https://github.com/inkandswitch/peritext/issues/31). 4 | I thought it's possible to build a useful lib that may add Peritext ability to a 5 | existing list CRDT lib without changing its behaviors. But it turns out to be 6 | too complicated to use and lack of the property my library need. The code is 7 | also included in the crdt-richtext under the legacy module. 8 | 9 | The initial motivation was to create a standalone module, decoupled from the 10 | underlying list CRDT algorithm. This was successfully implemented, but the final 11 | version was highly complex, posing significant integration challenges. 12 | 13 | Furthermore, it lacked a crucial attribute I initially hoped it would possess: 14 | the ability to compute version changes based purely on the operation sequence, 15 | independent of the original state. 16 | 17 | Ultimately, the additional overhead associated with this decoupled approach led 18 | me to abandon this idea. The method required synchronization of many basic 19 | operations on both sides, often involving similar calculations. This decoupled 20 | approach didn't allow for simultaneous resolution of repeated calculations. For 21 | instance, text insertion required updates to both the original text CRDT 22 | document and the range CRDT length mapping. 23 | 24 | Therefore, the value of this method became limited. Hence, it might be more 25 | beneficial to develop a comprehensive rich-text from scratch, which we can 26 | integrate to Loro CRDT more easily. 27 | -------------------------------------------------------------------------------- /src/legacy/mod.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | pub use range_map::tree_impl::TreeRangeMap; 3 | pub use range_map::RangeMap; 4 | mod range_map; 5 | #[cfg(feature = "test")] 6 | pub mod test_utils; 7 | use range_map::{AnnPosRelativeToInsert, Span}; 8 | 9 | #[derive(Debug)] 10 | pub struct CrdtRange { 11 | pub(crate) range_map: R, 12 | } 13 | 14 | impl CrdtRange { 15 | pub fn new() -> Self { 16 | let mut r = R::init(); 17 | r.insert_directly(0, 2); 18 | CrdtRange { range_map: r } 19 | } 20 | 21 | /// Insert a new span of text into the range. It's used to sync 22 | /// List Crdt insert ops. 23 | /// 24 | /// It will only generate new RangeOp(Patches) when inserting new 25 | /// text locally and there are annotations attached to the tombstones 26 | /// at `pos`. 27 | /// 28 | /// - `cmp(target)` returns whether the target is in right side or 29 | /// left side of the new inserted op. `target` may be any op id 30 | /// from the List CRDT because it's used to test both sides of an 31 | /// annotation 32 | pub fn insert_text( 33 | &mut self, 34 | pos: usize, 35 | len: usize, 36 | is_local: bool, 37 | left_id: Option, 38 | right_id: Option, 39 | next_lamport: Lamport, 40 | next_op_id: OpID, 41 | mut cmp: Cmp, 42 | ) -> Vec 43 | where 44 | Cmp: FnMut(OpID) -> Ordering, 45 | { 46 | let mut ans = vec![]; 47 | // Maybe add the zero-len filter rule as a requirement for the range_map? 48 | let spans = self.get_trimmed_spans_around(pos); 49 | assert!(spans.len() <= 3, "{}", spans.len()); 50 | assert!(spans.iter().map(|x| x.len).sum::() == 2); 51 | let non_empty_span_count = spans.iter().filter(|x| x.len != 0).count(); 52 | if is_local && non_empty_span_count > 1 { 53 | self.gen_patch( 54 | non_empty_span_count, 55 | spans, 56 | left_id, 57 | right_id, 58 | next_lamport, 59 | next_op_id, 60 | &mut ans, 61 | ); 62 | } 63 | 64 | self.range_map.insert(pos * 3 + 1, len * 3, |ann| { 65 | // dbg!(&tombstones, first_new_op_id, ann, relative); 66 | let start_before_insert = match ann.range.start.id { 67 | Some(id) => cmp(id) == Ordering::Less, 68 | None => true, 69 | }; 70 | let end_after_insert = match ann.range.end.id { 71 | Some(id) => cmp(id) == Ordering::Greater, 72 | None => true, 73 | }; 74 | match (start_before_insert, end_after_insert) { 75 | (true, true) => AnnPosRelativeToInsert::IncludeInsert, 76 | (true, false) => AnnPosRelativeToInsert::Before, 77 | (false, true) => AnnPosRelativeToInsert::After, 78 | (false, false) => unreachable!(), 79 | } 80 | }); 81 | 82 | ans 83 | } 84 | 85 | fn get_trimmed_spans_around(&mut self, pos: usize) -> Vec { 86 | let mut spans: Vec = self 87 | .range_map 88 | .get_annotations(pos * 3, 2) 89 | .into_iter() 90 | .skip_while(|x| x.len == 0) 91 | .collect(); 92 | for i in (0..spans.len()).rev() { 93 | if spans[i].len != 0 { 94 | spans.drain(i + 1..); 95 | break; 96 | } 97 | } 98 | spans 99 | } 100 | 101 | /// NOTE: This is error-prone, need more attention 102 | fn gen_patch( 103 | &mut self, 104 | non_empty_span_count: usize, 105 | spans: Vec, 106 | left_id: Option, 107 | right_id: Option, 108 | mut next_lamport: Lamport, 109 | mut next_op_id: OpID, 110 | ans: &mut Vec, 111 | ) { 112 | assert!(non_empty_span_count <= 2); 113 | let mut visited_left = false; 114 | let mut pure_left = BTreeSet::new(); 115 | let mut pure_middle = BTreeSet::new(); 116 | let mut left_annotations = BTreeSet::new(); 117 | let mut right_annotations = BTreeSet::new(); 118 | for span in spans { 119 | if !visited_left { 120 | // left 121 | assert_eq!(span.len, 1); 122 | visited_left = true; 123 | pure_left = span.annotations.clone(); 124 | left_annotations = span.annotations; 125 | } else if span.len == 0 { 126 | // middle 127 | pure_middle = span.annotations; 128 | } else { 129 | // right 130 | assert_eq!(span.len, 1); 131 | for ann in span.annotations.iter() { 132 | right_annotations.insert(ann.clone()); 133 | left_annotations.remove(ann); 134 | pure_middle.remove(ann); 135 | } 136 | } 137 | } 138 | 139 | for ann in pure_left { 140 | right_annotations.remove(&ann); 141 | pure_middle.remove(&ann); 142 | } 143 | 144 | for annotation in left_annotations { 145 | let end_id = annotation.range.end.id; 146 | if end_id != left_id && end_id != right_id { 147 | // TODO: simplify 148 | if AnchorType::Before == annotation.range.end.type_ { 149 | ans.push(RangeOp::Patch(Patch { 150 | id: next_op_id, 151 | lamport: next_lamport, 152 | target_range_id: annotation.id, 153 | move_start_to: annotation.range.start.id, 154 | move_end_to: right_id, 155 | })); 156 | self.range_map.adjust_annotation( 157 | annotation.id, 158 | next_lamport, 159 | next_op_id, 160 | None, 161 | Some((1, right_id)), 162 | ); 163 | next_op_id.counter += 1; 164 | next_lamport += 1; 165 | } else if !pure_middle.contains(&annotation) { 166 | ans.push(RangeOp::Patch(Patch { 167 | id: next_op_id, 168 | lamport: next_lamport, 169 | target_range_id: annotation.id, 170 | move_start_to: annotation.range.start.id, 171 | move_end_to: left_id, 172 | })); 173 | self.range_map.adjust_annotation( 174 | annotation.id, 175 | next_lamport, 176 | next_op_id, 177 | None, 178 | Some((-1, left_id)), 179 | ); 180 | next_op_id.counter += 1; 181 | next_lamport += 1; 182 | } 183 | } 184 | } 185 | 186 | for annotation in right_annotations { 187 | let start_id = annotation.range.start.id; 188 | if start_id != left_id && start_id != right_id { 189 | match annotation.range.start.type_ { 190 | AnchorType::Before => { 191 | if !pure_middle.contains(&annotation) { 192 | ans.push(RangeOp::Patch(Patch { 193 | id: next_op_id, 194 | lamport: next_lamport, 195 | target_range_id: annotation.id, 196 | move_start_to: right_id, 197 | move_end_to: annotation.range.end.id, 198 | })); 199 | self.range_map.adjust_annotation( 200 | annotation.id, 201 | next_lamport, 202 | next_op_id, 203 | Some((1, right_id)), 204 | None, 205 | ); 206 | next_op_id.counter += 1; 207 | next_lamport += 1; 208 | } 209 | } 210 | AnchorType::After => { 211 | ans.push(RangeOp::Patch(Patch { 212 | id: next_op_id, 213 | lamport: next_lamport, 214 | target_range_id: annotation.id, 215 | move_start_to: right_id, 216 | move_end_to: annotation.range.end.id, 217 | })); 218 | self.range_map.adjust_annotation( 219 | annotation.id, 220 | next_lamport, 221 | next_op_id, 222 | Some((-1, left_id)), 223 | None, 224 | ); 225 | next_op_id.counter += 1; 226 | next_lamport += 1; 227 | } 228 | } 229 | } 230 | } 231 | } 232 | 233 | /// NOTE: This is error-prone, need more attention 234 | fn apply_remote_patch(&mut self, patch: Patch, index: &Index) 235 | where 236 | Index: Fn(OpID) -> Result, 237 | { 238 | let Some((ann, pos)) = self.range_map.get_annotation_pos(patch.target_range_id) else { return }; 239 | let new_start = index_start( 240 | Anchor { 241 | id: patch.move_start_to, 242 | type_: ann.range.start.type_, 243 | }, 244 | index, 245 | ); 246 | let new_end = index_end( 247 | Anchor { 248 | id: patch.move_end_to, 249 | type_: ann.range.end.type_, 250 | }, 251 | index, 252 | ) 253 | .unwrap_or(self.range_map.len()); 254 | 255 | self.range_map.adjust_annotation( 256 | patch.target_range_id, 257 | patch.lamport, 258 | patch.id, 259 | Some((new_start as isize - pos.start as isize, patch.move_start_to)), 260 | Some((new_end as isize - pos.end as isize, patch.move_end_to)), 261 | ); 262 | } 263 | 264 | pub fn delete_text(&mut self, pos: usize, len: usize) { 265 | self.range_map.delete(pos * 3 + 1, len * 3); 266 | } 267 | 268 | pub fn annotate(&mut self, annotation: Annotation, range: impl RangeBounds) -> RangeOp { 269 | let start = match range.start_bound() { 270 | Bound::Included(x) => *x * 3 + 2, 271 | Bound::Excluded(x) => *x * 3 + 3, 272 | Bound::Unbounded => 0, 273 | }; 274 | assert!(annotation.range.start.type_ != AnchorType::After); 275 | assert!(annotation.range.start.id.is_some()); 276 | let end = match range.end_bound() { 277 | Bound::Included(x) => *x * 3 + 3, 278 | Bound::Excluded(x) => *x * 3 + 2, 279 | Bound::Unbounded => self.range_map.len(), 280 | }; 281 | self.range_map 282 | .annotate(start, end - start, annotation.clone()); 283 | RangeOp::Annotate(annotation) 284 | } 285 | 286 | pub fn delete_annotation(&mut self, lamport: Lamport, op_id: OpID, target_id: OpID) -> RangeOp { 287 | self.range_map.delete_annotation(target_id); 288 | RangeOp::Patch(Patch { 289 | id: op_id, 290 | target_range_id: target_id, 291 | move_start_to: None, 292 | move_end_to: None, 293 | lamport, 294 | }) 295 | } 296 | 297 | pub fn apply_remote_op(&mut self, op: RangeOp, index: &Index) 298 | where 299 | Index: Fn(OpID) -> Result, 300 | { 301 | match op { 302 | RangeOp::Patch(patch) => { 303 | self.apply_remote_patch(patch, index); 304 | } 305 | RangeOp::Annotate(a) => { 306 | let start = index_start(a.range.start, index); 307 | let end = index_end(a.range.end, index).unwrap_or(self.range_map.len()); 308 | self.range_map.annotate(start, end - start, a) 309 | } 310 | } 311 | } 312 | 313 | pub fn get_annotation_range(&mut self, id: OpID) -> Option> { 314 | let (_, range) = self.range_map.get_annotation_pos(id)?; 315 | Some((range.start / 3)..(range.end / 3)) 316 | } 317 | 318 | pub fn get_annotations(&mut self, range: impl RangeBounds) -> Vec { 319 | let start = match range.start_bound() { 320 | std::ops::Bound::Included(x) => x * 3 + 2, 321 | std::ops::Bound::Excluded(_) => unreachable!(), 322 | std::ops::Bound::Unbounded => 2, 323 | }; 324 | let end = match range.end_bound() { 325 | std::ops::Bound::Included(x) => x * 3 + 3, 326 | std::ops::Bound::Excluded(x) => x * 3, 327 | std::ops::Bound::Unbounded => self.range_map.len(), 328 | }; 329 | 330 | let mut last_index = 0; 331 | let mut ans = vec![]; 332 | for mut span in self 333 | .range_map 334 | .get_annotations(start, end - start) 335 | .into_iter() 336 | { 337 | let next_index = last_index + span.len; 338 | let len = (next_index + 2) / 3 - (last_index + 2) / 3; 339 | span.len = len; 340 | 341 | type Key = (Lamport, OpID); 342 | let mut annotations: HashMap>)> = 343 | HashMap::new(); 344 | for a in std::mem::take(&mut span.annotations) { 345 | if let Some(x) = annotations.get_mut(&a.type_) { 346 | if a.behavior == Behavior::AllowMultiple { 347 | x.1.push(a); 348 | } else if a.range_lamport > x.0 { 349 | *x = (a.range_lamport, vec![a]); 350 | } 351 | } else { 352 | annotations.insert(a.type_.clone(), (a.range_lamport, vec![a])); 353 | } 354 | } 355 | span.annotations = annotations.into_values().flat_map(|x| x.1).collect(); 356 | ans.push(span); 357 | last_index = next_index; 358 | } 359 | 360 | ans 361 | } 362 | 363 | pub fn len(&self) -> usize { 364 | self.range_map.len() / 3 365 | } 366 | 367 | pub fn is_empty(&self) -> bool { 368 | self.range_map.len() == 2 369 | } 370 | } 371 | 372 | fn index_start(start: Anchor, index: &Index) -> usize 373 | where 374 | Index: Fn(OpID) -> Result, 375 | { 376 | start 377 | .id 378 | .map(|x| match index(x) { 379 | Ok(x) => { 380 | if start.type_ == AnchorType::Before { 381 | x * 3 + 2 382 | } else { 383 | x * 3 + 3 384 | } 385 | } 386 | Err(x) => x * 3 + 1, 387 | }) 388 | .unwrap_or(0) 389 | } 390 | 391 | fn index_end(end: Anchor, index: &Index) -> Option 392 | where 393 | Index: Fn(OpID) -> Result, 394 | { 395 | end.id.map(|x| match index(x) { 396 | Ok(x) => { 397 | if end.type_ == AnchorType::Before { 398 | x * 3 + 2 399 | } else { 400 | x * 3 + 3 401 | } 402 | } 403 | 404 | Err(x) => x * 3 + 1, 405 | }) 406 | } 407 | 408 | impl Default for CrdtRange { 409 | fn default() -> Self { 410 | Self::new() 411 | } 412 | } 413 | 414 | #[cfg(all(test, feature = "test"))] 415 | pub mod test; 416 | -------------------------------------------------------------------------------- /src/legacy/range_map.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::BTreeSet, ops::Range, sync::Arc}; 2 | pub mod tree_impl; 3 | 4 | use crate::{Annotation, Lamport, OpID}; 5 | 6 | pub trait RangeMap { 7 | fn init() -> Self; 8 | /// f is used to position the annotations when they ends in the insert range 9 | fn insert(&mut self, pos: usize, len: usize, f: F) 10 | where 11 | F: FnMut(&Annotation) -> AnnPosRelativeToInsert; 12 | fn insert_directly(&mut self, pos: usize, len: usize) { 13 | self.insert(pos, len, |_| AnnPosRelativeToInsert::IncludeInsert); 14 | } 15 | fn delete(&mut self, pos: usize, len: usize); 16 | fn annotate(&mut self, pos: usize, len: usize, annotation: Annotation); 17 | /// should keep the shrink annotations around even if they are deleted completely 18 | fn adjust_annotation( 19 | &mut self, 20 | target_id: OpID, 21 | lamport: Lamport, 22 | patch_id: OpID, 23 | start_shift: Option<(isize, Option)>, 24 | end_shift: Option<(isize, Option)>, 25 | ); 26 | fn delete_annotation(&mut self, id: OpID); 27 | /// TODO: need to clarify the rules when encounter an empty span on the edges 28 | fn get_annotations(&mut self, pos: usize, len: usize) -> Vec; 29 | fn get_annotation_pos(&self, id: OpID) -> Option<(Arc, Range)>; 30 | fn len(&self) -> usize; 31 | } 32 | 33 | /// the position of annotation relative to a new insert 34 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 35 | pub enum AnnPosRelativeToInsert { 36 | Before, 37 | After, 38 | IncludeInsert, 39 | } 40 | 41 | #[derive(Clone, Debug, Default, PartialEq, Eq)] 42 | pub struct Span { 43 | pub annotations: BTreeSet>, 44 | pub len: usize, 45 | } 46 | 47 | impl Span { 48 | pub fn new(len: usize) -> Self { 49 | Span { 50 | annotations: BTreeSet::new(), 51 | len, 52 | } 53 | } 54 | } 55 | 56 | #[cfg(feature = "test")] 57 | pub mod dumb_impl; 58 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This CRDT lib combines [Peritext](https://www.inkandswitch.com/peritext/) and 2 | //! [Fugue](https://arxiv.org/abs/2305.00583)'s power, delivering impressive performance 3 | //! specifically tailored for rich text. 4 | //! 5 | 6 | #![deny(unsafe_code)] 7 | 8 | use std::{ 9 | cmp::Ordering, 10 | collections::{BTreeSet, HashMap}, 11 | fmt::Debug, 12 | ops::{Bound, Range, RangeBounds}, 13 | sync::Arc, 14 | }; 15 | 16 | use rich_text::Error; 17 | use serde::{Deserialize, Serialize}; 18 | use serde_json::Value; 19 | use string_cache::DefaultAtom; 20 | 21 | pub mod legacy; 22 | pub mod rich_text; 23 | pub use rich_text::{vv::VersionVector, RichText}; 24 | mod small_set; 25 | #[cfg(feature = "test")] 26 | mod test_utils; 27 | pub(crate) type InternalString = DefaultAtom; 28 | type Lamport = u32; 29 | type ClientID = u64; 30 | type Counter = u32; 31 | 32 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] 33 | pub struct OpID { 34 | client: ClientID, 35 | counter: Counter, 36 | } 37 | 38 | impl OpID { 39 | pub fn inc(&self, inc: Counter) -> Self { 40 | Self { 41 | client: self.client, 42 | counter: self.counter + inc as Counter, 43 | } 44 | } 45 | 46 | pub fn inc_i32(&self, inc: i32) -> Self { 47 | if inc > 0 { 48 | Self { 49 | client: self.client, 50 | counter: self.counter + inc as Counter, 51 | } 52 | } else { 53 | let (mut counter, overflow) = self.counter.overflowing_sub((-inc) as Counter); 54 | if overflow { 55 | counter = Counter::MAX; 56 | } 57 | 58 | Self { 59 | client: self.client, 60 | counter, 61 | } 62 | } 63 | } 64 | } 65 | 66 | pub(crate) struct IdSpan { 67 | id: OpID, 68 | len: Counter, 69 | } 70 | 71 | impl IdSpan { 72 | pub fn new(id: OpID, len: usize) -> Self { 73 | Self { 74 | id, 75 | len: len as Counter, 76 | } 77 | } 78 | 79 | pub fn contains(&self, id: OpID) -> bool { 80 | self.id.client == id.client 81 | && self.id.counter <= id.counter 82 | && id.counter < self.id.counter + self.len 83 | } 84 | } 85 | 86 | #[derive(Debug, Clone)] 87 | pub enum RangeOp { 88 | Patch(Patch), 89 | Annotate(Annotation), 90 | } 91 | 92 | #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] 93 | pub enum AnchorType { 94 | Before, 95 | After, 96 | } 97 | 98 | #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)] 99 | pub enum Behavior { 100 | /// When calculating the final state, it will keep all the ranges even if they have the same type 101 | /// 102 | /// For example, we would like to keep both comments alive even if they have overlapped regions 103 | AllowMultiple = 2, 104 | /// When calculating the final state, it will merge the ranges that have overlapped regions and have the same type 105 | /// 106 | /// For example, [bold 2~5] can be merged with [bold 1~4] to produce [bold 1-5] 107 | Merge = 0, 108 | /// It will delete the overlapped range that has smaller lamport && has the same type. 109 | /// But it will keep the `AllowMultiple` type unchanged 110 | Delete = 1, 111 | } 112 | 113 | /// If both `move_start_to` and `move_end_to` equal to None, the target range will be deleted 114 | #[derive(Clone, Copy, Debug)] 115 | pub struct Patch { 116 | pub id: OpID, 117 | pub target_range_id: OpID, 118 | pub move_start_to: Option, 119 | pub move_end_to: Option, 120 | pub lamport: Lamport, 121 | } 122 | 123 | #[derive(Clone, Debug, PartialEq, Eq)] 124 | pub struct Annotation { 125 | pub id: OpID, 126 | /// lamport value of the current range (it may be updated by patch) 127 | pub range_lamport: (Lamport, OpID), 128 | pub range: AnchorRange, 129 | pub behavior: Behavior, 130 | /// "bold", "comment", "italic", etc. 131 | pub type_: InternalString, 132 | pub value: Value, 133 | } 134 | 135 | impl PartialOrd for Annotation { 136 | fn partial_cmp(&self, other: &Self) -> Option { 137 | match self.id.partial_cmp(&other.id) { 138 | Some(core::cmp::Ordering::Equal) => {} 139 | ord => return ord, 140 | } 141 | 142 | self.range_lamport.partial_cmp(&other.range_lamport) 143 | } 144 | } 145 | 146 | impl Ord for Annotation { 147 | fn cmp(&self, other: &Self) -> Ordering { 148 | match self.id.cmp(&other.id) { 149 | core::cmp::Ordering::Equal => {} 150 | ord => return ord, 151 | } 152 | 153 | self.range_lamport.cmp(&other.range_lamport) 154 | } 155 | } 156 | 157 | #[derive(Clone, Copy, PartialEq, Eq, Debug)] 158 | pub enum Expand { 159 | None, 160 | Before, 161 | After, 162 | Both, 163 | } 164 | 165 | impl TryFrom<&str> for Expand { 166 | type Error = (); 167 | 168 | fn try_from(value: &str) -> Result { 169 | match value { 170 | "none" => Ok(Self::None), 171 | "start" => Ok(Self::Before), 172 | "after" => Ok(Self::After), 173 | "both" => Ok(Self::Both), 174 | _ => Err(()), 175 | } 176 | } 177 | } 178 | 179 | impl TryFrom> for Expand { 180 | type Error = (); 181 | 182 | fn try_from(value: Option<&str>) -> Result { 183 | if let Some(value) = value { 184 | match value { 185 | "none" => Ok(Self::None), 186 | "start" => Ok(Self::Before), 187 | "after" => Ok(Self::After), 188 | "both" => Ok(Self::Both), 189 | _ => Err(()), 190 | } 191 | } else { 192 | Ok(Self::After) 193 | } 194 | } 195 | } 196 | 197 | impl Expand { 198 | pub fn infer_insert_expand(type_: &str) -> Self { 199 | match type_ { 200 | "comment" => Self::None, 201 | "header" => Self::None, 202 | "indent" => Self::None, 203 | "list" => Self::None, 204 | "align" => Self::None, 205 | "direction" => Self::None, 206 | "code-block" => Self::None, 207 | "code" => Self::None, 208 | "link" => Self::None, 209 | "script" => Self::None, 210 | "formula" => Self::None, 211 | "image" => Self::None, 212 | "video" => Self::None, 213 | _ => Self::After, 214 | } 215 | } 216 | 217 | pub fn infer_delete_expand(type_: &str) -> Self { 218 | Self::infer_insert_expand(type_).toggle() 219 | } 220 | 221 | /// For a target format, the Expand type of insertion is different 222 | /// from the Expand type of deletion. This method will convert one 223 | // to another. 224 | pub fn toggle(self) -> Self { 225 | match self { 226 | Self::None => Self::Both, 227 | Self::Before => Self::Before, 228 | Self::After => Self::After, 229 | Self::Both => Self::None, 230 | } 231 | } 232 | 233 | pub fn start_type(self) -> AnchorType { 234 | match self { 235 | Self::None => AnchorType::Before, 236 | Self::Before => AnchorType::After, 237 | Self::After => AnchorType::Before, 238 | Self::Both => AnchorType::After, 239 | } 240 | } 241 | 242 | pub fn end_type(self) -> AnchorType { 243 | match self { 244 | Self::None => AnchorType::After, 245 | Self::Before => AnchorType::After, 246 | Self::After => AnchorType::Before, 247 | Self::Both => AnchorType::Before, 248 | } 249 | } 250 | } 251 | 252 | #[derive(Debug, Clone, PartialEq, Eq)] 253 | pub struct Style { 254 | pub expand: Expand, 255 | pub behavior: Behavior, 256 | /// "bold", "comment", "italic", etc. 257 | pub type_: InternalString, 258 | pub value: Value, 259 | } 260 | 261 | impl Style { 262 | pub(crate) fn new_from_expand( 263 | expand: Expand, 264 | type_: InternalString, 265 | value: Value, 266 | behavior: Behavior, 267 | ) -> Result { 268 | Ok(Style { 269 | expand, 270 | behavior, 271 | type_, 272 | value, 273 | }) 274 | } 275 | 276 | pub fn new_bold_like(type_: InternalString, value: Value) -> Self { 277 | Self { 278 | expand: Expand::After, 279 | behavior: Behavior::Merge, 280 | type_, 281 | value, 282 | } 283 | } 284 | 285 | pub fn new_erase_bold_like(type_: InternalString) -> Self { 286 | Self { 287 | expand: Expand::After, 288 | behavior: Behavior::Delete, 289 | type_, 290 | value: Value::Null, 291 | } 292 | } 293 | 294 | pub fn new_link_like(type_: InternalString, value: Value) -> Self { 295 | Self { 296 | expand: Expand::None, 297 | behavior: Behavior::Merge, 298 | type_, 299 | value, 300 | } 301 | } 302 | 303 | pub fn new_erase_link_like(type_: InternalString) -> Self { 304 | Self { 305 | expand: Expand::Both, 306 | behavior: Behavior::Delete, 307 | type_, 308 | value: Value::Null, 309 | } 310 | } 311 | 312 | pub fn new_comment_like(type_: InternalString, value: Value) -> Self { 313 | Self { 314 | expand: Expand::None, 315 | behavior: Behavior::AllowMultiple, 316 | type_, 317 | value, 318 | } 319 | } 320 | 321 | #[inline(always)] 322 | pub fn start_type(&self) -> AnchorType { 323 | self.expand.start_type() 324 | } 325 | 326 | #[inline(always)] 327 | pub fn end_type(&self) -> AnchorType { 328 | self.expand.end_type() 329 | } 330 | } 331 | 332 | #[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord)] 333 | pub struct AnchorRange { 334 | pub start: Anchor, 335 | pub end: Anchor, 336 | } 337 | 338 | #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] 339 | pub struct Anchor { 340 | /// if id is None, it means the anchor is at the beginning or the end of the document 341 | pub id: Option, 342 | pub type_: AnchorType, 343 | } 344 | 345 | impl RangeOp { 346 | fn id(&self) -> OpID { 347 | match self { 348 | RangeOp::Patch(x) => x.id, 349 | RangeOp::Annotate(x) => x.id, 350 | } 351 | } 352 | 353 | #[allow(unused)] 354 | fn set_id(&mut self, id: OpID) { 355 | match self { 356 | RangeOp::Patch(x) => x.id = id, 357 | RangeOp::Annotate(x) => x.id = id, 358 | } 359 | } 360 | 361 | #[allow(unused)] 362 | fn lamport(&self) -> Lamport { 363 | match self { 364 | RangeOp::Patch(x) => x.lamport, 365 | RangeOp::Annotate(x) => x.range_lamport.0, 366 | } 367 | } 368 | } 369 | 370 | impl Anchor { 371 | pub fn before(id: OpID) -> Self { 372 | Self { 373 | id: Some(id), 374 | type_: AnchorType::Before, 375 | } 376 | } 377 | 378 | pub fn after(id: OpID) -> Self { 379 | Self { 380 | id: Some(id), 381 | type_: AnchorType::After, 382 | } 383 | } 384 | 385 | pub fn before_none() -> Self { 386 | Self { 387 | id: None, 388 | type_: AnchorType::Before, 389 | } 390 | } 391 | 392 | pub fn after_none() -> Self { 393 | Self { 394 | id: None, 395 | type_: AnchorType::After, 396 | } 397 | } 398 | } 399 | 400 | impl> From for AnchorRange { 401 | fn from(range: T) -> Self { 402 | let start = match range.start_bound() { 403 | Bound::Included(x) => Anchor { 404 | id: Some(*x), 405 | type_: AnchorType::Before, 406 | }, 407 | Bound::Excluded(x) => Anchor { 408 | id: Some(*x), 409 | type_: AnchorType::After, 410 | }, 411 | Bound::Unbounded => Anchor { 412 | id: None, 413 | type_: AnchorType::After, 414 | }, 415 | }; 416 | let end = match range.end_bound() { 417 | Bound::Included(x) => Anchor { 418 | id: Some(*x), 419 | type_: AnchorType::After, 420 | }, 421 | Bound::Excluded(x) => Anchor { 422 | id: Some(*x), 423 | type_: AnchorType::Before, 424 | }, 425 | Bound::Unbounded => Anchor { 426 | id: None, 427 | type_: AnchorType::Before, 428 | }, 429 | }; 430 | Self { start, end } 431 | } 432 | } 433 | 434 | impl OpID { 435 | pub fn new(client: u64, counter: Counter) -> Self { 436 | Self { client, counter } 437 | } 438 | } 439 | -------------------------------------------------------------------------------- /src/rich_text/cursor.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | mem::replace, 3 | sync::{Arc, Mutex}, 4 | }; 5 | 6 | use generic_btree::{rle::HasLength, ArenaIndex, MoveEvent, MoveListener}; 7 | 8 | use crate::{Counter, OpID}; 9 | 10 | use super::{id_map::IdMap, rich_tree::Elem}; 11 | 12 | #[derive(Debug, Clone, PartialEq, Eq)] 13 | pub enum Cursor { 14 | Insert(ArenaIndex), 15 | // Delete(DeleteOp), 16 | // Ann(Arc), 17 | } 18 | 19 | #[derive(Debug)] 20 | pub struct CursorMap { 21 | map: Arc>>, 22 | } 23 | 24 | impl CursorMap { 25 | pub fn new() -> Self { 26 | CursorMap { 27 | map: Arc::new(Mutex::new(IdMap::new())), 28 | } 29 | } 30 | 31 | pub fn gen_update_fn(&self) -> MoveListener { 32 | let map = self.map.clone(); 33 | Box::new(move |event| { 34 | listen(event, &mut map.try_lock().unwrap()); 35 | }) 36 | } 37 | 38 | #[inline] 39 | pub fn update(&mut self, event: MoveEvent) { 40 | listen(event, &mut self.map.try_lock().unwrap()); 41 | } 42 | 43 | // pub fn register_del(&mut self, op: &Op) { 44 | // let mut map = self.map.try_lock().unwrap(); 45 | // let content = match &op.content { 46 | // OpContent::Del(del) => del, 47 | // _ => unreachable!(), 48 | // }; 49 | // if let Some(mut start) = map.get_last(op.id) { 50 | // if start.start_counter == op.id.counter { 51 | // debug_assert!(op.rle_len() > start.len); 52 | // let Cursor::Delete(del) = &mut start.value else { unreachable!() }; 53 | // debug_assert_eq!(del.start, content.start); 54 | // del.len = content.len; 55 | // start.len = op.rle_len(); 56 | // return; 57 | // } else if start.start_counter + start.len as Counter == op.id.counter { 58 | // if let Cursor::Delete(del) = &mut start.value { 59 | // if del.can_merge(content) { 60 | // del.merge_right(content); 61 | // start.len += content.rle_len(); 62 | // return; 63 | // } 64 | // } 65 | // } else { 66 | // // TODO: should we check here? 67 | // return; 68 | // } 69 | // } 70 | 71 | // map.insert( 72 | // op.id, 73 | // Cursor::Delete(*content), 74 | // content.len.unsigned_abs() as usize, 75 | // ); 76 | // } 77 | 78 | // pub fn register_ann(&mut self, op: &Op) { 79 | // let mut map = self.map.try_lock().unwrap(); 80 | // let content = match &op.content { 81 | // OpContent::Ann(ann) => ann, 82 | // _ => unreachable!(), 83 | // }; 84 | // map.insert(op.id, Cursor::Ann(content.clone()), 1); 85 | // } 86 | 87 | pub fn get_insert(&self, id: OpID) -> Option<(ArenaIndex, usize)> { 88 | let map = self.map.try_lock().unwrap(); 89 | if let Some(start) = map.get(id) { 90 | if start.start_counter <= id.counter 91 | && start.start_counter + start.len as Counter > id.counter 92 | { 93 | if let Cursor::Insert(leaf) = start.value { 94 | return Some(( 95 | leaf, 96 | start.len - (id.counter - start.start_counter) as usize, 97 | )); 98 | } else { 99 | unreachable!() 100 | } 101 | } 102 | } 103 | 104 | None 105 | } 106 | } 107 | 108 | fn listen(event: MoveEvent, m: &mut IdMap) { 109 | let Some(leaf) = event.target_leaf else { return }; 110 | let elem = event.elem; 111 | let mut id = elem.id; 112 | let mut cursor = Cursor::Insert(leaf); 113 | let mut len = elem.atom_len(); 114 | 'handle_old: { 115 | if let Some(nearest_last_span) = m.remove_range_return_last(elem.id, elem.atom_len()) { 116 | let mut nearest_last = nearest_last_span.borrow_mut(); 117 | if nearest_last.start_counter + (nearest_last.len as Counter) <= elem.id.counter { 118 | // It have no overlap with the new element, break here 119 | break 'handle_old; 120 | } 121 | 122 | if nearest_last.value == Cursor::Insert(leaf) { 123 | // already has the same value as new elem 124 | if nearest_last.start_counter + (nearest_last.len as Counter) 125 | < elem.id.counter + elem.atom_len() as Counter 126 | { 127 | // extend the length if it's not enough 128 | nearest_last.len = 129 | (elem.id.counter - nearest_last.start_counter) as usize + elem.atom_len(); 130 | } 131 | return; 132 | } 133 | 134 | if nearest_last.start_counter == elem.id.counter { 135 | // both have the same start counter 136 | if elem.rle_len() >= nearest_last.len { 137 | // if new elem is longer, replace the target value 138 | nearest_last.value = Cursor::Insert(leaf); 139 | nearest_last.len = elem.atom_len(); 140 | return; 141 | } else { 142 | // if new elem is shorter, split the last span: 143 | // 144 | // 1. set the new value and new len to the span 145 | // 2. insert the rest of the last span to the map 146 | let left_len = nearest_last.len - elem.atom_len(); 147 | let start_id = elem.id.inc(elem.atom_len() as Counter); 148 | let old_value = replace(&mut nearest_last.value, Cursor::Insert(leaf)); 149 | nearest_last.len = elem.atom_len(); 150 | id = start_id; 151 | cursor = old_value; 152 | len = left_len; 153 | } 154 | } else { 155 | // remove the overlapped part from last span 156 | nearest_last.len = nearest_last 157 | .len 158 | .min((elem.id.counter - nearest_last.start_counter) as usize); 159 | } 160 | } 161 | } 162 | 163 | m.insert(id, cursor, len); 164 | } 165 | 166 | impl Default for CursorMap { 167 | fn default() -> Self { 168 | Self::new() 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/rich_text/delta.rs: -------------------------------------------------------------------------------- 1 | use std::mem::swap; 2 | 3 | use fxhash::FxHashMap; 4 | use serde::{Deserialize, Serialize}; 5 | use serde_json::Value; 6 | 7 | use super::rich_tree::{ 8 | query::IndexType, 9 | utf16::{get_utf16_len, utf16_to_utf8}, 10 | }; 11 | 12 | #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] 13 | #[serde(untagged)] 14 | pub enum DeltaItem { 15 | Retain { 16 | retain: usize, 17 | attributes: Option>, 18 | }, 19 | Insert { 20 | insert: String, 21 | attributes: Option>, 22 | len: Option, 23 | index_type: Option, 24 | }, 25 | Delete { 26 | delete: usize, 27 | }, 28 | } 29 | 30 | impl DeltaItem { 31 | pub fn retain(retain: usize) -> Self { 32 | Self::Retain { 33 | retain, 34 | attributes: None, 35 | } 36 | } 37 | 38 | pub fn insert(insert: String, index_type: IndexType) -> Self { 39 | Self::Insert { 40 | len: Some(match index_type { 41 | IndexType::Utf8 => insert.len(), 42 | IndexType::Utf16 => get_utf16_len(&insert), 43 | }), 44 | insert, 45 | index_type: Some(index_type), 46 | attributes: None, 47 | } 48 | } 49 | 50 | pub fn delete(delete: usize) -> Self { 51 | Self::Delete { delete } 52 | } 53 | 54 | pub fn retain_with_attributes(retain: usize, attributes: FxHashMap) -> Self { 55 | Self::Retain { 56 | retain, 57 | attributes: Some(attributes), 58 | } 59 | } 60 | 61 | pub fn insert_with_attributes( 62 | insert: String, 63 | index_type: IndexType, 64 | attributes: FxHashMap, 65 | ) -> Self { 66 | Self::Insert { 67 | len: Some(match index_type { 68 | IndexType::Utf8 => insert.len(), 69 | IndexType::Utf16 => get_utf16_len(&insert), 70 | }), 71 | insert, 72 | index_type: Some(index_type), 73 | attributes: Some(attributes), 74 | } 75 | } 76 | 77 | pub fn is_retain(&self) -> bool { 78 | matches!(self, Self::Retain { .. }) 79 | } 80 | 81 | pub fn is_insert(&self) -> bool { 82 | matches!(self, Self::Insert { .. }) 83 | } 84 | 85 | pub fn is_delete(&self) -> bool { 86 | matches!(self, Self::Delete { .. }) 87 | } 88 | 89 | pub fn attributions(&self) -> Option<&FxHashMap> { 90 | match self { 91 | Self::Retain { attributes, .. } => attributes.as_ref(), 92 | Self::Insert { attributes, .. } => attributes.as_ref(), 93 | Self::Delete { .. } => None, 94 | } 95 | } 96 | 97 | pub fn length(&self) -> usize { 98 | match self { 99 | Self::Retain { retain, .. } => *retain, 100 | Self::Insert { len, insert, .. } => len.unwrap_or_else(|| get_utf16_len(insert)), 101 | Self::Delete { delete, .. } => *delete, 102 | } 103 | } 104 | 105 | pub fn should_remove(&self) -> bool { 106 | match self { 107 | Self::Retain { retain, .. } => *retain == 0, 108 | Self::Insert { .. } => false, 109 | Self::Delete { delete, .. } => *delete == 0, 110 | } 111 | } 112 | 113 | /// Take the first length characters from the delta item 114 | pub(crate) fn take(&mut self, length: usize) -> Self { 115 | match self { 116 | DeltaItem::Insert { 117 | insert, 118 | attributes, 119 | len, 120 | index_type, 121 | } => match index_type { 122 | Some(IndexType::Utf8) => { 123 | let mut v = insert.split_off(length); 124 | swap(&mut v, insert); 125 | *len = Some(insert.len()); 126 | 127 | Self::Insert { 128 | insert: v, 129 | len: Some(length), 130 | index_type: Some(IndexType::Utf8), 131 | attributes: attributes.clone(), 132 | } 133 | } 134 | None | Some(IndexType::Utf16) => { 135 | let utf8length = utf16_to_utf8(insert.as_bytes(), length); 136 | let mut v = insert.split_off(utf8length); 137 | swap(&mut v, insert); 138 | match len { 139 | Some(len) => { 140 | *len -= length; 141 | } 142 | None => *len = Some(get_utf16_len(&insert)), 143 | } 144 | 145 | Self::Insert { 146 | insert: v, 147 | len: Some(length), 148 | index_type: *index_type, 149 | attributes: attributes.clone(), 150 | } 151 | } 152 | }, 153 | DeltaItem::Retain { retain, attributes } => { 154 | *retain -= length; 155 | Self::Retain { 156 | retain: length, 157 | attributes: attributes.clone(), 158 | } 159 | } 160 | DeltaItem::Delete { delete } => { 161 | *delete -= length; 162 | Self::Delete { delete: length } 163 | } 164 | } 165 | } 166 | 167 | fn compose_meta(&mut self, next_op: &DeltaItem) { 168 | let attributions = match self { 169 | DeltaItem::Retain { attributes, .. } => attributes, 170 | DeltaItem::Insert { attributes, .. } => attributes, 171 | DeltaItem::Delete { .. } => return, 172 | }; 173 | 174 | if attributions.is_none() { 175 | *attributions = Some(FxHashMap::default()); 176 | } 177 | 178 | let self_attributions = attributions.as_mut().unwrap(); 179 | if let Some(attributions) = next_op.attributions() { 180 | for attr in attributions { 181 | self_attributions.insert(attr.0.clone(), attr.1.clone()); 182 | } 183 | } 184 | } 185 | } 186 | 187 | pub struct DeltaIterator { 188 | // The reversed Vec uses pop() to simulate getting the first element each time 189 | ops: Vec, 190 | } 191 | 192 | impl DeltaIterator { 193 | fn new(mut ops: Vec) -> Self { 194 | ops.reverse(); 195 | Self { ops } 196 | } 197 | 198 | #[inline(always)] 199 | fn next>>(&mut self, len: L) -> DeltaItem { 200 | self.next_impl(len.into()) 201 | } 202 | 203 | fn next_impl(&mut self, len: Option) -> DeltaItem { 204 | let length = len.unwrap_or(usize::MAX); 205 | let next_op = self.peek_mut(); 206 | if next_op.is_none() { 207 | return DeltaItem::Retain { 208 | retain: usize::MAX, 209 | attributes: None, 210 | }; 211 | } 212 | let op = next_op.unwrap(); 213 | let op_length = op.length(); 214 | if length < op_length { 215 | // a part of the peek op 216 | op.take(length) 217 | } else { 218 | self.take_peek().unwrap() 219 | } 220 | } 221 | 222 | fn next_with_ref(&mut self, len: usize, other: &DeltaItem) -> DeltaItem { 223 | let next_op = self.peek_mut(); 224 | if next_op.is_none() { 225 | return DeltaItem::Retain { 226 | retain: other.length(), 227 | attributes: other.attributions().cloned(), 228 | }; 229 | } 230 | let op = next_op.unwrap(); 231 | let op_length = op.length(); 232 | if len < op_length { 233 | // a part of the peek op 234 | op.take(len) 235 | } else { 236 | self.take_peek().unwrap() 237 | } 238 | } 239 | 240 | fn next_pair(&mut self, other: &mut Self) -> (DeltaItem, DeltaItem) { 241 | let self_len = self.peek_length(); 242 | let other_len = other.peek_length(); 243 | if self_len > other_len { 244 | let length = other_len; 245 | let other_op = other.next(None); 246 | debug_assert_eq!(other_op.length(), length); 247 | let this_op = self.next_with_ref(length, &other_op); 248 | (this_op, other_op) 249 | } else { 250 | let length = self_len; 251 | let this_op = self.next(None); 252 | debug_assert_eq!(this_op.length(), length); 253 | let other_op = other.next_with_ref(length, &this_op); 254 | (this_op, other_op) 255 | } 256 | } 257 | 258 | fn peek_mut(&mut self) -> Option<&mut DeltaItem> { 259 | self.ops.last_mut() 260 | } 261 | 262 | fn take_peek(&mut self) -> Option { 263 | self.ops.pop() 264 | } 265 | 266 | fn rest(mut self) -> Vec { 267 | self.ops.reverse(); 268 | self.ops 269 | } 270 | 271 | fn has_next(&self) -> bool { 272 | !self.ops.is_empty() 273 | } 274 | 275 | fn peek(&self) -> Option<&DeltaItem> { 276 | self.ops.last() 277 | } 278 | 279 | fn peek_length(&self) -> usize { 280 | if let Some(op) = self.peek() { 281 | op.length() 282 | } else { 283 | usize::MAX 284 | } 285 | } 286 | 287 | fn peek_is_insert(&self) -> bool { 288 | if let Some(op) = self.peek() { 289 | op.is_insert() 290 | } else { 291 | false 292 | } 293 | } 294 | 295 | fn peek_is_delete(&self) -> bool { 296 | if let Some(op) = self.peek() { 297 | op.is_delete() 298 | } else { 299 | false 300 | } 301 | } 302 | } 303 | 304 | pub fn compose(delta_a: Vec, delta_b: Vec) -> Vec { 305 | let mut this_iter = DeltaIterator::new(delta_a); 306 | let mut other_iter = DeltaIterator::new(delta_b); 307 | let mut ops = vec![]; 308 | let first_other = other_iter.peek(); 309 | if let Some(first_other) = first_other { 310 | // if other.delta starts with retain, we insert corresponding number of inserts from self.delta 311 | if first_other.is_retain() && first_other.attributions().is_none() { 312 | let mut first_left = first_other.length(); 313 | let mut first_this = this_iter.peek(); 314 | while let Some(first_this_inner) = first_this { 315 | if first_this_inner.is_insert() && first_this_inner.length() <= first_left { 316 | first_left -= first_this_inner.length(); 317 | let mut op = this_iter.next(None); 318 | op.compose_meta(first_other); 319 | ops.push(op); 320 | first_this = this_iter.peek(); 321 | } else { 322 | break; 323 | } 324 | } 325 | if first_other.length() - first_left > 0 { 326 | other_iter.next(first_other.length() - first_left); 327 | } 328 | } 329 | } 330 | let mut delta = ops; 331 | while this_iter.has_next() || other_iter.has_next() { 332 | if other_iter.peek_is_insert() { 333 | // nothing to compose here 334 | delta.push(other_iter.next(None)); 335 | } else if this_iter.peek_is_delete() { 336 | // nothing to compose here 337 | delta.push(this_iter.next(None)); 338 | } else { 339 | // possible cases: 340 | // 1. this: insert, other: retain 341 | // 2. this: retain, other: retain 342 | // 3. this: retain, other: delete 343 | // 4. this: insert, other: delete 344 | 345 | let (mut this_op, mut other_op) = this_iter.next_pair(&mut other_iter); 346 | if other_op.is_retain() { 347 | // 1. this: insert, other: retain 348 | // 2. this: retain, other: retain 349 | this_op.compose_meta(&other_op); 350 | delta.push(this_op); 351 | let concat_rest = !other_iter.has_next(); 352 | if concat_rest { 353 | let vec = this_iter.rest(); 354 | if vec.is_empty() { 355 | return chop(delta); 356 | } 357 | let mut rest = vec; 358 | delta.append(&mut rest); 359 | return chop(delta); 360 | } 361 | } else if other_op.is_delete() && this_op.is_retain() { 362 | // 3. this: retain, other: delete 363 | other_op.compose_meta(&this_op); 364 | // other deletes the retained text 365 | delta.push(other_op); 366 | } else { 367 | // 4. this: insert, other: delete 368 | // nothing to do here, because insert and delete have the same length 369 | } 370 | } 371 | } 372 | chop(delta) 373 | } 374 | 375 | fn chop(mut vec: Vec) -> Vec { 376 | let last_op = vec.last(); 377 | if let Some(last_op) = last_op { 378 | if last_op.is_retain() && last_op.attributions().is_none() { 379 | vec.pop(); 380 | } 381 | } 382 | 383 | vec 384 | } 385 | -------------------------------------------------------------------------------- /src/rich_text/encoding.rs: -------------------------------------------------------------------------------- 1 | use std::io::prelude::*; 2 | use std::ops::Deref; 3 | use std::{hash::Hash, sync::Arc}; 4 | 5 | use append_only_bytes::AppendOnlyBytes; 6 | use flate2::write::GzEncoder; 7 | use flate2::{read::GzDecoder, Compression}; 8 | use fxhash::FxHashMap; 9 | use generic_btree::rle::HasLength; 10 | use serde::{Deserialize, Serialize}; 11 | use serde_columnar::{columnar, from_bytes, to_vec}; 12 | 13 | use crate::{ 14 | Anchor, AnchorRange, AnchorType, Annotation, Behavior, ClientID, InternalString, OpID, 15 | }; 16 | 17 | use super::op::{DeleteOp, Op, OpContent, TextInsertOp}; 18 | const COMPRESS_THRESHOLD: usize = 1024; 19 | 20 | #[columnar(vec, ser, de)] 21 | #[derive(Debug, Clone, Serialize, Deserialize)] 22 | pub(super) struct OpEncoding { 23 | #[columnar(strategy = "DeltaRle")] 24 | lamport: u32, 25 | #[columnar(strategy = "Rle")] 26 | type_: u8, 27 | } 28 | 29 | #[columnar(vec, ser, de)] 30 | #[derive(Debug, Clone, Serialize, Deserialize)] 31 | pub(super) struct InsertEncoding { 32 | len: u32, 33 | #[columnar(strategy = "Rle")] 34 | left_client: u32, 35 | #[columnar(strategy = "DeltaRle")] 36 | left_counter: u32, 37 | #[columnar(strategy = "Rle")] 38 | right_client: u32, 39 | #[columnar(strategy = "DeltaRle")] 40 | right_counter: u32, 41 | } 42 | 43 | #[columnar(vec, ser, de)] 44 | #[derive(Debug, Clone, Serialize, Deserialize)] 45 | pub(super) struct DeleteEncoding { 46 | #[columnar(strategy = "Rle")] 47 | start_client: u32, 48 | #[columnar(strategy = "DeltaRle")] 49 | start_counter: u32, 50 | len: i32, 51 | } 52 | 53 | #[columnar(vec, ser, de)] 54 | #[derive(Debug, Clone, Serialize, Deserialize)] 55 | pub(super) struct AnnEncoding { 56 | start: Option, 57 | #[columnar(strategy = "Rle")] 58 | is_start_before_anchor: bool, 59 | end: Option, 60 | #[columnar(strategy = "Rle")] 61 | is_end_before_anchor: bool, 62 | behavior: Behavior, 63 | /// index to ann_types_and_values 64 | type_: u32, 65 | /// index to ann_types_and_values 66 | value: u32, 67 | } 68 | 69 | #[columnar(ser, de)] 70 | #[derive(Debug, Serialize, Deserialize)] 71 | struct DocEncoding { 72 | #[columnar(type = "vec")] 73 | ops: Vec, 74 | #[columnar(type = "vec")] 75 | inserts: Vec, 76 | #[columnar(type = "vec")] 77 | deletes: Vec, 78 | #[columnar(type = "vec")] 79 | annotations: Vec, 80 | 81 | str: Vec, 82 | compressed_str: bool, 83 | clients: Vec, 84 | ann_types_and_values: Vec, 85 | op_len: Vec, 86 | start_counters: Vec, 87 | } 88 | 89 | #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] 90 | pub(crate) enum OpContentType { 91 | Insert = 0, 92 | Delete = 1, 93 | Ann = 2, 94 | } 95 | 96 | impl From for u8 { 97 | fn from(value: OpContentType) -> Self { 98 | value as u8 99 | } 100 | } 101 | 102 | impl From for OpContentType { 103 | fn from(value: u8) -> Self { 104 | match value { 105 | 0 => OpContentType::Insert, 106 | 1 => OpContentType::Delete, 107 | 2 => OpContentType::Ann, 108 | _ => unreachable!(), 109 | } 110 | } 111 | } 112 | 113 | type InnerUpdates = FxHashMap>; 114 | 115 | pub fn encode(exported: InnerUpdates) -> Vec { 116 | let data = to_doc_encoding(exported); 117 | to_vec(&data).unwrap() 118 | } 119 | 120 | pub fn decode(encoded: &[u8]) -> InnerUpdates { 121 | from_doc_encoding(from_bytes(encoded).unwrap()) 122 | } 123 | 124 | fn to_doc_encoding(mut exported_map: InnerUpdates) -> DocEncoding { 125 | exported_map.retain(|_, v| !v.is_empty()); 126 | let mut inserts = Vec::new(); 127 | let mut deletes = Vec::new(); 128 | let mut annotations = Vec::new(); 129 | let mut client_mapping = VecMapping::new(); 130 | for client in exported_map.keys() { 131 | client_mapping.get_or_insert(*client); 132 | } 133 | 134 | let mut ann_str_mapping = VecMapping::new(); 135 | let mut op_len: Vec = Vec::new(); 136 | let mut start_counters: Vec = Vec::new(); 137 | let mut ops = Vec::with_capacity(exported_map.iter().map(|x| x.1.len()).sum()); 138 | let mut str = Vec::new(); 139 | 140 | for (_, op_arr) in exported_map.iter() { 141 | op_len.push(op_arr.len() as u32); 142 | start_counters.push(op_arr[0].id.counter); 143 | for op in op_arr { 144 | let type_ = match &op.content { 145 | crate::rich_text::op::OpContent::Text(text) => { 146 | str.extend_from_slice(&text.text); 147 | let zero = OpID::new(0, 0); 148 | inserts.push(InsertEncoding { 149 | len: text.text.len() as u32, 150 | left_client: text 151 | .left 152 | .map(|x| client_mapping.get_or_insert(x.client) as u32) 153 | .unwrap_or(u32::MAX), 154 | left_counter: text.left.unwrap_or(zero).counter, 155 | right_client: text 156 | .right 157 | .map(|x| client_mapping.get_or_insert(x.client) as u32) 158 | .unwrap_or(u32::MAX), 159 | right_counter: text.right.unwrap_or(zero).counter, 160 | }); 161 | OpContentType::Insert 162 | } 163 | crate::rich_text::op::OpContent::Del(del) => { 164 | deletes.push(DeleteEncoding { 165 | start_client: client_mapping.get_or_insert(del.start.client) as u32, 166 | start_counter: del.start.counter, 167 | len: del.len, 168 | }); 169 | OpContentType::Delete 170 | } 171 | crate::rich_text::op::OpContent::Ann(ann) => { 172 | let start = ann.range.start.id; 173 | let end = ann.range.end.id; 174 | let type_ = ann_str_mapping.get_or_insert(ann.type_.clone()); 175 | let value = serde_json::to_string(&ann.value).unwrap(); 176 | let value = ann_str_mapping.get_or_insert(value.into()); 177 | annotations.push(AnnEncoding { 178 | start, 179 | is_start_before_anchor: ann.range.start.type_ == AnchorType::Before, 180 | end, 181 | is_end_before_anchor: ann.range.end.type_ == AnchorType::Before, 182 | behavior: ann.behavior, 183 | type_: type_ as u32, 184 | value: value as u32, 185 | }); 186 | OpContentType::Ann 187 | } 188 | }; 189 | 190 | ops.push(OpEncoding { 191 | lamport: op.lamport, 192 | type_: type_.into(), 193 | }); 194 | } 195 | } 196 | 197 | assert_eq!(op_len.len(), exported_map.len()); 198 | assert_eq!(op_len.len(), start_counters.len()); 199 | assert_eq!(op_len.iter().sum::() as usize, ops.len()); 200 | debug_assert_eq!( 201 | str.len(), 202 | inserts.iter().map(|x| x.len).sum::() as usize 203 | ); 204 | let mut compressed_str = false; 205 | if str.len() > COMPRESS_THRESHOLD { 206 | compressed_str = true; 207 | let mut e = GzEncoder::new(Vec::new(), Compression::default()); 208 | e.write_all(&str).unwrap(); 209 | str = e.finish().unwrap(); 210 | } 211 | 212 | DocEncoding { 213 | ops, 214 | inserts, 215 | deletes, 216 | annotations, 217 | compressed_str, 218 | clients: client_mapping.vec, 219 | ann_types_and_values: ann_str_mapping.vec, 220 | op_len, 221 | start_counters, 222 | str, 223 | } 224 | } 225 | 226 | fn from_doc_encoding(exported: DocEncoding) -> InnerUpdates { 227 | let clients = &exported.clients; 228 | let mut str = AppendOnlyBytes::new(); 229 | if exported.compressed_str { 230 | let mut d = GzDecoder::new(exported.str.deref()); 231 | let mut ans = vec![]; 232 | d.read_to_end(&mut ans).unwrap(); 233 | str.push_slice(&ans); 234 | } else { 235 | str.push_slice(&exported.str); 236 | } 237 | let mut str_index = 0; 238 | let mut ans: InnerUpdates = Default::default(); 239 | let mut insert_iter = exported.inserts.iter(); 240 | let mut delete_iter = exported.deletes.iter(); 241 | let mut ann_iter = exported.annotations.iter(); 242 | let mut op_iter = exported.ops.iter(); 243 | for ((client, op_len), counter) in exported 244 | .clients 245 | .iter() 246 | .zip(exported.op_len.iter()) 247 | .zip(exported.start_counters.iter()) 248 | { 249 | let mut counter = *counter; 250 | let mut arr = Vec::with_capacity((*op_len) as usize); 251 | for _ in 0..*op_len { 252 | let op = op_iter.next().unwrap(); 253 | let id = OpID { 254 | client: *client, 255 | counter, 256 | }; 257 | let content = match op.type_.into() { 258 | OpContentType::Insert => { 259 | let insert = insert_iter.next().unwrap(); 260 | let left = if insert.left_client != u32::MAX { 261 | Some(OpID { 262 | client: clients[insert.left_client as usize], 263 | counter: insert.left_counter, 264 | }) 265 | } else { 266 | None 267 | }; 268 | let right = if insert.right_client != u32::MAX { 269 | Some(OpID { 270 | client: clients[insert.right_client as usize], 271 | counter: insert.right_counter, 272 | }) 273 | } else { 274 | None 275 | }; 276 | let end = str_index + insert.len as usize; 277 | let text = str.slice(str_index..end); 278 | str_index = end; 279 | OpContent::Text(TextInsertOp { left, right, text }) 280 | } 281 | OpContentType::Delete => { 282 | let delete = delete_iter.next().unwrap(); 283 | OpContent::Del(DeleteOp { 284 | start: OpID { 285 | client: clients[delete.start_client as usize], 286 | counter: delete.start_counter, 287 | }, 288 | len: delete.len, 289 | }) 290 | } 291 | OpContentType::Ann => { 292 | let ann = ann_iter.next().unwrap(); 293 | let range = AnchorRange { 294 | start: Anchor { 295 | id: ann.start, 296 | type_: if ann.is_start_before_anchor { 297 | AnchorType::Before 298 | } else { 299 | AnchorType::After 300 | }, 301 | }, 302 | end: Anchor { 303 | id: ann.end, 304 | type_: if ann.is_end_before_anchor { 305 | AnchorType::Before 306 | } else { 307 | AnchorType::After 308 | }, 309 | }, 310 | }; 311 | 312 | OpContent::Ann(Arc::new(Annotation { 313 | range, 314 | behavior: ann.behavior, 315 | type_: exported.ann_types_and_values[ann.type_ as usize].clone(), 316 | id, 317 | range_lamport: (op.lamport, id), 318 | value: serde_json::from_str( 319 | &exported.ann_types_and_values[ann.value as usize], 320 | ) 321 | .unwrap(), 322 | })) 323 | } 324 | }; 325 | 326 | let op = Op { 327 | id, 328 | lamport: op.lamport, 329 | content, 330 | }; 331 | counter += op.rle_len() as u32; 332 | arr.push(op); 333 | } 334 | 335 | ans.insert(*client, arr); 336 | } 337 | 338 | ans 339 | } 340 | 341 | struct VecMapping { 342 | vec: Vec, 343 | map: FxHashMap, 344 | } 345 | 346 | impl VecMapping { 347 | fn new() -> Self { 348 | Self { 349 | vec: Vec::new(), 350 | map: FxHashMap::default(), 351 | } 352 | } 353 | 354 | fn get(&self, idx: usize) -> &T { 355 | &self.vec[idx] 356 | } 357 | 358 | fn get_or_insert(&mut self, val: T) -> usize { 359 | if let Some(idx) = self.map.get(&val) { 360 | *idx 361 | } else { 362 | let idx = self.vec.len(); 363 | self.vec.push(val.clone()); 364 | self.map.insert(val, idx); 365 | idx 366 | } 367 | } 368 | } 369 | -------------------------------------------------------------------------------- /src/rich_text/error.rs: -------------------------------------------------------------------------------- 1 | #[derive(thiserror::Error, Debug)] 2 | pub enum Error { 3 | #[error("Decode error")] 4 | DecodeError, 5 | #[error("Invalid expand")] 6 | InvalidExpand, 7 | } 8 | -------------------------------------------------------------------------------- /src/rich_text/event.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | use super::{delta::DeltaItem, rich_tree::query::IndexType}; 4 | 5 | #[derive(Debug, Serialize, Deserialize, Clone)] 6 | pub struct Event { 7 | pub ops: Vec, 8 | pub is_local: bool, 9 | pub index_type: IndexType, 10 | } 11 | -------------------------------------------------------------------------------- /src/rich_text/id_map.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | cell::{RefCell, RefMut}, 3 | collections::BTreeMap, 4 | rc::Rc, 5 | }; 6 | 7 | use fxhash::FxHashMap; 8 | 9 | use crate::{ClientID, Counter, OpID}; 10 | 11 | type Tree = BTreeMap>>>; 12 | /// This structure helps to map a range of IDs to a value. 13 | /// 14 | /// It's the call site's responsibility to ensure there is no overlap in the range 15 | #[derive(Debug, Default)] 16 | pub(super) struct IdMap { 17 | pub(super) map: FxHashMap>, 18 | } 19 | 20 | #[derive(Debug)] 21 | pub(super) struct Entry { 22 | pub len: usize, 23 | pub start_counter: Counter, 24 | pub value: Value, 25 | } 26 | 27 | impl IdMap { 28 | pub fn new() -> Self { 29 | Self { 30 | map: Default::default(), 31 | } 32 | } 33 | 34 | #[allow(unused)] 35 | pub fn len(&self) -> usize { 36 | self.map.len() 37 | } 38 | 39 | #[allow(unused)] 40 | pub fn is_empty(&self) -> bool { 41 | self.map.is_empty() 42 | } 43 | 44 | pub fn get(&self, id: OpID) -> Option>> { 45 | let client_map = self.map.get(&id.client)?; 46 | client_map 47 | .range(..=id.counter) 48 | .next_back() 49 | .and_then(|(counter, v)| { 50 | let v = v.borrow_mut(); 51 | debug_assert_eq!(v.start_counter, *counter); 52 | if counter + v.len as Counter > id.counter { 53 | Some(v) 54 | } else { 55 | None 56 | } 57 | }) 58 | } 59 | 60 | pub fn get_last(&self, id: OpID) -> Option>> { 61 | let client_map = self.map.get(&id.client)?; 62 | client_map 63 | .range(..=id.counter) 64 | .next_back() 65 | .map(|(counter, v)| { 66 | let v = v.borrow_mut(); 67 | debug_assert_eq!(v.start_counter, *counter); 68 | v 69 | }) 70 | } 71 | 72 | /// Remove any entries that start within the range of (exclusive_from, exclusive_from + len) 73 | /// 74 | /// It'll return the pointer to the alive last entry, which is the same as [`IdMap::get_last`] 75 | pub fn remove_range_return_last( 76 | &mut self, 77 | exclusive_from: OpID, 78 | len: usize, 79 | ) -> Option>>> { 80 | let last_id = exclusive_from.inc((len - 1) as Counter); 81 | let Some(client_map) = self.map.get_mut(&last_id.client) else { return None }; 82 | loop { 83 | let mutex_item = client_map 84 | .range(..=last_id.counter) 85 | .next_back() 86 | .map(|(_, v)| v); 87 | 88 | let item = mutex_item.map(|x| x.borrow_mut()); 89 | let Some(item_inner) = item.as_ref() else { return None }; 90 | let item_counter = item_inner.start_counter; 91 | let item_end = item_inner.len as Counter + item_counter; 92 | if item_inner.start_counter <= exclusive_from.counter { 93 | return mutex_item.cloned(); 94 | } 95 | 96 | drop(item); 97 | let item = client_map.remove(&item_counter).unwrap(); 98 | let new_item_counter = last_id.counter + 1; 99 | if item_end > new_item_counter { 100 | let mut inner = item.borrow_mut(); 101 | inner.len = (item_end - new_item_counter) as usize; 102 | inner.start_counter = new_item_counter; 103 | drop(inner); 104 | client_map.insert(new_item_counter, item); 105 | } 106 | } 107 | } 108 | 109 | pub fn insert(&mut self, id: OpID, v: Value, len: usize) { 110 | debug_assert!( 111 | self.get(id).is_none(), 112 | "Unexpected overlap {:?} {:?} {:#?}", 113 | id, 114 | &v, 115 | self.get(id).unwrap() 116 | ); 117 | let client_map = self.map.entry(id.client).or_default(); 118 | let elem = Rc::new(RefCell::new(Entry { 119 | len, 120 | value: v, 121 | start_counter: id.counter, 122 | })); 123 | client_map.insert(id.counter, elem); 124 | } 125 | 126 | #[allow(unused)] 127 | pub fn remove(&mut self, id: OpID, len: usize) -> bool { 128 | let Some(mut g) = self.get(id) else { return false }; 129 | if g.start_counter == id.counter && g.len == len { 130 | // remove entry directly 131 | drop(g); 132 | let client_map = self.map.get_mut(&id.client).unwrap(); 133 | client_map.remove(&id.counter); 134 | } else if g.start_counter == id.counter { 135 | // split entry 136 | g.start_counter += len as Counter; 137 | g.len -= len; 138 | drop(g); 139 | let client_map = self.map.get_mut(&id.client).unwrap(); 140 | let Some((_, value)) = client_map.remove_entry(&id.counter) else {unreachable!()}; 141 | client_map.insert(id.counter + len as Counter, value); 142 | } else if g.start_counter + g.len as Counter == id.counter + len as Counter { 143 | // adjust length 144 | g.len -= len; 145 | } else { 146 | // adjust length + split 147 | let start_counter = id.counter + len as Counter; 148 | let new_elem = Rc::new(RefCell::new(Entry { 149 | len: g.len - len - (id.counter - g.start_counter) as usize, 150 | value: g.value.clone(), 151 | start_counter, 152 | })); 153 | g.len -= len; 154 | drop(g); 155 | let client_map = self.map.get_mut(&id.client).unwrap(); 156 | client_map.insert(start_counter, new_elem); 157 | } 158 | true 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/rich_text/iter.rs: -------------------------------------------------------------------------------- 1 | use std::mem::take; 2 | 3 | use fxhash::FxHashMap; 4 | use generic_btree::{rle::Mergeable, QueryResult}; 5 | 6 | use crate::Behavior; 7 | 8 | use super::{ 9 | ann::{Span, StyleCalculator}, 10 | RichText, 11 | }; 12 | 13 | pub struct Iter<'a> { 14 | text: &'a RichText, 15 | style_calc: StyleCalculator, 16 | cursor: QueryResult, 17 | end: Option, 18 | pending_return: Option, 19 | done: bool, 20 | } 21 | 22 | impl<'a> Iter<'a> { 23 | pub(crate) fn new(text: &'a RichText) -> Self { 24 | let leaf = text.content.first_leaf(); 25 | Self { 26 | style_calc: text.init_styles.clone(), 27 | text, 28 | cursor: QueryResult { 29 | leaf, 30 | elem_index: 0, 31 | offset: 0, 32 | found: true, 33 | }, 34 | pending_return: None, 35 | done: false, 36 | end: None, 37 | } 38 | } 39 | 40 | pub(crate) fn new_range( 41 | text: &'a RichText, 42 | start: QueryResult, 43 | end: Option, 44 | style: StyleCalculator, 45 | ) -> Self { 46 | Self { 47 | style_calc: style, 48 | text, 49 | cursor: start, 50 | pending_return: None, 51 | done: false, 52 | end, 53 | } 54 | } 55 | } 56 | 57 | impl<'a> Iterator for Iter<'a> { 58 | type Item = Span; 59 | 60 | fn next(&mut self) -> Option { 61 | let mut pending_return = take(&mut self.pending_return); 62 | loop { 63 | if self.done { 64 | return pending_return; 65 | } 66 | 67 | let mut leaf = self.text.content.get_node(self.cursor.leaf); 68 | let mut is_end_leaf = self.end.map_or(false, |end| end.leaf == self.cursor.leaf); 69 | loop { 70 | while self.cursor.elem_index >= leaf.elements().len() { 71 | if is_end_leaf { 72 | self.done = true; 73 | return pending_return; 74 | } 75 | 76 | // index out of range, find next valid leaf node 77 | let next = if let Some(next) = 78 | self.text.content.next_same_level_node(self.cursor.leaf) 79 | { 80 | next 81 | } else { 82 | self.done = true; 83 | return pending_return; 84 | }; 85 | self.cursor.elem_index = 0; 86 | self.cursor.leaf = next; 87 | is_end_leaf = self.end.map_or(false, |end| end.leaf == self.cursor.leaf); 88 | leaf = self.text.content.get_node(self.cursor.leaf); 89 | } 90 | 91 | let elements = leaf.elements(); 92 | 93 | // skip zero len (deleted) elements 94 | while self.cursor.elem_index < elements.len() 95 | && elements[self.cursor.elem_index].content_len() == 0 96 | { 97 | self.style_calc 98 | .apply_start(&elements[self.cursor.elem_index].anchor_set); 99 | self.style_calc 100 | .apply_end(&elements[self.cursor.elem_index].anchor_set); 101 | self.cursor.elem_index += 1; 102 | } 103 | 104 | if self.cursor.elem_index < elements.len() { 105 | break; 106 | } 107 | } 108 | 109 | let leaf = leaf; 110 | let is_end_leaf = is_end_leaf; 111 | let elem = &leaf.elements()[self.cursor.elem_index]; 112 | let is_end_elem = is_end_leaf 113 | && self 114 | .end 115 | .map_or(false, |end| end.elem_index == self.cursor.elem_index); 116 | self.style_calc.apply_start(&elem.anchor_set); 117 | let annotations: FxHashMap<_, _> = self 118 | .style_calc 119 | .calc_styles(&self.text.ann) 120 | .filter_map(|x| { 121 | if x.behavior == Behavior::Delete { 122 | None 123 | } else { 124 | Some((x.type_.clone(), x.value.clone())) 125 | } 126 | }) 127 | .collect(); 128 | self.style_calc.apply_end(&elem.anchor_set); 129 | self.cursor.elem_index += 1; 130 | let ans = Span { 131 | insert: if is_end_elem { 132 | std::str::from_utf8(&elem.string[self.cursor.offset..self.end.unwrap().offset]) 133 | .unwrap() 134 | .to_string() 135 | } else { 136 | std::str::from_utf8(&elem.string[self.cursor.offset..]) 137 | .unwrap() 138 | .to_string() 139 | }, 140 | attributes: annotations, 141 | }; 142 | 143 | self.cursor.offset = 0; 144 | if is_end_elem { 145 | self.done = true; 146 | } 147 | 148 | if let Some(mut pending) = pending_return { 149 | if pending.can_merge(&ans) { 150 | pending.merge_right(&ans); 151 | pending_return = Some(pending); 152 | continue; 153 | } 154 | 155 | self.pending_return = Some(ans); 156 | return Some(pending); 157 | } else { 158 | pending_return = Some(ans); 159 | continue; 160 | } 161 | } 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /src/rich_text/op.rs: -------------------------------------------------------------------------------- 1 | use std::{ops::Deref, sync::Arc}; 2 | 3 | use append_only_bytes::BytesSlice; 4 | use fxhash::FxHashMap; 5 | use generic_btree::rle::{HasLength, Mergeable, Sliceable}; 6 | 7 | use crate::{Annotation, ClientID, Counter, Lamport, OpID}; 8 | 9 | use super::vv::VersionVector; 10 | 11 | #[derive(Debug, Clone, PartialEq, Eq)] 12 | pub struct Op { 13 | pub id: OpID, 14 | pub lamport: Lamport, 15 | pub content: OpContent, 16 | } 17 | 18 | #[derive(Debug, Clone, PartialEq, Eq)] 19 | pub enum OpContent { 20 | Ann(Arc), 21 | Text(TextInsertOp), 22 | Del(DeleteOp), 23 | } 24 | 25 | impl OpContent { 26 | pub fn new_insert(left: Option, right: Option, slice: BytesSlice) -> Self { 27 | OpContent::Text(TextInsertOp { 28 | text: slice, 29 | left, 30 | right, 31 | }) 32 | } 33 | 34 | pub fn new_delete(mut start: OpID, mut len: i32) -> Self { 35 | if len > 0 { 36 | // prefer negative del 37 | start = start.inc_i32(len - 1); 38 | len = -len; 39 | } 40 | OpContent::Del(DeleteOp { start, len }) 41 | } 42 | 43 | pub fn new_ann(ann: Arc) -> Self { 44 | OpContent::Ann(ann) 45 | } 46 | } 47 | 48 | #[derive(Clone)] 49 | pub struct TextInsertOp { 50 | pub text: BytesSlice, 51 | pub left: Option, 52 | pub right: Option, 53 | } 54 | 55 | impl PartialEq for TextInsertOp { 56 | fn eq(&self, other: &Self) -> bool { 57 | self.text.deref() == other.text.deref() 58 | && self.left == other.left 59 | && self.right == other.right 60 | } 61 | } 62 | 63 | impl Eq for TextInsertOp {} 64 | 65 | impl std::fmt::Debug for TextInsertOp { 66 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 67 | f.debug_struct("TextInsertOp") 68 | .field("text", &std::str::from_utf8(&self.text)) 69 | .field("left", &self.left) 70 | .field("right", &self.right) 71 | .finish() 72 | } 73 | } 74 | 75 | #[derive(Debug, Clone, Copy)] 76 | pub struct DeleteOp { 77 | pub start: OpID, 78 | // can be negative, so we can merge backward 79 | pub len: i32, 80 | } 81 | 82 | impl HasLength for DeleteOp { 83 | fn rle_len(&self) -> usize { 84 | self.len.unsigned_abs() as usize 85 | } 86 | } 87 | 88 | impl PartialEq for DeleteOp { 89 | fn eq(&self, other: &Self) -> bool { 90 | if self.start.client != other.start.client { 91 | return false; 92 | } 93 | 94 | let p = if other.len > 0 { 95 | self.positive() 96 | } else { 97 | self.negative() 98 | }; 99 | 100 | p.start.counter == other.start.counter && p.len == other.len 101 | } 102 | } 103 | 104 | impl Eq for DeleteOp {} 105 | 106 | impl DeleteOp { 107 | fn slice(&self, start: usize, end: usize) -> Self { 108 | let len = end - start; 109 | assert!(end <= self.len as usize); 110 | if self.len > 0 { 111 | Self { 112 | start: self.start.inc(start as Counter), 113 | len: len as i32, 114 | } 115 | } else { 116 | Self { 117 | start: self.start.inc_i32(-(start as i32)), 118 | len: -(len as i32), 119 | } 120 | } 121 | } 122 | 123 | fn next_counter(&self) -> (i32, Option) { 124 | ( 125 | self.start.counter as i32 + self.len, 126 | if self.len.abs() == 1 { 127 | if self.len > 0 { 128 | Some(self.start.counter as i32 - 1) 129 | } else { 130 | Some(self.start.counter as i32 + 1) 131 | } 132 | } else { 133 | None 134 | }, 135 | ) 136 | } 137 | 138 | pub fn positive(&self) -> DeleteOp { 139 | if self.len > 0 { 140 | *self 141 | } else { 142 | DeleteOp { 143 | start: self.start.inc_i32(self.len + 1), 144 | len: -self.len, 145 | } 146 | } 147 | } 148 | 149 | pub fn negative(&self) -> DeleteOp { 150 | if self.len < 0 { 151 | *self 152 | } else { 153 | DeleteOp { 154 | start: self.start.inc_i32(self.len - 1), 155 | len: -self.len, 156 | } 157 | } 158 | } 159 | 160 | fn direction(&self) -> u8 { 161 | if self.len.abs() == 1 { 162 | 0b11 163 | } else if self.len > 0 { 164 | 0b01 165 | } else { 166 | 0b10 167 | } 168 | } 169 | } 170 | 171 | impl Mergeable for DeleteOp { 172 | fn can_merge(&self, rhs: &Self) -> bool { 173 | if self.start.client != rhs.start.client || (self.direction() & rhs.direction()) == 0 { 174 | return false; 175 | } 176 | 177 | let (a, b) = self.next_counter(); 178 | a == rhs.start.counter as i32 || b == Some(rhs.start.counter as i32) 179 | } 180 | 181 | fn merge_right(&mut self, rhs: &Self) { 182 | if self.len > 1 { 183 | self.len += rhs.len.abs(); 184 | } else if self.len < -1 { 185 | self.len -= rhs.len.abs(); 186 | } else if self.len.abs() == 1 { 187 | if rhs.start.counter > self.start.counter { 188 | self.len = rhs.len.abs() + 1; 189 | } else { 190 | self.len = -rhs.len.abs() - 1; 191 | } 192 | } else { 193 | unreachable!() 194 | } 195 | } 196 | 197 | fn merge_left(&mut self, left: &Self) { 198 | let mut left = *left; 199 | left.merge_right(self); 200 | *self = left; 201 | } 202 | } 203 | 204 | impl HasLength for Op { 205 | fn rle_len(&self) -> usize { 206 | match &self.content { 207 | OpContent::Ann(_) => 1, 208 | OpContent::Text(text) => text.text.len(), 209 | OpContent::Del(del) => del.len.unsigned_abs() as usize, 210 | } 211 | } 212 | } 213 | 214 | impl Mergeable for Op { 215 | fn can_merge(&self, rhs: &Self) -> bool { 216 | self.id.client == rhs.id.client 217 | && self.id.counter + self.rle_len() as Counter == rhs.id.counter 218 | && self.lamport + self.rle_len() as Counter == rhs.lamport 219 | && match (&self.content, &rhs.content) { 220 | (OpContent::Text(left), OpContent::Text(right)) => { 221 | right.left == Some(self.id.inc(self.rle_len() as Counter - 1)) 222 | && right.right == left.right 223 | && left.text.can_merge(&right.text) 224 | } 225 | (OpContent::Del(a), OpContent::Del(b)) => a.can_merge(b), 226 | _ => false, 227 | } 228 | } 229 | 230 | fn merge_right(&mut self, rhs: &Self) { 231 | match (&mut self.content, &rhs.content) { 232 | (OpContent::Text(ins), OpContent::Text(ins2)) => { 233 | ins.text.try_merge(&ins2.text).unwrap(); 234 | } 235 | (OpContent::Del(del), OpContent::Del(del2)) => del.merge_right(del2), 236 | _ => unreachable!(), 237 | } 238 | } 239 | 240 | fn merge_left(&mut self, _left: &Self) { 241 | unimplemented!() 242 | } 243 | } 244 | 245 | impl Sliceable for Op { 246 | fn slice(&self, range: impl std::ops::RangeBounds) -> Self { 247 | let start = match range.start_bound() { 248 | std::ops::Bound::Included(i) => *i, 249 | std::ops::Bound::Excluded(i) => *i + 1, 250 | std::ops::Bound::Unbounded => 0, 251 | }; 252 | let end = match range.end_bound() { 253 | std::ops::Bound::Included(i) => *i + 1, 254 | std::ops::Bound::Excluded(i) => *i, 255 | std::ops::Bound::Unbounded => self.rle_len(), 256 | }; 257 | match &self.content { 258 | OpContent::Ann(a) => Op { 259 | id: self.id.inc(start as Counter), 260 | lamport: self.lamport + (start as Lamport), 261 | content: OpContent::Ann(a.clone()), 262 | }, 263 | OpContent::Text(text) => Op { 264 | id: self.id.inc(start as Counter), 265 | lamport: self.lamport + (start as Lamport), 266 | content: OpContent::Text(TextInsertOp { 267 | text: text.text.slice_clone(start..end), 268 | left: if start == 0 { 269 | text.left 270 | } else { 271 | Some(self.id.inc(start as Counter - 1)) 272 | }, 273 | right: if end == self.rle_len() { 274 | text.right 275 | } else { 276 | Some(self.id.inc(end as Counter)) 277 | }, 278 | }), 279 | }, 280 | OpContent::Del(del) => Op { 281 | id: self.id.inc(start as Counter), 282 | lamport: self.lamport + (start as Lamport), 283 | content: OpContent::Del(del.slice(start, end)), 284 | }, 285 | } 286 | } 287 | } 288 | 289 | pub struct OpStore { 290 | map: FxHashMap>, 291 | pub(crate) client: ClientID, 292 | next_lamport: Lamport, 293 | } 294 | 295 | impl std::fmt::Debug for OpStore { 296 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 297 | f.debug_struct("OpStore") 298 | .field("client", &self.client) 299 | .field("next_lamport", &self.next_lamport) 300 | .field("map", &self.map.len()) 301 | .finish()?; 302 | 303 | for (key, value) in self.map.iter() { 304 | f.write_str("\n")?; 305 | f.write_str(&key.to_string())?; 306 | for op in value.iter() { 307 | f.write_str("\n ")?; 308 | f.write_fmt(format_args!("{:?}", op))?; 309 | } 310 | } 311 | 312 | Ok(()) 313 | } 314 | } 315 | 316 | impl OpStore { 317 | pub fn new(client: ClientID) -> Self { 318 | Self { 319 | map: Default::default(), 320 | client, 321 | next_lamport: 0, 322 | } 323 | } 324 | 325 | pub fn insert_local(&mut self, content: OpContent) -> &Op { 326 | let op = Op { 327 | id: self.next_id(), 328 | lamport: self.next_lamport, 329 | content, 330 | }; 331 | self.next_lamport += op.rle_len() as Lamport; 332 | self.insert(op) 333 | } 334 | 335 | pub fn insert(&mut self, op: Op) -> &Op { 336 | if op.lamport + op.rle_len() as Lamport >= self.next_lamport { 337 | self.next_lamport = op.lamport + op.rle_len() as Lamport; 338 | } 339 | let vec = self.map.entry(op.id.client).or_default(); 340 | let mut done = false; 341 | if let Some(last) = vec.last_mut() { 342 | if last.can_merge(&op) { 343 | last.merge_right(&op); 344 | done = true; 345 | } 346 | } 347 | 348 | if done { 349 | vec.last().as_ref().unwrap() 350 | } else { 351 | vec.push(op); 352 | vec.last().as_ref().unwrap() 353 | } 354 | } 355 | 356 | pub fn export(&self, other_vv: &VersionVector) -> FxHashMap> { 357 | let mut ans: FxHashMap> = FxHashMap::default(); 358 | for (client, vec) in self.map.iter() { 359 | let target_counter = other_vv.vv.get(client).unwrap_or(&0); 360 | if *target_counter 361 | >= vec 362 | .last() 363 | .map(|x| x.id.counter + x.rle_len() as Counter) 364 | .unwrap_or(0) 365 | { 366 | continue; 367 | } 368 | 369 | let mut i = match vec.binary_search_by_key(target_counter, |op| op.id.counter) { 370 | Ok(i) => i, 371 | Err(i) => i.max(1) - 1, 372 | }; 373 | if *target_counter >= vec[i].id.counter + vec[i].rle_len() as Counter { 374 | i += 1; 375 | } 376 | let vec = if vec[i].id.counter < *target_counter { 377 | let mut new_vec: Vec = Vec::with_capacity(vec.len() - i); 378 | new_vec.push(vec[i].slice(*target_counter as usize - vec[i].id.counter as usize..)); 379 | new_vec.extend_from_slice(&vec[i + 1..]); 380 | new_vec 381 | } else { 382 | assert!(vec[i].id.counter == *target_counter); 383 | vec[i..].to_vec() 384 | }; 385 | ans.insert(*client, vec); 386 | } 387 | 388 | ans 389 | } 390 | 391 | pub fn vv(&self) -> VersionVector { 392 | let mut ans = VersionVector::default(); 393 | for (client, vec) in self.map.iter() { 394 | if let Some(last) = vec.last() { 395 | ans.vv 396 | .insert(*client, last.id.counter + last.rle_len() as Counter); 397 | } 398 | } 399 | 400 | ans 401 | } 402 | 403 | pub fn next_id(&self) -> OpID { 404 | OpID { 405 | client: self.client, 406 | counter: self 407 | .map 408 | .get(&self.client) 409 | .and_then(|v| v.last().map(|x| x.id.counter + x.rle_len() as Counter)) 410 | .unwrap_or(0), 411 | } 412 | } 413 | 414 | pub fn can_apply(&self, op: &Op) -> CanApply { 415 | let Some(vec) = self.map.get(&op.id.client) else { 416 | if op.id.counter == 0 { 417 | return CanApply::Yes; 418 | } else { 419 | return CanApply::Pending; 420 | } 421 | }; 422 | let end = vec 423 | .last() 424 | .map(|x| x.id.counter + x.rle_len() as Counter) 425 | .unwrap_or(0); 426 | if end == op.id.counter { 427 | return CanApply::Yes; 428 | } 429 | if end < op.id.counter { 430 | return CanApply::Pending; 431 | } 432 | if end >= op.id.counter + op.rle_len() as Counter { 433 | return CanApply::Seen; 434 | } 435 | 436 | CanApply::Trim(end - op.id.counter) 437 | } 438 | 439 | #[inline(always)] 440 | pub fn next_lamport(&self) -> u32 { 441 | self.next_lamport 442 | } 443 | 444 | pub fn op_len(&self) -> usize { 445 | self.map.iter().map(|x| x.1.len()).sum() 446 | } 447 | } 448 | 449 | pub enum CanApply { 450 | Yes, 451 | Trim(Counter), 452 | Pending, 453 | Seen, 454 | } 455 | 456 | #[cfg(test)] 457 | mod test { 458 | use generic_btree::rle::Mergeable; 459 | 460 | use crate::OpID; 461 | 462 | use super::DeleteOp; 463 | 464 | #[test] 465 | fn del_merge() { 466 | let a = DeleteOp { 467 | start: OpID::new(1, 1), 468 | len: 2, 469 | }; 470 | let b = DeleteOp { 471 | start: OpID::new(1, 3), 472 | len: -1, 473 | }; 474 | assert!(a.can_merge(&b)) 475 | } 476 | } 477 | -------------------------------------------------------------------------------- /src/rich_text/rich_tree/query.rs: -------------------------------------------------------------------------------- 1 | use generic_btree::{BTreeTrait, FindResult, Query}; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | use crate::rich_text::{ 5 | ann::StyleCalculator, 6 | rich_tree::utf16::{line_start_to_utf8, utf16_to_utf8}, 7 | }; 8 | 9 | use super::*; 10 | 11 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] 12 | pub enum IndexType { 13 | Utf8, 14 | Utf16, 15 | } 16 | 17 | pub(crate) struct IndexFinderWithStyles { 18 | left: usize, 19 | pub(crate) style_calculator: StyleCalculator, 20 | index_type: IndexType, 21 | } 22 | 23 | pub(crate) struct IndexFinder { 24 | left: usize, 25 | index_type: IndexType, 26 | } 27 | 28 | pub(crate) struct LineStartFinder { 29 | left: usize, 30 | pub(crate) style_calculator: StyleCalculator, 31 | } 32 | 33 | struct AnnotationFinderStart { 34 | target: AnnIdx, 35 | visited_len: usize, 36 | } 37 | 38 | struct AnnotationFinderEnd { 39 | target: AnnIdx, 40 | visited_len: usize, 41 | } 42 | 43 | impl Query for IndexFinder { 44 | type QueryArg = (usize, IndexType); 45 | 46 | fn init(target: &Self::QueryArg) -> Self { 47 | IndexFinder { 48 | left: target.0, 49 | index_type: target.1, 50 | } 51 | } 52 | 53 | /// should prefer zero len element 54 | fn find_node( 55 | &mut self, 56 | _: &Self::QueryArg, 57 | child_caches: &[generic_btree::Child], 58 | ) -> generic_btree::FindResult { 59 | if child_caches.is_empty() { 60 | return FindResult::new_missing(0, 0); 61 | } 62 | 63 | let mut last_left = self.left; 64 | for (i, cache) in child_caches.iter().enumerate() { 65 | let cache_len = match self.index_type { 66 | IndexType::Utf8 => cache.cache.len, 67 | IndexType::Utf16 => cache.cache.utf16_len, 68 | }; 69 | // prefer the end of an element 70 | if self.left >= cache_len as usize { 71 | last_left = self.left; 72 | self.left -= cache_len as usize; 73 | } else { 74 | return FindResult::new_found(i, self.left); 75 | } 76 | } 77 | 78 | self.left = last_left; 79 | FindResult::new_missing(child_caches.len() - 1, last_left) 80 | } 81 | 82 | /// should prefer zero len element 83 | fn find_element(&mut self, _: &Self::QueryArg, elements: &[Elem]) -> generic_btree::FindResult { 84 | if elements.is_empty() { 85 | return FindResult::new_missing(0, 0); 86 | } 87 | 88 | let mut last_left = self.left; 89 | for (i, cache) in elements.iter().enumerate() { 90 | let len = match self.index_type { 91 | IndexType::Utf8 => cache.content_len(), 92 | IndexType::Utf16 => { 93 | if cache.status.is_dead() { 94 | 0 95 | } else { 96 | cache.utf16_len as usize 97 | } 98 | } 99 | }; 100 | // prefer the end of an element 101 | if self.left >= len { 102 | // use content len here, because we need to skip deleted/future spans 103 | last_left = self.left; 104 | self.left -= len; 105 | } else { 106 | return FindResult::new_found( 107 | i, 108 | reset_left_to_utf8(self.left, self.index_type, cache), 109 | ); 110 | } 111 | } 112 | 113 | FindResult::new_missing( 114 | elements.len() - 1, 115 | reset_left_to_utf8(last_left, self.index_type, elements.last().unwrap()), 116 | ) 117 | } 118 | } 119 | 120 | impl Query for LineStartFinder { 121 | type QueryArg = usize; 122 | 123 | fn init(target: &Self::QueryArg) -> Self { 124 | LineStartFinder { 125 | left: *target, 126 | style_calculator: StyleCalculator::default(), 127 | } 128 | } 129 | 130 | fn find_node( 131 | &mut self, 132 | _: &Self::QueryArg, 133 | child_caches: &[generic_btree::Child], 134 | ) -> generic_btree::FindResult { 135 | if self.left == 0 { 136 | return FindResult::new_found(0, 0); 137 | } 138 | 139 | if child_caches.is_empty() { 140 | return FindResult::new_missing(0, 0); 141 | } 142 | 143 | for (i, cache) in child_caches.iter().enumerate() { 144 | self.style_calculator 145 | .apply_node_start(&cache.cache.anchor_set); 146 | if self.left > cache.cache.line_breaks as usize { 147 | self.left -= cache.cache.line_breaks as usize; 148 | } else { 149 | return FindResult::new_found(i, self.left); 150 | } 151 | self.style_calculator 152 | .apply_node_end(&cache.cache.anchor_set); 153 | } 154 | 155 | FindResult::new_missing(child_caches.len() - 1, self.left) 156 | } 157 | 158 | fn find_element(&mut self, _: &Self::QueryArg, elements: &[Elem]) -> generic_btree::FindResult { 159 | if self.left == 0 { 160 | return FindResult::new_found(0, 0); 161 | } 162 | 163 | if elements.is_empty() { 164 | return FindResult::new_missing(0, 0); 165 | } 166 | 167 | for (i, cache) in elements.iter().enumerate() { 168 | self.style_calculator.apply_start(&cache.anchor_set); 169 | if cache.is_dead() { 170 | self.style_calculator.apply_end(&cache.anchor_set); 171 | continue; 172 | } 173 | 174 | if self.left > cache.line_breaks as usize { 175 | self.left -= cache.line_breaks as usize; 176 | } else { 177 | return FindResult::new_found( 178 | i, 179 | line_start_to_utf8(&cache.string, self.left).unwrap(), 180 | ); 181 | } 182 | self.style_calculator.apply_end(&cache.anchor_set); 183 | } 184 | 185 | FindResult::new_missing(elements.len() - 1, elements.last().unwrap().atom_len()) 186 | } 187 | } 188 | 189 | type TreeTrait = RichTreeTrait; 190 | 191 | impl Query for IndexFinderWithStyles { 192 | type QueryArg = (usize, IndexType); 193 | 194 | fn init(target: &Self::QueryArg) -> Self { 195 | IndexFinderWithStyles { 196 | left: target.0, 197 | style_calculator: StyleCalculator::default(), 198 | index_type: target.1, 199 | } 200 | } 201 | 202 | fn find_node( 203 | &mut self, 204 | _: &Self::QueryArg, 205 | child_caches: &[generic_btree::Child], 206 | ) -> generic_btree::FindResult { 207 | if child_caches.is_empty() { 208 | return FindResult::new_missing(0, self.left); 209 | } 210 | 211 | let mut last_left = self.left; 212 | for (i, cache) in child_caches.iter().enumerate() { 213 | let cache_len = match self.index_type { 214 | IndexType::Utf8 => cache.cache.len, 215 | IndexType::Utf16 => cache.cache.utf16_len, 216 | }; 217 | if self.left >= cache_len as usize { 218 | last_left = self.left; 219 | self.left -= cache_len as usize; 220 | } else { 221 | return FindResult::new_found(i, self.left); 222 | } 223 | 224 | self.style_calculator 225 | .apply_node_start(&cache.cache.anchor_set); 226 | self.style_calculator 227 | .apply_node_end(&cache.cache.anchor_set); 228 | } 229 | 230 | self.left = last_left; 231 | FindResult::new_missing(child_caches.len() - 1, last_left) 232 | } 233 | 234 | fn find_element(&mut self, _: &Self::QueryArg, elements: &[Elem]) -> generic_btree::FindResult { 235 | if elements.is_empty() { 236 | return FindResult::new_missing(0, self.left); 237 | } 238 | 239 | let mut last_left = self.left; 240 | for (i, cache) in elements.iter().enumerate() { 241 | let len = match self.index_type { 242 | IndexType::Utf8 => cache.content_len(), 243 | IndexType::Utf16 => { 244 | if cache.status.is_dead() { 245 | 0 246 | } else { 247 | cache.utf16_len as usize 248 | } 249 | } 250 | }; 251 | self.style_calculator.apply_start(&cache.anchor_set); 252 | self.style_calculator.cache_end(&cache.anchor_set); 253 | if self.left >= len { 254 | last_left = self.left; 255 | self.left -= len; 256 | } else { 257 | return FindResult::new_found( 258 | i, 259 | reset_left_to_utf8(self.left, self.index_type, cache), 260 | ); 261 | } 262 | 263 | self.style_calculator.commit_cache(); 264 | } 265 | 266 | FindResult::new_missing( 267 | elements.len() - 1, 268 | reset_left_to_utf8(last_left, self.index_type, elements.last().unwrap()), 269 | ) 270 | } 271 | } 272 | 273 | fn reset_left_to_utf8(left: usize, index_type: IndexType, element: &Elem) -> usize { 274 | if left == 0 { 275 | return left; 276 | } 277 | 278 | match index_type { 279 | IndexType::Utf8 => left, 280 | IndexType::Utf16 => { 281 | assert!(element.utf16_len as usize >= left); 282 | if element.utf16_len as usize == left { 283 | return element.atom_len(); 284 | } 285 | 286 | utf16_to_utf8(&element.string, left) 287 | } 288 | } 289 | } 290 | 291 | impl Query for AnnotationFinderStart { 292 | type QueryArg = AnnIdx; 293 | 294 | fn init(target: &Self::QueryArg) -> Self { 295 | Self { 296 | target: *target, 297 | visited_len: 0, 298 | } 299 | } 300 | 301 | fn find_node( 302 | &mut self, 303 | _: &Self::QueryArg, 304 | child_caches: &[generic_btree::Child], 305 | ) -> FindResult { 306 | for (i, cache) in child_caches.iter().enumerate() { 307 | if cache.cache.anchor_set.contains_start(self.target) { 308 | return FindResult::new_found(i, 0); 309 | } 310 | self.visited_len += cache.cache.len as usize; 311 | } 312 | 313 | FindResult::new_missing(0, 0) 314 | } 315 | 316 | fn find_element( 317 | &mut self, 318 | _: &Self::QueryArg, 319 | elements: &[::Elem], 320 | ) -> FindResult { 321 | for (i, cache) in elements.iter().enumerate() { 322 | let (contains_start, inclusive) = cache.anchor_set.contains_start(self.target); 323 | if contains_start { 324 | if !inclusive { 325 | self.visited_len += cache.content_len(); 326 | } 327 | 328 | return FindResult::new_found(i, 0); 329 | } 330 | self.visited_len += cache.content_len(); 331 | } 332 | 333 | FindResult::new_missing(0, 0) 334 | } 335 | } 336 | 337 | impl Query for AnnotationFinderEnd { 338 | type QueryArg = AnnIdx; 339 | 340 | fn init(target: &Self::QueryArg) -> Self { 341 | Self { 342 | target: *target, 343 | visited_len: 0, 344 | } 345 | } 346 | 347 | fn find_node( 348 | &mut self, 349 | _: &Self::QueryArg, 350 | child_caches: &[generic_btree::Child], 351 | ) -> FindResult { 352 | for (i, cache) in child_caches.iter().enumerate().rev() { 353 | if cache.cache.anchor_set.contains_end(self.target) { 354 | return FindResult::new_found(i, cache.cache.len as usize); 355 | } 356 | self.visited_len += cache.cache.len as usize; 357 | } 358 | 359 | FindResult::new_missing(0, 0) 360 | } 361 | 362 | fn find_element( 363 | &mut self, 364 | _: &Self::QueryArg, 365 | elements: &[::Elem], 366 | ) -> FindResult { 367 | for (i, cache) in elements.iter().enumerate().rev() { 368 | let (contains_end, inclusive) = cache.anchor_set.contains_end(self.target); 369 | if contains_end { 370 | if !inclusive { 371 | self.visited_len += cache.content_len(); 372 | } 373 | 374 | return FindResult::new_found(i, cache.content_len()); 375 | } 376 | self.visited_len += cache.content_len(); 377 | } 378 | 379 | FindResult::new_missing(0, 0) 380 | } 381 | } 382 | -------------------------------------------------------------------------------- /src/rich_text/rich_tree/rich_tree_btree_impl.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use generic_btree::{rle, BTreeTrait}; 3 | 4 | #[derive(Debug, Clone)] 5 | pub(crate) struct RichTreeTrait; 6 | 7 | impl BTreeTrait for RichTreeTrait { 8 | type Elem = Elem; 9 | 10 | type Cache = Cache; 11 | 12 | type CacheDiff = CacheDiff; 13 | 14 | const MAX_LEN: usize = 16; 15 | 16 | fn calc_cache_internal( 17 | cache: &mut Self::Cache, 18 | caches: &[generic_btree::Child], 19 | diff: Option, 20 | ) -> Option { 21 | match diff { 22 | Some(diff) => { 23 | cache.apply_diff(&diff); 24 | Some(diff) 25 | } 26 | None => { 27 | let mut len = 0; 28 | let mut utf16_len = 0; 29 | let mut line_breaks = 0; 30 | let mut anchor_set = CacheAnchorSet::default(); 31 | for child in caches.iter() { 32 | len += child.cache.len; 33 | utf16_len += child.cache.utf16_len; 34 | line_breaks += child.cache.line_breaks; 35 | anchor_set.union_(&child.cache.anchor_set); 36 | } 37 | 38 | let anchor_diff = anchor_set.calc_diff(&cache.anchor_set); 39 | let diff = CacheDiff { 40 | anchor_diff, 41 | len_diff: len as isize - cache.len as isize, 42 | utf16_len_diff: utf16_len as isize - cache.utf16_len as isize, 43 | line_break_diff: line_breaks as isize - cache.line_breaks as isize, 44 | }; 45 | 46 | cache.len = len; 47 | cache.utf16_len = utf16_len; 48 | cache.line_breaks = line_breaks; 49 | Some(diff) 50 | } 51 | } 52 | } 53 | 54 | fn calc_cache_leaf( 55 | cache: &mut Self::Cache, 56 | caches: &[Self::Elem], 57 | diff: Option, 58 | ) -> Self::CacheDiff { 59 | match diff { 60 | Some(diff) => { 61 | cache.apply_diff(&diff); 62 | diff 63 | } 64 | None => { 65 | let mut len = 0; 66 | let mut utf16_len = 0; 67 | let mut line_breaks = 0; 68 | let mut anchor_set = CacheAnchorSet::default(); 69 | for child in caches.iter() { 70 | if !child.is_dead() { 71 | len += child.string.len(); 72 | utf16_len += child.utf16_len; 73 | line_breaks += child.line_breaks; 74 | } 75 | anchor_set.union_elem_set(&child.anchor_set); 76 | } 77 | 78 | let anchor_diff = cache.anchor_set.calc_diff(&anchor_set); 79 | let diff = CacheDiff { 80 | anchor_diff, 81 | len_diff: len as isize - cache.len as isize, 82 | utf16_len_diff: utf16_len as isize - cache.utf16_len as isize, 83 | line_break_diff: line_breaks as isize - cache.line_breaks as isize, 84 | }; 85 | cache.len = len as u32; 86 | cache.utf16_len = utf16_len; 87 | cache.line_breaks = line_breaks; 88 | diff 89 | } 90 | } 91 | } 92 | 93 | fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { 94 | diff1.anchor_diff.merge(&diff2.anchor_diff); 95 | diff1.len_diff += diff2.len_diff; 96 | diff1.utf16_len_diff += diff2.utf16_len_diff; 97 | diff1.line_break_diff += diff2.line_break_diff; 98 | } 99 | 100 | fn insert( 101 | elements: &mut generic_btree::HeapVec, 102 | index: usize, 103 | offset: usize, 104 | elem: Self::Elem, 105 | ) { 106 | rle::insert_with_split(elements, index, offset, elem) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/rich_text/rich_tree/utf16.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | pub struct Utf16LenAndLineBreaks { 4 | pub utf16: u32, 5 | pub line_breaks: u32, 6 | } 7 | 8 | pub fn get_utf16_len(str: &str) -> usize { 9 | if str.is_empty() { 10 | return 0; 11 | } 12 | 13 | let iter = encode_utf16(str); 14 | iter.count() 15 | } 16 | 17 | #[inline(always)] 18 | pub fn get_utf16_len_and_line_breaks(bytes: &[u8]) -> Utf16LenAndLineBreaks { 19 | if bytes.is_empty() { 20 | return Utf16LenAndLineBreaks { 21 | line_breaks: 0, 22 | utf16: 0, 23 | }; 24 | } 25 | 26 | let str = bytes_to_str(bytes); 27 | let mut iter = encode_utf16(str); 28 | let mut utf16 = 0; 29 | for _ in iter.by_ref() { 30 | utf16 += 1; 31 | } 32 | 33 | Utf16LenAndLineBreaks { 34 | utf16, 35 | line_breaks: iter.line_breaks, 36 | } 37 | } 38 | 39 | pub fn utf16_to_utf8(bytes: &[u8], utf16_index: usize) -> usize { 40 | if utf16_index == 0 { 41 | return 0; 42 | } 43 | 44 | let str = bytes_to_str(bytes); 45 | let mut iter = encode_utf16(str); 46 | for _ in 0..utf16_index { 47 | iter.next(); 48 | } 49 | 50 | iter.visited 51 | } 52 | 53 | /// get the index of nth line start in bytes (in utf8) 54 | /// 55 | /// if n exceed the number of lines in bytes, return None 56 | pub fn line_start_to_utf8(bytes: &BytesSlice, n: usize) -> Option { 57 | if n == 0 { 58 | return Some(0); 59 | } 60 | 61 | let str = bytes_to_str(bytes); 62 | let mut visited_bytes = 0; 63 | let mut iter_line_breaks = 0; 64 | for c in str.chars() { 65 | if c.eq(&'\n') { 66 | iter_line_breaks += 1; 67 | if iter_line_breaks == n { 68 | return Some(visited_bytes + 1); 69 | } 70 | } 71 | 72 | visited_bytes += c.len_utf8(); 73 | } 74 | 75 | None 76 | } 77 | 78 | #[inline(always)] 79 | pub fn bytes_to_str(bytes: &[u8]) -> &str { 80 | #[allow(unsafe_code)] 81 | // SAFETY: we are sure the range is valid utf8 82 | let str = unsafe { std::str::from_utf8_unchecked(&bytes[..]) }; 83 | str 84 | } 85 | 86 | fn encode_utf16(s: &str) -> EncodeUtf16 { 87 | EncodeUtf16 { 88 | chars: s.chars(), 89 | extra: 0, 90 | visited: 0, 91 | line_breaks: 0, 92 | } 93 | } 94 | 95 | // from std 96 | #[derive(Clone)] 97 | pub struct EncodeUtf16<'a> { 98 | chars: Chars<'a>, 99 | extra: u16, 100 | visited: usize, 101 | line_breaks: u32, 102 | } 103 | 104 | impl fmt::Debug for EncodeUtf16<'_> { 105 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 106 | f.debug_struct("EncodeUtf16").finish_non_exhaustive() 107 | } 108 | } 109 | 110 | impl<'a> Iterator for EncodeUtf16<'a> { 111 | type Item = u16; 112 | 113 | #[inline] 114 | fn next(&mut self) -> Option { 115 | if self.extra != 0 { 116 | let tmp = self.extra; 117 | self.extra = 0; 118 | return Some(tmp); 119 | } 120 | 121 | let mut buf = [0; 2]; 122 | self.chars.next().map(|ch| { 123 | self.visited += ch.len_utf8(); 124 | self.line_breaks += if ch.eq(&'\n') { 1 } else { 0 }; 125 | let n = ch.encode_utf16(&mut buf).len(); 126 | if n == 2 { 127 | self.extra = buf[1]; 128 | } 129 | buf[0] 130 | }) 131 | } 132 | 133 | #[inline] 134 | fn size_hint(&self) -> (usize, Option) { 135 | let (low, high) = self.chars.size_hint(); 136 | // every char gets either one u16 or two u16, 137 | // so this iterator is between 1 or 2 times as 138 | // long as the underlying iterator. 139 | (low, high.and_then(|n| n.checked_mul(2))) 140 | } 141 | } 142 | 143 | #[cfg(test)] 144 | mod test { 145 | use super::line_start_to_utf8; 146 | 147 | #[test] 148 | fn line_breaks() { 149 | use append_only_bytes::AppendOnlyBytes; 150 | let mut bytes = AppendOnlyBytes::new(); 151 | bytes.push_str("abc\ndragon\nzz"); 152 | assert_eq!(bytes.len(), 13); 153 | assert_eq!(line_start_to_utf8(&bytes.slice(..), 0).unwrap(), 0); 154 | assert_eq!(line_start_to_utf8(&bytes.slice(..), 1).unwrap(), 4); 155 | assert_eq!(line_start_to_utf8(&bytes.slice(..), 2).unwrap(), 11); 156 | assert!(line_start_to_utf8(&bytes.slice(..), 3).is_none()); 157 | assert_eq!(line_start_to_utf8(&bytes.slice(0..0), 0).unwrap(), 0); 158 | assert_eq!(line_start_to_utf8(&bytes.slice(0..0), 1), None); 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/rich_text/test_utils.rs: -------------------------------------------------------------------------------- 1 | use std::{cell::RefCell, rc::Rc}; 2 | 3 | use crate::{test_utils::AnnotationType, InternalString}; 4 | 5 | use super::*; 6 | use arbitrary::Arbitrary; 7 | 8 | mod fuzz_line_breaks; 9 | pub use fuzz_line_breaks::{fuzzing_line_break, Action as LineBreakFuzzAction}; 10 | 11 | pub struct Actor { 12 | pub text: RichText, 13 | } 14 | 15 | #[derive(Arbitrary, Clone, Debug, Copy)] 16 | pub enum Action { 17 | Insert { 18 | actor: u8, 19 | pos: u8, 20 | content: u16, 21 | }, 22 | Delete { 23 | actor: u8, 24 | pos: u8, 25 | len: u8, 26 | }, 27 | Annotate { 28 | actor: u8, 29 | pos: u8, 30 | len: u8, 31 | annotation: AnnotationType, 32 | }, 33 | Sync(u8, u8), 34 | } 35 | 36 | pub fn preprocess_action(actors: &[Actor], action: &mut Action) { 37 | match action { 38 | Action::Insert { 39 | actor, 40 | pos, 41 | content: _, 42 | } => { 43 | *actor %= actors.len() as u8; 44 | *pos = (*pos as usize % (actors[*actor as usize].len() + 1)) as u8; 45 | } 46 | Action::Delete { actor, pos, len } => { 47 | *actor %= actors.len() as u8; 48 | *pos = (*pos as usize % (actors[*actor as usize].len() + 1)) as u8; 49 | *len = (*len).min(10); 50 | *len %= (actors[*actor as usize].len().max(*pos as usize + 1) - *pos as usize) 51 | .min(255) 52 | .max(1) as u8; 53 | } 54 | Action::Annotate { 55 | actor, 56 | pos, 57 | len, 58 | annotation: _, 59 | } => { 60 | *actor %= actors.len() as u8; 61 | *pos = (*pos as usize % (actors[*actor as usize].len() + 1)) as u8; 62 | *len = (*len).min(10); 63 | *len %= (actors[*actor as usize].len().max(*pos as usize + 1) - *pos as usize) 64 | .min(255) 65 | .max(1) as u8; 66 | } 67 | Action::Sync(a, b) => { 68 | *a %= actors.len() as u8; 69 | *b %= actors.len() as u8; 70 | if b == a { 71 | *b = (*a + 1) % actors.len() as u8; 72 | } 73 | } 74 | } 75 | } 76 | 77 | pub fn preprocess_action_utf16(actors: &[Actor], action: &mut Action) { 78 | match action { 79 | Action::Insert { 80 | actor, 81 | pos, 82 | content: _, 83 | } => { 84 | *actor %= actors.len() as u8; 85 | *pos = (*pos as usize % (actors[*actor as usize].len_utf16() + 1)) as u8; 86 | } 87 | Action::Delete { actor, pos, len } => { 88 | *actor %= actors.len() as u8; 89 | *pos = (*pos as usize % (actors[*actor as usize].len_utf16() + 1)) as u8; 90 | *len = (*len).min(10); 91 | *len %= (actors[*actor as usize].len_utf16().max(*pos as usize + 1) - *pos as usize) 92 | .min(255) 93 | .max(1) as u8; 94 | } 95 | Action::Annotate { 96 | actor, 97 | pos, 98 | len, 99 | annotation: _, 100 | } => { 101 | *actor %= actors.len() as u8; 102 | *pos = (*pos as usize % (actors[*actor as usize].len_utf16() + 1)) as u8; 103 | *len = (*len).min(10); 104 | *len %= (actors[*actor as usize].len_utf16().max(*pos as usize + 1) - *pos as usize) 105 | .min(255) 106 | .max(1) as u8; 107 | } 108 | Action::Sync(a, b) => { 109 | *a %= actors.len() as u8; 110 | *b %= actors.len() as u8; 111 | if b == a { 112 | *b = (*a + 1) % actors.len() as u8; 113 | } 114 | } 115 | } 116 | } 117 | 118 | pub fn apply_action(actors: &mut [Actor], action: Action) { 119 | match action { 120 | Action::Insert { 121 | actor, 122 | pos, 123 | content, 124 | } => { 125 | actors[actor as usize].insert(pos as usize, content.to_string().as_str()); 126 | // actors[actor as usize].check(); 127 | } 128 | Action::Delete { actor, pos, len } => { 129 | if len == 0 { 130 | return; 131 | } 132 | 133 | actors[actor as usize].delete(pos as usize, len as usize); 134 | // actors[actor as usize].check(); 135 | } 136 | Action::Annotate { 137 | actor, 138 | pos, 139 | len, 140 | annotation, 141 | } => { 142 | if len == 0 { 143 | return; 144 | } 145 | 146 | actors[actor as usize].annotate(pos as usize..pos as usize + len as usize, annotation); 147 | // actors[actor as usize].check(); 148 | } 149 | Action::Sync(a, b) => { 150 | let (a, b) = arref::array_mut_ref!(actors, [a as usize, b as usize]); 151 | a.merge(b); 152 | a.text.debug_log(true); 153 | // a.check(); 154 | } 155 | } 156 | } 157 | 158 | pub fn apply_action_utf16(actors: &mut [Actor], action: Action) { 159 | match action { 160 | Action::Insert { 161 | actor, 162 | pos, 163 | content, 164 | } => { 165 | actors[actor as usize].insert_utf16(pos as usize, content.to_string().as_str()); 166 | // actors[actor as usize].check(); 167 | } 168 | Action::Delete { actor, pos, len } => { 169 | if len == 0 { 170 | return; 171 | } 172 | 173 | actors[actor as usize].delete_utf16(pos as usize, len as usize); 174 | // actors[actor as usize].check(); 175 | } 176 | Action::Annotate { 177 | actor, 178 | pos, 179 | len, 180 | annotation, 181 | } => { 182 | if len == 0 { 183 | return; 184 | } 185 | 186 | actors[actor as usize] 187 | .annotate_utf16(pos as usize..pos as usize + len as usize, annotation); 188 | // actors[actor as usize].check(); 189 | } 190 | Action::Sync(a, b) => { 191 | let (a, b) = arref::array_mut_ref!(actors, [a as usize, b as usize]); 192 | a.merge(b); 193 | // a.check(); 194 | } 195 | } 196 | } 197 | 198 | pub fn fuzzing(actor_num: usize, actions: Vec) { 199 | let mut actors = vec![]; 200 | for i in 0..actor_num { 201 | actors.push(Actor::new(i)); 202 | } 203 | 204 | for mut action in actions { 205 | preprocess_action(&actors, &mut action); 206 | // println!("{:?},", &action); 207 | debug_log::group!("{:?},", &action); 208 | apply_action(&mut actors, action); 209 | debug_log::group_end!(); 210 | } 211 | 212 | for i in 0..actors.len() { 213 | for j in (i + 1)..actors.len() { 214 | let (a, b) = arref::array_mut_ref!(&mut actors, [i, j]); 215 | debug_log::group!("merge {i}<-{j}"); 216 | a.merge(b); 217 | debug_log::group_end!(); 218 | debug_log::group!("merge {i}->{j}"); 219 | b.merge(a); 220 | assert_eq!(a.text.get_spans(), b.text.get_spans()); 221 | debug_log::group_end!(); 222 | } 223 | } 224 | } 225 | 226 | pub fn fuzzing_utf16(actor_num: usize, actions: Vec) { 227 | let mut actors = vec![]; 228 | let followers = vec![ 229 | Rc::new(RefCell::new(String::new())), 230 | Rc::new(RefCell::new(String::new())), 231 | ]; 232 | for i in 0..actor_num { 233 | if i <= 1 { 234 | let mut actor = Actor::new(i); 235 | let f = followers[i].clone(); 236 | actor.text.observe(Box::new(move |event| { 237 | let mut index = 0; 238 | for op in event.ops.iter() { 239 | match op { 240 | crate::rich_text::delta::DeltaItem::Retain { retain, .. } => { 241 | index += *retain; 242 | } 243 | crate::rich_text::delta::DeltaItem::Insert { insert, .. } => { 244 | f.borrow_mut().insert_str(index, insert); 245 | index += insert.len(); 246 | } 247 | crate::rich_text::delta::DeltaItem::Delete { delete } => { 248 | f.borrow_mut().drain(index..index + *delete); 249 | } 250 | } 251 | } 252 | })); 253 | 254 | actors.push(actor); 255 | } else { 256 | actors.push(Actor::new(i)); 257 | } 258 | } 259 | 260 | for mut action in actions { 261 | preprocess_action_utf16(&actors, &mut action); 262 | // println!("{:?},", &action); 263 | debug_log::group!("{:?},", &action); 264 | apply_action_utf16(&mut actors, action); 265 | debug_log::group_end!(); 266 | } 267 | 268 | for i in 0..actors.len() { 269 | for j in (i + 1)..actors.len() { 270 | let (a, b) = arref::array_mut_ref!(&mut actors, [i, j]); 271 | debug_log::group!("merge {i}<-{j}"); 272 | a.merge(b); 273 | debug_log::group_end!(); 274 | debug_log::group!("merge {i}->{j}"); 275 | b.merge(a); 276 | debug_log::group_end!(); 277 | assert_eq!(a.text.get_spans(), b.text.get_spans()); 278 | if i <= 1 { 279 | assert_eq!(a.text.to_string(), followers[i].borrow().to_string()); 280 | } 281 | } 282 | } 283 | } 284 | 285 | pub fn fuzzing_match_str(actions: Vec) { 286 | let word_choices: [InternalString; 8] = [ 287 | "a".into(), 288 | "b".into(), 289 | "c".into(), 290 | "d".into(), 291 | "一".into(), 292 | "二".into(), 293 | "三".into(), 294 | "四".into(), 295 | ]; 296 | let mut actor = Actor::new(1); 297 | let mut s: Vec = vec![]; 298 | for action in actions { 299 | if matches!(action, Action::Sync(_, _) | Action::Annotate { .. }) { 300 | continue; 301 | } 302 | 303 | match action { 304 | Action::Insert { pos, content, .. } => { 305 | let mut pos = pos as usize; 306 | if s.is_empty() { 307 | pos = 0; 308 | } else { 309 | pos %= s.len(); 310 | } 311 | 312 | let content = &word_choices[content as usize % word_choices.len()]; 313 | s.insert(pos, content.clone()); 314 | debug_log::group!( 315 | "INSERT pos={} content={} ans={}", 316 | pos, 317 | content, 318 | s.iter().fold(String::new(), |mut left, cur| { 319 | left.push_str(cur); 320 | left 321 | }) 322 | ); 323 | actor.insert_utf16(pos, content); 324 | // actor.check(); 325 | } 326 | Action::Delete { pos, len, .. } => { 327 | let mut pos = pos as usize; 328 | if s.is_empty() { 329 | pos = 0; 330 | } else { 331 | pos %= s.len(); 332 | } 333 | let len = (len as usize).min(s.len() - pos); 334 | s.drain(pos..pos + len); 335 | debug_log::group!( 336 | "DELETE pos={} len={} ans={}", 337 | pos, 338 | len, 339 | s.iter().fold(String::new(), |mut left, cur| { 340 | left.push_str(cur); 341 | left 342 | }) 343 | ); 344 | actor.delete_utf16(pos, len); 345 | // actor.check(); 346 | } 347 | _ => {} 348 | } 349 | 350 | debug_log::group_end!(); 351 | } 352 | 353 | let mut ans = String::new(); 354 | for span in s { 355 | ans.push_str(&span) 356 | } 357 | 358 | assert_eq!(&actor.text.to_string(), &ans) 359 | } 360 | 361 | impl Actor { 362 | pub fn new(id: usize) -> Self { 363 | Self { 364 | text: RichText::new(id as u64), 365 | } 366 | } 367 | 368 | pub fn insert(&mut self, pos: usize, content: &str) { 369 | self.text.insert(pos, content); 370 | } 371 | 372 | pub fn insert_utf16(&mut self, pos: usize, as_str: &str) { 373 | self.text.insert_utf16(pos, as_str) 374 | } 375 | 376 | /// this should happen after the op is integrated to the list crdt 377 | pub fn delete(&mut self, pos: usize, len: usize) { 378 | self.text.delete(pos..pos + len) 379 | } 380 | 381 | pub fn delete_utf16(&mut self, pos: usize, len: usize) { 382 | self.text.delete_utf16(pos..pos + len) 383 | } 384 | 385 | pub fn annotate(&mut self, range: impl RangeBounds, type_: AnnotationType) { 386 | self._annotate(range, type_, IndexType::Utf8) 387 | } 388 | 389 | pub fn annotate_utf16(&mut self, range: impl RangeBounds, type_: AnnotationType) { 390 | self._annotate(range, type_, IndexType::Utf16) 391 | } 392 | 393 | fn _annotate( 394 | &mut self, 395 | range: impl RangeBounds, 396 | type_: AnnotationType, 397 | index_type: IndexType, 398 | ) { 399 | match type_ { 400 | AnnotationType::Bold => self.text.annotate_inner( 401 | range, 402 | Style { 403 | expand: Expand::After, 404 | behavior: crate::Behavior::Merge, 405 | type_: "bold".into(), 406 | value: serde_json::Value::Null, 407 | }, 408 | index_type, 409 | ), 410 | AnnotationType::Link => self.text.annotate_inner( 411 | range, 412 | Style { 413 | expand: Expand::None, 414 | behavior: crate::Behavior::Merge, 415 | type_: "link".into(), 416 | value: serde_json::Value::Bool(true), 417 | }, 418 | index_type, 419 | ), 420 | AnnotationType::Comment => self.text.annotate_inner( 421 | range, 422 | Style { 423 | expand: Expand::None, 424 | behavior: crate::Behavior::AllowMultiple, 425 | type_: "comment".into(), 426 | value: serde_json::Value::String("This is a comment".to_owned()), 427 | }, 428 | index_type, 429 | ), 430 | AnnotationType::UnBold => self.text.annotate_inner( 431 | range, 432 | Style { 433 | expand: Expand::After, 434 | behavior: crate::Behavior::Delete, 435 | type_: "bold".into(), 436 | value: serde_json::Value::Null, 437 | }, 438 | index_type, 439 | ), 440 | AnnotationType::UnLink => self.text.annotate_inner( 441 | range, 442 | Style { 443 | expand: Expand::Both, 444 | behavior: crate::Behavior::Delete, 445 | type_: "link".into(), 446 | value: serde_json::Value::Null, 447 | }, 448 | index_type, 449 | ), 450 | }; 451 | } 452 | 453 | fn merge(&mut self, other: &Self) { 454 | self.text.merge(&other.text) 455 | } 456 | 457 | #[allow(unused)] 458 | fn check_eq(&mut self, other: &mut Self) { 459 | assert_eq!(self.len(), other.len()); 460 | assert_eq!(self.text.to_string(), other.text.to_string()); 461 | } 462 | 463 | #[allow(unused)] 464 | fn len(&self) -> usize { 465 | self.text.len() 466 | } 467 | 468 | #[allow(unused)] 469 | fn len_utf16(&self) -> usize { 470 | self.text.len_utf16() 471 | } 472 | 473 | fn check(&self) { 474 | self.text.check() 475 | } 476 | } 477 | -------------------------------------------------------------------------------- /src/rich_text/vv.rs: -------------------------------------------------------------------------------- 1 | use fxhash::FxHashMap; 2 | use serde::{Deserialize, Serialize}; 3 | use serde_columnar::to_vec; 4 | 5 | use crate::{ClientID, Counter}; 6 | 7 | #[derive(Default, Debug, Clone, Serialize, Deserialize)] 8 | pub struct VersionVector { 9 | pub vv: FxHashMap, 10 | } 11 | 12 | #[derive(Serialize, Clone, Copy, Deserialize)] 13 | struct Item { 14 | client: ClientID, 15 | counter: Counter, 16 | } 17 | 18 | impl VersionVector { 19 | pub fn encode(&self) -> Vec { 20 | let v: Vec = self 21 | .vv 22 | .iter() 23 | .map(|x| Item { 24 | client: *x.0, 25 | counter: *x.1, 26 | }) 27 | .collect(); 28 | to_vec(&v).unwrap() 29 | } 30 | 31 | pub fn decode(data: &[u8]) -> VersionVector { 32 | let v: Vec = serde_columnar::from_bytes(data).unwrap(); 33 | let mut vv = VersionVector::default(); 34 | for item in v { 35 | vv.vv.insert(item.client, item.counter); 36 | } 37 | vv 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/small_set.rs: -------------------------------------------------------------------------------- 1 | use fxhash::FxHashSet; 2 | const STACK_LEN: usize = 4; 3 | 4 | /// When the set is small it only live in the stack memory. 5 | /// 6 | /// It's used to store the difference of the annotation, which is 7 | /// usually small or empty. It uses positive(or negative) values to represent new 8 | /// insertions(or deletion). 0 is illegal value. 9 | /// 10 | /// A absolute value can only exist once in the set. 11 | /// i.e. if 3 is in the set, -3 must not be in the set. 12 | /// 13 | /// If 3 is in the set and user insert -3, the insertion will not happen, 14 | /// instead the 3 will be removed from the set. 15 | #[derive(Debug, Clone)] 16 | pub enum SmallSetI32 { 17 | Stack(([i32; STACK_LEN], u8)), 18 | Heap(FxHashSet), 19 | } 20 | 21 | impl Default for SmallSetI32 { 22 | fn default() -> Self { 23 | Self::new() 24 | } 25 | } 26 | 27 | impl SmallSetI32 { 28 | const EMPTY_VALUE: i32 = 0; 29 | pub(crate) fn new() -> Self { 30 | SmallSetI32::Stack(([Self::EMPTY_VALUE; STACK_LEN], 0)) 31 | } 32 | 33 | pub(crate) fn insert(&mut self, value: i32) { 34 | assert!(value != 0); 35 | if self.contains(-value) { 36 | self.remove(-value); 37 | return; 38 | } 39 | 40 | match self { 41 | SmallSetI32::Stack((stack, size)) => { 42 | for entry in stack.iter() { 43 | if *entry == value { 44 | // already exists 45 | return; 46 | } 47 | } 48 | 49 | for entry in stack.iter_mut() { 50 | if *entry == Self::EMPTY_VALUE { 51 | *entry = value; 52 | *size += 1; 53 | return; 54 | } 55 | } 56 | 57 | let mut set = 58 | FxHashSet::with_capacity_and_hasher(STACK_LEN * 2, Default::default()); 59 | 60 | for &v in stack.iter() { 61 | // we already know it's non empty 62 | set.insert(v); 63 | } 64 | set.insert(value); 65 | *self = SmallSetI32::Heap(set); 66 | } 67 | SmallSetI32::Heap(set) => { 68 | set.insert(value); 69 | } 70 | } 71 | } 72 | 73 | pub(crate) fn contains(&self, value: i32) -> bool { 74 | if value == 0 { 75 | return false; 76 | } 77 | 78 | match self { 79 | SmallSetI32::Stack((stack, size)) => { 80 | if *size == 0 { 81 | return false; 82 | } 83 | 84 | for entry in stack.iter() { 85 | if *entry == value { 86 | return true; 87 | } 88 | } 89 | 90 | false 91 | } 92 | SmallSetI32::Heap(set) => set.contains(&value), 93 | } 94 | } 95 | 96 | pub(crate) fn remove(&mut self, value: i32) { 97 | match self { 98 | SmallSetI32::Stack((stack, size)) => { 99 | if *size == 0 { 100 | return; 101 | } 102 | 103 | for entry in stack.iter_mut() { 104 | if *entry == value { 105 | *entry = Self::EMPTY_VALUE; 106 | *size -= 1; 107 | } 108 | } 109 | } 110 | SmallSetI32::Heap(set) => { 111 | set.remove(&value); 112 | } 113 | } 114 | } 115 | 116 | pub(crate) fn iter(&self) -> SmallSetIter { 117 | match self { 118 | SmallSetI32::Stack((stack, _)) => SmallSetIter::Stack(stack.iter()), 119 | SmallSetI32::Heap(set) => SmallSetIter::Heap(set.iter()), 120 | } 121 | } 122 | 123 | pub(crate) fn len(&self) -> usize { 124 | match self { 125 | SmallSetI32::Stack((_, size)) => *size as usize, 126 | SmallSetI32::Heap(set) => set.len(), 127 | } 128 | } 129 | 130 | pub(crate) fn is_empty(&self) -> bool { 131 | self.len() == 0 132 | } 133 | } 134 | 135 | pub(crate) enum SmallSetIter<'a> { 136 | Stack(std::slice::Iter<'a, i32>), 137 | Heap(std::collections::hash_set::Iter<'a, i32>), 138 | } 139 | 140 | impl<'a> Iterator for SmallSetIter<'a> { 141 | type Item = i32; 142 | 143 | fn next(&mut self) -> Option { 144 | match self { 145 | SmallSetIter::Stack(iter) => { 146 | let mut ans = iter.next(); 147 | while ans == Some(&SmallSetI32::EMPTY_VALUE) { 148 | ans = iter.next(); 149 | } 150 | ans.copied() 151 | } 152 | SmallSetIter::Heap(iter) => iter.next().copied(), 153 | } 154 | } 155 | } 156 | 157 | #[cfg(test)] 158 | mod test { 159 | use super::SmallSetI32; 160 | 161 | #[test] 162 | fn test() { 163 | let mut set = SmallSetI32::new(); 164 | set.insert(1); 165 | set.insert(2); 166 | set.insert(2); 167 | set.insert(2); 168 | set.insert(1); 169 | assert_eq!(set.len(), 2); 170 | assert!(set.contains(2)); 171 | assert!(set.contains(1)); 172 | assert!(!set.contains(0)); 173 | assert!(!set.contains(-2)); 174 | set.remove(2); 175 | assert!(!set.contains(2)); 176 | assert!(set.len() == 1); 177 | } 178 | 179 | #[test] 180 | fn smallset_size() { 181 | assert_eq!(32, std::mem::size_of::()); 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /src/test_utils.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashSet; 2 | 3 | use super::*; 4 | use arbitrary::Arbitrary; 5 | 6 | #[derive(Debug, PartialEq, Eq)] 7 | pub(crate) struct SimpleSpan { 8 | pub len: usize, 9 | pub annotations: HashSet, 10 | } 11 | 12 | #[derive(Arbitrary, Clone, Copy, Debug)] 13 | pub enum AnnotationType { 14 | Link, 15 | Bold, 16 | Comment, 17 | UnBold, 18 | UnLink, 19 | } 20 | 21 | #[derive(Arbitrary, Clone, Debug, Copy)] 22 | pub enum Action { 23 | Insert { 24 | actor: u8, 25 | pos: u8, 26 | len: u8, 27 | }, 28 | Delete { 29 | actor: u8, 30 | pos: u8, 31 | len: u8, 32 | }, 33 | Annotate { 34 | actor: u8, 35 | pos: u8, 36 | len: u8, 37 | annotation: AnnotationType, 38 | }, 39 | Sync(u8, u8), 40 | } 41 | 42 | #[allow(unused)] 43 | pub(crate) fn make_spans(spans: &[(Vec<&str>, usize)]) -> Vec { 44 | spans 45 | .iter() 46 | .map(|(annotations, len)| SimpleSpan { 47 | annotations: annotations.iter().map(|x| (*x).into()).collect(), 48 | len: *len, 49 | }) 50 | .collect() 51 | } 52 | --------------------------------------------------------------------------------