├── .github └── workflows │ └── test.yml ├── .gitignore ├── .vscode └── launch.json ├── BINARY.md ├── Cargo.toml ├── INTERNALS.md ├── README.md ├── bench.sh ├── benchmark_data ├── A1.dt ├── A2.dt ├── C1.dt ├── C2.dt ├── S1.dt ├── S2.dt ├── S3.dt ├── automerge-paper.dt ├── automerge-paper.json.gz ├── clownschool.dt ├── clownschool_flat.json.gz ├── egwalker.dt ├── egwalker.json.gz ├── friendsforever.dt ├── friendsforever_flat.json.gz ├── friendsforever_raw.dt ├── git-makefile.dt ├── idxtrace_clownschool.json.gz ├── idxtrace_friendsforever.json.gz ├── idxtrace_git-makefile.json.gz ├── idxtrace_node_nodecc.json.gz ├── node_nodecc.dt ├── rustcode.json.gz ├── seph-blog1.dt ├── seph-blog1.json.gz └── sveltecomponent.json.gz ├── bucket_counts.js ├── build-swift.sh ├── build_wasm.sh ├── crates ├── bench │ ├── Cargo.toml │ └── src │ │ ├── idxtrace.rs │ │ ├── main.rs │ │ └── utils.rs ├── crdt-testdata │ ├── Cargo.toml │ └── src │ │ ├── lib.rs │ │ └── nonlinear.rs ├── dt-cli │ ├── Cargo.toml │ └── src │ │ ├── dot.rs │ │ ├── export.rs │ │ ├── git.rs │ │ └── main.rs ├── dt-swift │ ├── Cargo.toml │ ├── build.rs │ └── src │ │ └── lib.rs ├── dt-wasm │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── example.mjs │ ├── foo.mjs │ ├── src │ │ ├── lib.rs │ │ └── utils.rs │ └── tests │ │ └── web.rs ├── rle │ ├── Cargo.toml │ ├── README.md │ └── src │ │ ├── append_rle.rs │ │ ├── intersect.rs │ │ ├── lib.rs │ │ ├── merge_iter.rs │ │ ├── rlerun.rs │ │ ├── splitable_span.rs │ │ ├── take_max_iter.rs │ │ └── zip.rs └── trace-alloc │ ├── Cargo.toml │ └── src │ └── lib.rs ├── examples ├── posstats.rs ├── print.rs └── profile.rs ├── js ├── .mocharc.json ├── example.js ├── package.json ├── public │ └── index.html ├── src │ ├── client.ts │ ├── fancydb │ │ ├── causal-graph.ts │ │ ├── index.ts │ │ └── rle.ts │ ├── msgs.ts │ ├── server.ts │ ├── simpledb.ts │ ├── text │ │ └── text.ts │ ├── types.ts │ └── utils.ts ├── tests │ ├── causal-graph.ts │ └── cg-tools.ts ├── tsconfig.json └── yarn.lock ├── npm-pkg-isomorphic ├── README.md ├── index.js └── package.json ├── prev_oplog ├── branch.rs ├── hack.rs ├── lib.rs ├── operation.rs ├── oplog.rs ├── path.rs └── simpledb.rs ├── profile.sh ├── src ├── branch.rs ├── causalgraph │ ├── agent_assignment │ │ ├── mod.rs │ │ └── remote_ids.rs │ ├── agent_span.rs │ ├── causalgraph.rs │ ├── check.rs │ ├── dot.rs │ ├── enc_fuzzer.rs │ ├── entry.rs │ ├── eq.rs │ ├── graph │ │ ├── check.rs │ │ ├── conflict_subgraph.rs │ │ ├── mod.rs │ │ ├── random_graphs.rs │ │ ├── scope.rs │ │ ├── simple.rs │ │ ├── subgraph.rs │ │ └── tools.rs │ ├── mod.rs │ ├── storage.rs │ └── summary.rs ├── check.rs ├── dtrange.rs ├── encoding │ ├── bufparser.rs │ ├── cg_entry.rs │ ├── chunk_reader.rs │ ├── map.rs │ ├── mod.rs │ ├── op.rs │ ├── op_contents.rs │ ├── parents.rs │ ├── parseerror.rs │ ├── tools.rs │ └── varint.rs ├── frontier.rs ├── fuzzer.rs ├── lib.rs ├── list │ ├── branch.rs │ ├── buffered_iter.rs │ ├── check.rs │ ├── encoding │ │ ├── README.md │ │ ├── decode_oplog.rs │ │ ├── decode_tools.rs │ │ ├── encode_oplog.rs │ │ ├── encode_options.rs │ │ ├── encode_tools.rs │ │ ├── fuzzer.rs │ │ ├── leb.rs │ │ ├── mod.rs │ │ ├── save_transformed.rs │ │ ├── tests.rs │ │ └── txn_trace.rs │ ├── eq.rs │ ├── gen_random.rs │ ├── list.rs │ ├── merge.rs │ ├── mod.rs │ ├── old_fuzzer_tools.rs │ ├── op_iter.rs │ ├── op_metrics.rs │ ├── operation.rs │ ├── oplog.rs │ ├── oplog_merge.rs │ ├── oplog_merge_fuzzer.rs │ └── stochastic_summary.rs ├── list_fuzzer_tools.rs ├── listmerge │ ├── README.md │ ├── advance_retreat.rs │ ├── dot.rs │ ├── fuzzer.rs │ ├── markers.rs │ ├── merge.rs │ ├── mod.rs │ ├── plan.rs │ ├── simple_oplog.rs │ ├── xf_old.rs │ └── yjsspan.rs ├── listmerge2 │ ├── README.md │ ├── action_plan.rs │ ├── dot.rs │ ├── index_gap_buffer.rs │ ├── mod.rs │ ├── test_conversion.rs │ └── yjsspan.rs ├── oplog.rs ├── ost │ ├── content_tree.rs │ ├── index_tree.rs │ └── mod.rs ├── rev_range.rs ├── rle │ ├── mod.rs │ └── rle_vec.rs ├── serde_helpers.rs ├── simple_checkout.rs ├── stats.rs ├── storage │ ├── README.md │ ├── file.rs │ ├── mod.rs │ └── page.rs ├── textinfo.rs ├── unicount.rs └── wal.rs ├── test_data ├── causal_graph │ ├── conflicting.json │ ├── diff.json │ └── version_contains.json ├── conformance.json.br └── ot │ ├── apply.json │ ├── compose.json │ └── transform.json ├── vis ├── build.sh ├── package.json ├── public │ └── index.html ├── snowpack.config.mjs ├── src │ ├── App.svelte │ ├── DocInfo.svelte │ ├── Editor.svelte │ ├── index.ts │ └── utils.ts ├── svelte.config.js ├── tsconfig.json ├── types │ └── static.d.ts └── yarn.lock └── wiki ├── client ├── dt_doc.ts ├── index.css ├── index.tsx └── mdstyle.css ├── common ├── ratelimit.ts └── utils.ts ├── index.html ├── package.json ├── server └── server.ts ├── tsconfig.json ├── vite.config.ts └── yarn.lock /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push, pull_request] 3 | 4 | env: 5 | CARGO_TERM_COLOR: always 6 | 7 | jobs: 8 | test: 9 | name: Test Suite 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout sources 13 | uses: actions/checkout@v3 14 | 15 | - name: Install stable toolchain 16 | uses: actions-rs/toolchain@v1 17 | with: 18 | profile: minimal 19 | toolchain: stable 20 | override: true 21 | 22 | - run: cargo test 23 | - run: cargo test -p dt-cli -p rle -p dt-wasm -p dt-swift 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | node_modules 4 | vis/build 5 | .DS_Store 6 | /*.dt 7 | wiki/dist 8 | wiki/dist-client 9 | pkg-web 10 | pkg-node 11 | js/dist 12 | bundle* 13 | generated 14 | 15 | db.dtjson 16 | .vscode 17 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "lldb", 9 | "request": "launch", 10 | "name": "Launch", 11 | "program": "${workspaceFolder}/target/debug/deps/text_crdt_rust-899d31645cc1cbb6", 12 | "args": ["--nocapture", "random_concurrency"], 13 | "cwd": "${workspaceFolder}", 14 | "sourceLanguages": ["rust"], 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "diamond-types" 3 | version = "2.0.0" 4 | edition = "2021" 5 | exclude = [ 6 | ".idea", ".vscode", 7 | "vis", "wiki", "js", 8 | "benchmark_data", "test_data", 9 | ".github" 10 | ] 11 | license = "ISC" 12 | description = "The world's fastest text CRDT" 13 | 14 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 15 | 16 | [workspace] 17 | members = ["crates/*"] 18 | 19 | [dependencies] 20 | smartstring = "1.0.1" 21 | str_indices = "0.4.3" 22 | smallvec = "2.0.0-alpha.6" 23 | lazy_static = "1.4.0" 24 | 25 | # Used by wasm module, CLI. 26 | serde = { version = "1.0.183", features = ["derive"], optional = true } 27 | rle = { version = "0.2.0", path = "crates/rle", features = ["smallvec"] } 28 | 29 | # Only used for generating testing data. 30 | serde_json = { version = "1.0.104", optional = true } 31 | 32 | bumpalo = { version = "3.16.0", features = ["collections"] } 33 | 34 | #jumprope = { path = "../../../jumprope-rs" } 35 | 36 | # Its tempting to disable default-features in jumprope because it means we don't need to hook in crypto random, which 37 | # saves some size in the wasm output size. But I think its better to default to having this feature enabled. 38 | #jumprope = { path = "../jumprope-rs", version = "1.1.0" } 39 | jumprope = "1.1.2" 40 | humansize = "2.0.0" 41 | num_enum = "0.7.2" 42 | 43 | # crc32c might be faster, but it adds 10kb to the wasm bundle size. crc only adds 1kb. 44 | #crc32c = "0.6" 45 | crc = "3.0.0" 46 | lz4_flex = { version = "0.11.3", optional = true } 47 | 48 | #bitvec = "1.0.1" 49 | 50 | # Needed for macos F_BARRIERFSYNC. 51 | libc = "0.2.139" 52 | 53 | rand = { version = "0.8.5", features = ["small_rng"], optional = true } 54 | 55 | 56 | [dev-dependencies] 57 | rand = { version = "0.8.5", features = ["small_rng"] } 58 | crdt-testdata = { path = "crates/crdt-testdata" } 59 | trace-alloc = { path = "crates/trace-alloc" } 60 | 61 | # For OT fuzz data tests 62 | #json_minimal = "0.1.3" 63 | 64 | [features] 65 | #default = ["lz4", "storage", "rand"] # rand is only used in testing code, but there's no way to specify that. 66 | default = ["lz4", "storage"] 67 | memusage = ["trace-alloc/memusage"] 68 | lz4 = ["dep:lz4_flex"] 69 | serde = ["dep:serde", "smallvec/serde", "smartstring/serde"] 70 | dot_export = [] 71 | wchar_conversion = ["jumprope/wchar_conversion"] 72 | merge_conflict_checks = [] 73 | storage = [] 74 | expose_benchmarking = ["serde", "serde_json"] 75 | stats = [] 76 | 77 | # This is internal only for generating JSON testing data. To generate, run test suite with 78 | # rm *_tests.json; cargo test --features gen_test_data causalgraph::parents::tools -- --test-threads 1 79 | gen_test_data = ["serde", "serde_json", "rand"] 80 | 81 | [lib] 82 | bench = false 83 | 84 | [profile.release] 85 | #debug = true 86 | lto = true 87 | codegen-units = 1 88 | #opt-level = "s" 89 | panic = "abort" 90 | 91 | [profile.release.package.dt-wasm] 92 | opt-level = 2 93 | #opt-level = "s" 94 | #debugging = true 95 | 96 | [profile.release.package.dt-cli] 97 | opt-level = "s" 98 | #lto = false 99 | strip = true 100 | 101 | [profile.release.package.bench] 102 | codegen-units = 1 103 | 104 | # Use with cargo build --profile profiling 105 | [profile.profiling] 106 | inherits = "release" 107 | debug = true 108 | #opt-level = 0 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Diamond Types 2 | 3 | [**📦 Cargo package**](https://crates.io/crates/diamond-types) 4 | 5 | [**📖 Documentation on docs.rs**](https://docs.rs/diamond-types/latest/diamond_types/) 6 | 7 | [**🇳 NodeJS package on npm (via WASM)**](https://www.npmjs.com/package/diamond-types-node) 8 | 9 | [**🌐 Web browser package on npm (via WASM)**](https://www.npmjs.com/package/diamond-types-web) 10 | 11 | This repository contains a high performance rust CRDT for text editing. This is a special data type which supports concurrent editing of lists or strings 12 | (text documents) by multiple users in a P2P network without needing a 13 | centralized server. 14 | 15 | This version of diamond types only supports plain text editing. Work is underway to add support for other JSON-style data types. See the `more_types` branch for details. 16 | 17 | This project was initially created as a prototype to see how fast a well optimized CRDT could be made to go. The answer is really fast - faster than other similar libraries. This library is currently in the process of being expanded into a fast, feature rich CRDT in its own right. 18 | 19 | For detail about how to *use* diamond types, see the [package level documentation at docs.rs](https://docs.rs/diamond-types/latest/diamond_types/). 20 | 21 | Note the package published to cargo is quite out of date, both in terms of API and performance. 22 | 23 | For much more detail about how this library *works*, see: 24 | 25 | - The talk I gave on this library at a recent [braid user meetings](https://braid.org/meeting-14) or 26 | - [INTERNALS.md](INTERNALS.md) in this repository. 27 | - [This blog post on making diamond types 5000x faster than competing CRDT implementations](https://josephg.com/blog/crdts-go-brrr/) 28 | - And since that blog post came out, performance has increased another 10-80x (!). 29 | 30 | As well as being lightning fast, this library is also designed to be interoperable with positional updates. This allows simple peers to interact with the data structure via operational transform. 31 | 32 | 33 | ## Internals 34 | 35 | Each client / device has a unique ID. Each character typed or deleted on 36 | each device is assigned an incrementing sequence number (starting at 0). 37 | Each character in the document can thus be uniquely identified by the 38 | tuple of `(client ID, sequence number)`. This allows any location in the 39 | document to be uniquely named. 40 | 41 | The internal data structures are designed to optimize two main operations: 42 | 43 | - Text edit to CRDT operation (Eg, "user A inserts at position 100" -> "user A 44 | seq 1000 inserts at (B, 50)") 45 | - CRDT operation to text edit ("user A 46 | seq 1000 inserts at (B, 50)" -> "insert at document position 100") 47 | 48 | Much more detail on the internal data structures used is in [INTERNALS.md](INTERNALS.md). 49 | 50 | 51 | # LICENSE 52 | 53 | This code is published under the ISC license. 54 | 55 | 56 | # Acknowledgements 57 | 58 | This work has been made possible by funding from the [Invisible College](https://invisible.college/). -------------------------------------------------------------------------------- /bench.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | #set -o xtrace 4 | 5 | start_time=$(date +%s) # Capture start time in seconds 6 | 7 | cargo build --release -p bench 8 | 9 | end_time=$(date +%s) # Capture end time in seconds 10 | # Calculate duration 11 | duration=$((end_time - start_time)) 12 | 13 | # Check if duration is less than 1 second 14 | if [ $duration -gt 1 ]; then 15 | echo "Waiting 5s for CPU to cool down" 16 | sleep 5 17 | fi 18 | 19 | #taskset 0x1 nice -10 cargo run --release -p bench -- --bench $@ 20 | taskset 0x1 nice -10 target/release/bench --bench "$@" 21 | -------------------------------------------------------------------------------- /benchmark_data/A1.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/A1.dt -------------------------------------------------------------------------------- /benchmark_data/A2.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/A2.dt -------------------------------------------------------------------------------- /benchmark_data/C1.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/C1.dt -------------------------------------------------------------------------------- /benchmark_data/C2.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/C2.dt -------------------------------------------------------------------------------- /benchmark_data/S1.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/S1.dt -------------------------------------------------------------------------------- /benchmark_data/S2.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/S2.dt -------------------------------------------------------------------------------- /benchmark_data/S3.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/S3.dt -------------------------------------------------------------------------------- /benchmark_data/automerge-paper.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/automerge-paper.dt -------------------------------------------------------------------------------- /benchmark_data/automerge-paper.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/automerge-paper.json.gz -------------------------------------------------------------------------------- /benchmark_data/clownschool.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/clownschool.dt -------------------------------------------------------------------------------- /benchmark_data/clownschool_flat.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/clownschool_flat.json.gz -------------------------------------------------------------------------------- /benchmark_data/egwalker.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/egwalker.dt -------------------------------------------------------------------------------- /benchmark_data/egwalker.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/egwalker.json.gz -------------------------------------------------------------------------------- /benchmark_data/friendsforever.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/friendsforever.dt -------------------------------------------------------------------------------- /benchmark_data/friendsforever_flat.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/friendsforever_flat.json.gz -------------------------------------------------------------------------------- /benchmark_data/friendsforever_raw.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/friendsforever_raw.dt -------------------------------------------------------------------------------- /benchmark_data/git-makefile.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/git-makefile.dt -------------------------------------------------------------------------------- /benchmark_data/idxtrace_clownschool.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/idxtrace_clownschool.json.gz -------------------------------------------------------------------------------- /benchmark_data/idxtrace_friendsforever.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/idxtrace_friendsforever.json.gz -------------------------------------------------------------------------------- /benchmark_data/idxtrace_git-makefile.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/idxtrace_git-makefile.json.gz -------------------------------------------------------------------------------- /benchmark_data/idxtrace_node_nodecc.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/idxtrace_node_nodecc.json.gz -------------------------------------------------------------------------------- /benchmark_data/node_nodecc.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/node_nodecc.dt -------------------------------------------------------------------------------- /benchmark_data/rustcode.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/rustcode.json.gz -------------------------------------------------------------------------------- /benchmark_data/seph-blog1.dt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/seph-blog1.dt -------------------------------------------------------------------------------- /benchmark_data/seph-blog1.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/seph-blog1.json.gz -------------------------------------------------------------------------------- /benchmark_data/sveltecomponent.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/benchmark_data/sveltecomponent.json.gz -------------------------------------------------------------------------------- /bucket_counts.js: -------------------------------------------------------------------------------- 1 | const bucket_items_50 = [ 2 | 0, 3 | 615, 4 | 205, 5 | 290, 6 | 180, 7 | 169, 8 | 132, 9 | 94, 10 | 82, 11 | 91, 12 | 79, 13 | 55, 14 | 43, 15 | 59, 16 | 61, 17 | 58, 18 | 44, 19 | 117, 20 | 115, 21 | 85, 22 | 73, 23 | 68, 24 | 92, 25 | 109, 26 | 105, 27 | 327, 28 | 244, 29 | 39, 30 | 10, 31 | 2, 32 | 0, 33 | 2, 34 | 1, 35 | 0, 36 | 0, 37 | 1, 38 | ] 39 | const bucket_items_10 = [ 40 | 0, 41 | 7567, 42 | 1512, 43 | 1095, 44 | 2417, 45 | 3104, 46 | 2367, 47 | 137, 48 | 19, 49 | 13, 50 | 1, 51 | ] 52 | 53 | 54 | const size_counts = [ 55 | 0, 56 | 5496, 57 | 24211, 58 | 7895, 59 | 899, 60 | 329, 61 | 258, 62 | 240, 63 | 232, 64 | 194, 65 | 204, 66 | 182, 67 | 158, 68 | 140, 69 | 146, 70 | 139, 71 | 134, 72 | 110, 73 | 104, 74 | 108, 75 | 81, 76 | 89, 77 | 81, 78 | 59, 79 | 77, 80 | 59, 81 | 60, 82 | 59, 83 | 58, 84 | 63, 85 | 40, 86 | 38, 87 | 38, 88 | 41, 89 | 40, 90 | 55, 91 | 36, 92 | 27, 93 | 29, 94 | 42, 95 | 27, 96 | 40, 97 | 34, 98 | 26, 99 | 25, 100 | 25, 101 | 22, 102 | 26, 103 | 28, 104 | 25, 105 | 614, 106 | ] 107 | 108 | const mean = nums => { 109 | const num = nums.reduce((a, b, idx) => a+b, 0) 110 | const sum = nums.reduce((a, b, idx) => a+b*idx, 0) 111 | console.log('sum', sum, num) 112 | console.log('mean', sum / num) 113 | } 114 | 115 | mean(size_counts) 116 | mean(bucket_items_50) 117 | mean(bucket_items_10) 118 | -------------------------------------------------------------------------------- /build-swift.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | THISDIR=$(dirname $0) 6 | cd $THISDIR 7 | 8 | #FLAGS="" 9 | #MODE="debug" 10 | FLAGS="--release" 11 | MODE="release" 12 | 13 | export SWIFT_BRIDGE_OUT_DIR="$(pwd)/crates/dt-swift/generated" 14 | export RUSTFLAGS="" 15 | # Build the project for the desired platforms: 16 | cargo build $FLAGS --target x86_64-apple-darwin -p dt-swift 17 | cargo build $FLAGS --target aarch64-apple-darwin -p dt-swift 18 | mkdir -p ./target/universal-macos/"$MODE" 19 | 20 | lipo \ 21 | ./target/aarch64-apple-darwin/"$MODE"/libdt_swift.a \ 22 | ./target/x86_64-apple-darwin/"$MODE"/libdt_swift.a \ 23 | -create -output ./target/universal-macos/"$MODE"/libdt_swift.a 24 | 25 | cargo build $FLAGS --target aarch64-apple-ios -p dt-swift 26 | #cargo build --target x86_64-apple-ios 27 | cargo build $FLAGS --target aarch64-apple-ios-sim -p dt-swift 28 | mkdir -p ./target/universal-ios/"$MODE" 29 | 30 | #lipo \ 31 | # ./target/aarch64-apple-ios-sim/"$MODE"/libdt_swift.a \ 32 | # -create -output ./target/universal-ios/"$MODE"/libdt_swift.a 33 | # ./target/aarch64-apple-ios/"$MODE"/libdt_swift.a \ 34 | 35 | swift-bridge-cli create-package \ 36 | --bridges-dir "$SWIFT_BRIDGE_OUT_DIR" \ 37 | --out-dir target/dt-swift \ 38 | --ios ./target/aarch64-apple-ios/"$MODE"/libdt_swift.a \ 39 | --simulator ./target/aarch64-apple-ios-sim/"$MODE"/libdt_swift.a \ 40 | --macos ./target/universal-macos/"$MODE"/libdt_swift.a \ 41 | --name DiamondTypes 42 | 43 | 44 | #--simulator target/universal-ios/"$MODE"/libdt_swift.a \ 45 | -------------------------------------------------------------------------------- /build_wasm.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | RUSTFLAGS="" 4 | #cd crates/diamond-wasm 5 | 6 | echo "=== Before ===" 7 | ls -l pkg-web pkg-node || true 8 | #wasm-pack build --target nodejs 9 | #wasm-pack build --target bundler 10 | #wasm-pack build --target web --dev 11 | 12 | rm -rf pkg-* 13 | #wasm-pack build --target web --out-dir ../../pkg-web --out-name dt crates/dt-wasm --profiling 14 | wasm-pack build --target web --out-dir ../../pkg-web --out-name dt crates/dt-wasm 15 | wasm-pack build --target nodejs --out-dir ../../pkg-node --out-name dt crates/dt-wasm 16 | 17 | # sed -i '3i\ \ "type": "module",' pkg/package.json 18 | 19 | # Set version 20 | #sed -i.old 's/: "0.1.0"/: "0.1.1"/' pkg-*/package.json 21 | 22 | # Web code needs to have "main" defined since its an es6 module package 23 | sed -i.old 's/"module":/"main":/' pkg-web/package.json 24 | sed -i.old 's/"name": "dt-wasm"/"name": "diamond-types-web"/' pkg-web/package.json 25 | sed -i.old 's/"name": "dt-wasm"/"name": "diamond-types-node"/' pkg-node/package.json 26 | sed -i.old 's/"files": \[/"files": \[\n "dt_bg.wasm.br",/' pkg-web/package.json 27 | #perl -wlpi -e 'print " \"type\": \"module\"," if $. == 2' pkg-web/package.json 28 | 29 | sed -i.old 's/"0.1.0"/"1.0.2"/' pkg-web/package.json 30 | sed -i.old 's/"0.1.0"/"1.0.2"/' pkg-node/package.json 31 | 32 | rm pkg-*/package.json.old 33 | 34 | brotli -f pkg-web/*.wasm 35 | 36 | echo "=== After ===" 37 | ls -l pkg-web pkg-node 38 | 39 | cat pkg-web/package.json 40 | -------------------------------------------------------------------------------- /crates/bench/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bench" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | diamond-types = { path = "../..", features = ["expose_benchmarking"] } 10 | rle = { path = "../rle" } 11 | #criterion = { version = "0.5.1", features = ["html_reports"] } 12 | criterion = { version = "0.5.1", features = [] } 13 | crdt-testdata = { path = "../crdt-testdata" } 14 | flate2 = { version = "1.0.33"} 15 | jumprope = "1.1.2" -------------------------------------------------------------------------------- /crates/bench/src/idxtrace.rs: -------------------------------------------------------------------------------- 1 | //! This file contains benchmarks for replaying index traces 2 | //! 3 | //! The code this is built on has been removed from diamond-types. 4 | 5 | use std::fs::File; 6 | use std::io::{BufReader, Read}; 7 | use criterion::{BenchmarkId, Criterion}; 8 | use flate2::bufread::GzDecoder; 9 | use diamond_types::IndexTreeTrace; 10 | 11 | const DATASETS: &[&str] = &["node_nodecc", "git-makefile", "friendsforever", "clownschool"]; 12 | 13 | pub fn idxtrace_benchmarks(c: &mut Criterion) { 14 | for name in DATASETS { 15 | let mut group = c.benchmark_group("dt"); 16 | 17 | let filename = format!("benchmark_data/idxtrace_{name}.json.gz"); 18 | let reader = BufReader::new(File::open(filename).unwrap()); 19 | let mut reader = GzDecoder::new(reader); 20 | let mut raw_json = vec!(); 21 | reader.read_to_end(&mut raw_json).unwrap(); 22 | 23 | let trace = IndexTreeTrace::from_json(&raw_json); 24 | 25 | group.bench_function(BenchmarkId::new("idxtrace", name), |b| { 26 | b.iter(|| { 27 | trace.replay(); 28 | }) 29 | }); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /crates/bench/src/utils.rs: -------------------------------------------------------------------------------- 1 | use diamond_types::list::*; 2 | use crdt_testdata::{TestTxn, TestPatch}; 3 | use diamond_types::list::operation::TextOperation; 4 | use rle::AppendRle; 5 | 6 | #[inline(always)] 7 | pub fn apply_edits_direct(doc: &mut ListCRDT, txns: &Vec) { 8 | let id = doc.get_or_create_agent_id("jeremy"); 9 | 10 | for (_i, txn) in txns.iter().enumerate() { 11 | for TestPatch(pos, del_span, ins_content) in &txn.patches { 12 | if *del_span > 0 { 13 | // doc.delete(id, *pos .. *pos + *del_span); 14 | doc.delete_without_content(id, *pos .. *pos + *del_span); 15 | } 16 | 17 | if !ins_content.is_empty() { 18 | doc.insert(id, *pos, ins_content); 19 | } 20 | } 21 | } 22 | } 23 | 24 | #[inline(always)] 25 | pub fn apply_edits_push_merge(doc: &mut ListCRDT, txns: &Vec) { 26 | let id = doc.get_or_create_agent_id("jeremy"); 27 | 28 | for (_i, txn) in txns.iter().enumerate() { 29 | for TestPatch(pos, del_span, ins_content) in &txn.patches { 30 | let pos = *pos; 31 | let del_span = *del_span; 32 | 33 | if del_span > 0 { 34 | doc.oplog.add_delete_without_content(id, pos..pos + del_span); 35 | } 36 | 37 | if !ins_content.is_empty() { 38 | doc.oplog.add_insert(id, pos, ins_content); 39 | } 40 | } 41 | } 42 | 43 | doc.branch.merge(&doc.oplog, &doc.oplog.local_frontier_ref()); 44 | } 45 | 46 | #[inline(always)] 47 | pub fn apply_grouped(doc: &mut ListCRDT, txns: &Vec) { 48 | let id = doc.get_or_create_agent_id("jeremy"); 49 | 50 | let mut ops: Vec = Vec::new(); 51 | 52 | for (_i, txn) in txns.iter().enumerate() { 53 | for TestPatch(pos, del_span, ins_content) in &txn.patches { 54 | if *del_span > 0 { 55 | ops.push(TextOperation::new_delete(*pos .. *pos + *del_span)); 56 | } 57 | 58 | if !ins_content.is_empty() { 59 | ops.push(TextOperation::new_insert(*pos, ins_content)); 60 | } 61 | } 62 | } 63 | 64 | doc.apply_local_operations(id, &ops); 65 | // doc.branch.merge(&doc.oplog, &doc.oplog.local_version()); 66 | } 67 | 68 | #[inline(always)] 69 | pub fn as_grouped_ops_rle(txns: &Vec) -> Vec { 70 | let mut ops: Vec = Vec::new(); 71 | 72 | for (_i, txn) in txns.iter().enumerate() { 73 | for TestPatch(pos, del_span, ins_content) in &txn.patches { 74 | 75 | if *del_span > 0 { 76 | ops.push_rle(TextOperation::new_delete(*pos .. *pos + *del_span)); 77 | } 78 | 79 | if !ins_content.is_empty() { 80 | ops.push_rle(TextOperation::new_insert(*pos, ins_content)); 81 | } 82 | } 83 | } 84 | 85 | ops 86 | } 87 | 88 | #[inline(always)] 89 | pub fn apply_ops(doc: &mut ListCRDT, ops: &[TextOperation]) { 90 | let id = doc.get_or_create_agent_id("jeremy"); 91 | doc.apply_local_operations(id, &ops); 92 | } 93 | -------------------------------------------------------------------------------- /crates/crdt-testdata/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "crdt-testdata" 3 | version = "0.0.0" 4 | authors = ["Seph Gentle "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | flate2 = { version = "1.0.33" } 9 | serde = { version = "1.0.136", features = ["derive"] } 10 | serde_json = "1.0.79" 11 | -------------------------------------------------------------------------------- /crates/crdt-testdata/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod nonlinear; 2 | 3 | // use std::time::SystemTime; 4 | use std::fs::File; 5 | use std::io::{BufReader, Read}; 6 | use flate2::bufread::GzDecoder; 7 | use serde::Deserialize; 8 | 9 | /// This file contains some simple helpers for loading test data. Its used by benchmarking and 10 | /// testing code. 11 | 12 | /// (position, delete length, insert content). 13 | #[derive(Debug, Clone, Deserialize)] 14 | pub struct TestPatch(pub usize, pub usize, pub String); 15 | 16 | #[derive(Debug, Clone, Deserialize)] 17 | pub struct TestTxn { 18 | // time: String, // ISO String. Unused. 19 | pub patches: Vec 20 | } 21 | 22 | #[derive(Debug, Clone, Deserialize)] 23 | pub struct TestData { 24 | #[serde(rename = "startContent")] 25 | pub start_content: String, 26 | #[serde(rename = "endContent")] 27 | pub end_content: String, 28 | 29 | pub txns: Vec, 30 | } 31 | 32 | impl TestData { 33 | pub fn len(&self) -> usize { 34 | self.txns.iter() 35 | .map(|txn| { txn.patches.len() }) 36 | .sum::() 37 | } 38 | 39 | pub fn len_keystrokes(&self) -> usize { 40 | self.txns.iter() 41 | .flat_map(|txn| { txn.patches.iter() }) 42 | .map(|TestPatch(_pos, del, ins)| { 43 | *del + ins.chars().count() 44 | }) 45 | .sum() 46 | } 47 | 48 | pub fn is_empty(&self) -> bool { 49 | !self.txns.iter().any(|txn| !txn.patches.is_empty()) 50 | } 51 | } 52 | 53 | // TODO: Make a try_ version of this method, which returns an appropriate Error object. 54 | pub fn load_testing_data(filename: &str) -> TestData { 55 | // let start = SystemTime::now(); 56 | // let mut file = File::open("benchmark_data/automerge-paper.json.gz").unwrap(); 57 | let file = File::open(filename).unwrap(); 58 | 59 | let reader = BufReader::new(file); 60 | // We could pass the GzDecoder straight to serde, but it makes it way slower to parse for 61 | // some reason. 62 | let mut reader = GzDecoder::new(reader); 63 | let mut raw_json = vec!(); 64 | reader.read_to_end(&mut raw_json).unwrap(); 65 | 66 | // println!("uncompress time {}", start.elapsed().unwrap().as_millis()); 67 | 68 | // let start = SystemTime::now(); 69 | let data: TestData = serde_json::from_reader(raw_json.as_slice()).unwrap(); 70 | // println!("JSON parse time {}", start.elapsed().unwrap().as_millis()); 71 | 72 | data 73 | } 74 | 75 | #[cfg(test)] 76 | mod tests { 77 | use crate::load_testing_data; 78 | 79 | #[test] 80 | fn it_works() { 81 | let data = load_testing_data("../../benchmark_data/sveltecomponent.json.gz"); 82 | assert!(data.txns.len() > 0); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /crates/crdt-testdata/src/nonlinear.rs: -------------------------------------------------------------------------------- 1 | // Nonlinear data has some different fields. 2 | 3 | use std::fs::File; 4 | use std::io::BufReader; 5 | use crate::TestPatch; 6 | use serde::Deserialize; 7 | 8 | #[derive(Debug, Clone, Deserialize)] 9 | pub struct NLId { 10 | pub agent: u32, 11 | pub seq: usize, 12 | } 13 | 14 | #[derive(Debug, Clone, Deserialize)] 15 | pub struct NLPatch { 16 | pub id: NLId, 17 | pub parents: Vec, 18 | pub timestamp: String, 19 | pub patch: TestPatch, 20 | } 21 | 22 | #[derive(Debug, Clone, Deserialize)] 23 | pub struct NLDataset { 24 | #[serde(rename = "startContent")] 25 | pub start_content: String, 26 | pub ops: Vec, 27 | } 28 | 29 | 30 | pub fn load_nl_testing_data(filename: &str) -> NLDataset { 31 | let file = File::open(filename).unwrap(); 32 | let reader = BufReader::new(file); 33 | 34 | // TODO: Add gzip compression. 35 | serde_json::from_reader(reader).unwrap() 36 | } 37 | 38 | // #[test] 39 | // fn foo() { 40 | // let d = load_nl_testing_data("/home/seph/src/crdt-benchmarks/xml/out/G1-3.json"); 41 | // dbg!(&d); 42 | // } -------------------------------------------------------------------------------- /crates/dt-cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dt-cli" 3 | version = "0.2.0" 4 | edition = "2021" 5 | description = "CLI for interacting with diamond-types data" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [[bin]] 10 | name = "dt" 11 | path = "src/main.rs" 12 | 13 | [dependencies] 14 | diamond-types = { path = "../..", features = ["serde", "dot_export", "merge_conflict_checks", "gen_test_data"] } 15 | clap = { version = "4.2.4", features = ["derive"] } 16 | similar = "2.1.0" 17 | rand = "0.8.5" 18 | serde = "1.0.136" 19 | serde_json = "1.0.79" 20 | anyhow = "1.0.71" 21 | smallvec = { version = "2.0.0-alpha.6", features = ["serde"] } 22 | smartstring = "1.0.1" 23 | chrono = { version = "0.4.24", default-features = false, features = ["alloc", "std", "serde"] } 24 | rle = { path = "../rle" } 25 | 26 | git2 = { version = "0.17.1", optional = true } 27 | indicatif = { version = "0.17.3", optional = true } 28 | 29 | [features] 30 | default = [] 31 | git = ["dep:git2", "dep:indicatif"] 32 | -------------------------------------------------------------------------------- /crates/dt-cli/src/dot.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | use std::ffi::OsString; 3 | /// This file contains some helper code to create SVG images from time DAGs to show whats going on 4 | /// in a document. 5 | /// 6 | /// It was mostly made as an aide to debugging. Compilation is behind a feature flag (dot_export) 7 | 8 | use std::fmt::{Display, Formatter}; 9 | use std::io::Write as _; 10 | use std::process::{Command, Stdio}; 11 | 12 | // pub fn name_of(time: LV) -> String { 13 | // if time == LV::MAX { panic!("Should not see ROOT_TIME here"); } 14 | // 15 | // format!("{}", time) 16 | // } 17 | 18 | #[derive(Debug)] 19 | struct DotError; 20 | 21 | impl Display for DotError { 22 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 23 | f.write_str("dot command failed with an error") 24 | } 25 | } 26 | 27 | impl Error for DotError {} 28 | 29 | pub fn generate_svg_with_dot(dot_content: String, dot_path: Option) -> Result> { 30 | let dot_path = dot_path.unwrap_or_else(|| "dot".into()); 31 | let mut child = Command::new(dot_path) 32 | // .arg("-Tpng") 33 | .arg("-Tsvg") 34 | .stdin(Stdio::piped()) 35 | .stdout(Stdio::piped()) 36 | .stderr(Stdio::piped()) 37 | .spawn()?; 38 | 39 | let mut stdin = child.stdin.take().unwrap(); 40 | // Spawn is needed here to prevent a potential deadlock. See: 41 | // https://doc.rust-lang.org/std/process/index.html#handling-io 42 | std::thread::spawn(move || { 43 | stdin.write_all(dot_content.as_bytes()).unwrap(); 44 | }); 45 | 46 | let out = child.wait_with_output()?; 47 | 48 | // Pipe stderr. 49 | std::io::stderr().write_all(&out.stderr)?; 50 | 51 | if out.status.success() { 52 | Ok(String::from_utf8(out.stdout)?) 53 | } else { 54 | // May as well pipe stdout too. 55 | std::io::stdout().write_all(&out.stdout)?; 56 | Err(DotError.into()) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /crates/dt-swift/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dt-swift" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | #build = "build.rs" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [lib] 11 | crate-type = ["staticlib"] 12 | 13 | [build-dependencies] 14 | swift-bridge-build = "0.1.35" 15 | 16 | [dependencies] 17 | swift-bridge = "0.1.35" 18 | diamond-types = { path = "../..", features = ["serde", "wchar_conversion"] } 19 | rand = { version = "0.8.5" } -------------------------------------------------------------------------------- /crates/dt-swift/build.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | fn main() { 4 | let out_dir = PathBuf::from("./generated"); 5 | 6 | let bridges = vec!["src/lib.rs"]; 7 | for path in bridges.iter() { 8 | println!("cargo:rerun-if-changed={}", path); 9 | } 10 | 11 | swift_bridge_build::parse_bridges(bridges) 12 | .write_all_concatenated(out_dir, env!("CARGO_PKG_NAME")); 13 | } 14 | -------------------------------------------------------------------------------- /crates/dt-swift/src/lib.rs: -------------------------------------------------------------------------------- 1 | use diamond_types::AgentId; 2 | use diamond_types::list::{ListCRDT as InnerListCRDT}; 3 | use diamond_types::list::encoding::ENCODE_FULL; 4 | use rand::{distributions::Alphanumeric, Rng}; 5 | 6 | #[swift_bridge::bridge] 7 | mod ffi { 8 | extern "Rust" { 9 | type ListCRDT; 10 | 11 | #[swift_bridge(init)] 12 | fn new() -> ListCRDT; 13 | // #[swift_bridge(init)] 14 | // fn new(agent_name: Option<&str>) -> ListCRDT; 15 | 16 | pub fn replace_wchar(&mut self, wchar_pos: usize, remove: usize, ins: &str); 17 | 18 | pub fn encode(&self) -> Vec; 19 | pub fn save(&self, path: &str); 20 | 21 | pub fn to_string(&self) -> String; 22 | 23 | fn decode(bytes: &[u8]) -> ListCRDT; 24 | fn load_or_new(path: &str) -> ListCRDT; 25 | } 26 | 27 | } 28 | 29 | 30 | pub struct ListCRDT { 31 | inner: InnerListCRDT, 32 | agent_id: AgentId, 33 | } 34 | 35 | 36 | fn create_agent(crdt: &mut InnerListCRDT) -> AgentId { 37 | let s: String = rand::thread_rng() 38 | .sample_iter(&Alphanumeric) 39 | .take(8) 40 | .map(char::from) 41 | .collect(); 42 | crdt.get_or_create_agent_id(&s) 43 | } 44 | // fn get_agent(crdt: &mut InnerListCRDT, agent_name: Option<&str>) -> AgentId { 45 | // agent_name.map(|name| { 46 | // crdt.get_or_create_agent_id(name) 47 | // }).unwrap_or_else(|| { 48 | // let s: String = rand::thread_rng() 49 | // .sample_iter(&Alphanumeric) 50 | // .take(8) 51 | // .map(char::from) 52 | // .collect(); 53 | // crdt.get_or_create_agent_id(&s) 54 | // }) 55 | // } 56 | 57 | pub fn decode(bytes: &[u8]) -> ListCRDT { 58 | let mut inner = InnerListCRDT::load_from(bytes).expect("Failed to decode DT object from bytes"); 59 | let agent_id = create_agent(&mut inner); 60 | ListCRDT { inner, agent_id } 61 | } 62 | fn load_or_new(path: &str) -> ListCRDT { 63 | // TODO: Only make a new file if the error is ENOENT 64 | std::fs::read(path) 65 | .map(|data| decode(&data)) 66 | .unwrap_or_else(|err| { 67 | eprintln!("Could not read file: {err}"); 68 | ListCRDT::new() 69 | }) 70 | } 71 | 72 | impl ListCRDT { 73 | pub fn new() -> Self { 74 | let mut inner = InnerListCRDT::new(); 75 | let agent_id = create_agent(&mut inner); 76 | 77 | Self { inner, agent_id } 78 | } 79 | 80 | // pub fn ins_unicode(&mut self, pos: usize, content: &str) -> usize { 81 | // self.inner.insert(self.agent_id, pos, content) 82 | // // let parents: LocalVersion = self.inner.local_version_ref().into(); 83 | // // self.inner.add_insert_at(self.agent_id.unwrap(), &parents, pos, content) 84 | // } 85 | 86 | pub fn replace_wchar(&mut self, wchar_pos: usize, remove: usize, ins: &str) { 87 | if remove > 0 { 88 | self.inner.delete_at_wchar(self.agent_id, wchar_pos..wchar_pos + remove); 89 | } 90 | if !ins.is_empty() { 91 | self.inner.insert_at_wchar(self.agent_id, wchar_pos, ins); 92 | } 93 | } 94 | 95 | pub fn encode(&self) -> Vec { 96 | self.inner.oplog.encode(&ENCODE_FULL) 97 | } 98 | 99 | pub fn save(&self, path: &str) { 100 | let data = self.encode(); 101 | std::fs::write(path, data).unwrap() 102 | } 103 | 104 | pub fn to_string(&self) -> String { 105 | self.inner.branch.content().to_string() 106 | } 107 | } 108 | 109 | // pub struct Branch(DTBranch); 110 | // 111 | // pub struct OpLog { 112 | // inner: DTOpLog, 113 | // agent_id: Option, 114 | // } 115 | -------------------------------------------------------------------------------- /crates/dt-wasm/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | Cargo.lock 4 | bin/ 5 | pkg/ 6 | wasm-pack.log 7 | -------------------------------------------------------------------------------- /crates/dt-wasm/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dt-wasm" 3 | version = "0.1.0" 4 | authors = ["Seph Gentle "] 5 | edition = "2021" 6 | license = "ISC OR Apache-2.0" 7 | description = "Javascript wrapper for diamond-types" 8 | repository = "https://github.com/josephg/diamond-types" 9 | 10 | [lib] 11 | crate-type = ["cdylib", "rlib"] 12 | 13 | [features] 14 | default = ["console_error_panic_hook"] 15 | 16 | [dependencies] 17 | wasm-bindgen = "0.2.79" 18 | serde-wasm-bindgen = "0.4.2" 19 | smallvec = { version = "1.8.0", features = ["union"] } 20 | serde = "1.0.136" 21 | 22 | # Needed for jumprope. 23 | getrandom = { version = "0.2.4", features = ["js"] } 24 | 25 | # The `console_error_panic_hook` crate provides better debugging of panics by 26 | # logging them with `console.error`. This is great for development, but requires 27 | # all the `std::fmt` and `std::panicking` infrastructure, so isn't great for 28 | # code size when deploying. 29 | console_error_panic_hook = { version = "0.1.7", optional = true } 30 | 31 | # `wee_alloc` is a tiny allocator for wasm that is only ~1K in code size 32 | # compared to the default allocator's ~10K. It is slower than the default 33 | # allocator, however. 34 | # 35 | # Unfortunately, `wee_alloc` requires nightly Rust when targeting wasm for now. 36 | #wee_alloc = { version = "0.4.5", optional = true } 37 | 38 | #diamond-types = { version = "0.1.0", features = ["serde"] } 39 | #diamond-core = { path = "../diamond-core" } 40 | diamond-types = { path = "../..", default-features = false, features = ["lz4", "serde", "wchar_conversion"] } 41 | 42 | 43 | [dev-dependencies] 44 | wasm-bindgen-test = "0.3.13" 45 | 46 | #[package.metadata.wasm-pack.profile.release] 47 | #wasm-opt = false 48 | 49 | [package.metadata.wasm-pack.profile.profiling] 50 | wasm-opt = false -------------------------------------------------------------------------------- /crates/dt-wasm/README.md: -------------------------------------------------------------------------------- 1 | # Diamond types JS wrapper library 2 | 3 | This is a javascript + WASM wrapper around [diamond types](https://github.com/josephg/diamond-types). 4 | 5 | Note the API is still in flux and will change. 6 | 7 | This library is published as two separate modules: `diamond-types-web` and `diamond-types-node`. 8 | 9 | > TODO: Fill me in! 10 | 11 | Example usage: 12 | 13 | ```javascript 14 | // Nodejs version: 15 | const {Doc, Branch, OpLog} = require('diamond-types-node') 16 | 17 | // console.log(new Doc().getRemoteVersion()) 18 | 19 | let oplog = new OpLog("seph") 20 | oplog.ins(0, "hi there") 21 | 22 | let oplog2 = oplog.clone() 23 | 24 | let v = oplog.getLocalVersion() 25 | console.log('v', v, oplog.localToRemoteVersion(v)) 26 | oplog.del(1, 2) 27 | let patch = oplog.getPatchSince(v) 28 | 29 | console.log('patch', patch) 30 | 31 | let result_v = oplog2.addFromBytes(patch) 32 | console.log('mergebytes returned', result_v) 33 | console.log(oplog.getOps()) 34 | console.log(oplog2.getOps()) 35 | 36 | console.log(oplog2.localToRemoteVersion([2, 3])) 37 | ``` 38 | 39 | ### Building 40 | 41 | ``` 42 | $ wasm-pack build --target nodejs 43 | ``` 44 | 45 | See example.js for a simple usage example. Note the API is in flux and will change. 46 | 47 | 48 | # License 49 | 50 | ISC -------------------------------------------------------------------------------- /crates/dt-wasm/example.mjs: -------------------------------------------------------------------------------- 1 | import {Console} from 'console' 2 | import fs from 'fs' 3 | 4 | import {default as init, Branch, Doc, OpLog} from './pkg/diamond_wasm.js' 5 | // import * as x from './pkg/diamond_wasm.js' 6 | 7 | global.console = new Console({ 8 | stdout: process.stdout, 9 | stderr: process.stderr, 10 | inspectOptions: {depth: null} 11 | }) 12 | 13 | const bytes = fs.readFileSync('pkg/diamond_wasm_bg.wasm') 14 | const wasmModule = new WebAssembly.Module(bytes) 15 | const wasmReady = init(wasmModule) 16 | 17 | 18 | 19 | ;(async () => { 20 | 21 | await wasmReady 22 | console.log('wasm init ok') 23 | 24 | let oplog = new OpLog("seph") 25 | oplog.ins(0, "hi there") 26 | 27 | let oplog2 = oplog.clone() 28 | 29 | let v = oplog.getLocalVersion() 30 | console.log('v', v, oplog.localToRemoteVersion(v)) 31 | oplog.del(1, 2) 32 | let patch = oplog.getPatchSince(v) 33 | 34 | console.log('patch', patch) 35 | 36 | let result_v = oplog2.addFromBytes(patch) 37 | console.log('mergebytes returned', result_v) 38 | console.log(oplog.getOps()) 39 | console.log(oplog2.getOps()) 40 | 41 | console.log(oplog2.localToRemoteVersion([2, 3])) 42 | 43 | // let oplog3 = new OpLog() 44 | // oplog3.apply_op({ tag: 'Ins', start: 0, end: 8, fwd: true, content: 'yooo' }) 45 | // console.log(oplog3.getOps()) 46 | 47 | 48 | // console.log(new OpLog().toBytes()) 49 | 50 | console.log('\n\n') 51 | let oplog3 = new OpLog() 52 | oplog3.setAgent('b') 53 | oplog3.ins(0, 'BBB', []) 54 | oplog3.setAgent('a') 55 | oplog3.ins(0, 'AAA', []) 56 | console.log('ops', oplog3.getOps()) 57 | // console.log(oplog3.getXFSince([])) 58 | console.log('xf ops', oplog3.getXF()) 59 | console.log("history", oplog3.getHistory()) 60 | 61 | console.log('\n\n') 62 | let oplog4 = new OpLog('seph') 63 | let t = oplog4.ins(0, 'aaa') 64 | // And double delete 65 | oplog4.setAgent('a') 66 | oplog4.del(0, 2, [t]) 67 | oplog4.setAgent('b') 68 | oplog4.del(1, 2, [t]) 69 | console.log('ops', oplog4.getOps()) 70 | // console.log(oplog4.getXFSince([])) 71 | console.log('xf ops', oplog4.getXF()) 72 | 73 | console.log("history", oplog4.getHistory()) 74 | })() 75 | 76 | 77 | // const {Console} = require('console') 78 | 79 | // const {Branch, OpLog} = import('./pkg/diamond_wasm.js') 80 | 81 | // console.log(Branch, OpLog) 82 | // 83 | // const ops = new OpLog() 84 | // let t = ops.ins(0, "hi there") 85 | // console.log(t) 86 | // let t2 = ops.del(3, 3) 87 | // 88 | // console.log("local branch", ops.getLocalVersion()) 89 | // console.log("frontier", ops.getFrontier()) 90 | // console.log("ops", ops.getOps()) 91 | // console.log("history", ops.getHistory()) 92 | // 93 | // console.log("bytes", ops.toBytes()) 94 | // 95 | // oplog2 = OpLog.fromBytes(ops.toBytes()) 96 | // console.log("ops", oplog2.getOps()) 97 | 98 | 99 | // const branch = new Branch() 100 | // branch.merge(ops, t) 101 | // console.log('branch', `"${branch.get()}"`) 102 | // console.log("branch branch", branch.getFrontier()) 103 | 104 | // const c2 = Checkout.all(ops) 105 | // console.log(c2.get()) 106 | 107 | 108 | 109 | // const ops2 = new OpLog() 110 | // ops2.ins(0, "aaa") 111 | // ops2.ins(0, "bbb", []) 112 | // 113 | // const checkout2 = Checkout.all(ops2) 114 | // console.log(checkout2.get()) 115 | // console.log("checkout branch", checkout2.getBranch()) 116 | 117 | // console.log(ops2.getOps()) 118 | // console.log(ops2.getHistory()) 119 | 120 | -------------------------------------------------------------------------------- /crates/dt-wasm/foo.mjs: -------------------------------------------------------------------------------- 1 | import {Console} from 'console' 2 | import fs from 'fs' 3 | 4 | import {default as init, Branch, Doc, OpLog} from './pkg/diamond_wasm.js' 5 | // import * as x from './pkg/diamond_wasm.js' 6 | 7 | global.console = new Console({ 8 | stdout: process.stdout, 9 | stderr: process.stderr, 10 | inspectOptions: {depth: null} 11 | }) 12 | 13 | const bytes = fs.readFileSync('pkg/diamond_wasm_bg.wasm') 14 | const wasmModule = new WebAssembly.Module(bytes) 15 | const wasmReady = init(wasmModule) 16 | 17 | 18 | 19 | ;(async () => { 20 | 21 | await wasmReady 22 | console.log('wasm init ok') 23 | 24 | let x = [68,77,78,68,84,89,80,83,0,1,224,1,3,221,1,12,52,111,114,55,75,56,78,112,52,109,122,113,12,90,77,80,70,45,69,49,95,116,114,114,74,12,68,80,84,95,104,99,107,75,121,55,102,77,12,82,56,108,87,77,99,112,54,76,68,99,83,12,53,98,78,79,116,82,85,56,120,88,113,83,12,100,85,101,81,83,77,66,54,122,45,72,115,12,50,105,105,80,104,101,116,101,85,107,57,49,12,108,65,71,75,68,90,68,53,108,111,99,75,12,78,113,55,109,65,70,55,104,67,56,52,122,12,116,51,113,52,84,101,121,73,76,85,54,53,12,120,95,120,51,68,95,105,109,81,100,78,115,12,102,120,103,87,90,100,82,111,105,108,73,99,12,115,87,67,73,67,97,78,100,68,65,77,86,12,110,100,56,118,55,74,79,45,114,81,122,45,12,110,85,69,75,69,73,53,81,49,49,45,83,12,120,97,55,121,102,81,88,98,45,120,54,87,12,85,116,82,100,98,71,117,106,57,49,98,49,10,7,12,2,0,0,13,1,4,20,157,2,24,182,1,0,13,174,1,4,120,100,102,120,120,102,100,115,49,120,120,121,122,113,119,101,114,115,100,102,115,100,115,100,97,115,100,115,100,115,100,115,100,97,115,100,97,115,100,113,119,101,119,113,101,119,113,119,107,106,107,106,107,106,107,107,106,107,106,107,108,106,108,107,106,108,107,106,108,107,106,101,101,114,108,106,107,114,101,108,107,116,101,114,116,101,111,114,106,116,111,105,101,106,114,116,111,105,119,106,100,97,98,99,49,49,49,57,49,98,115,110,102,103,104,102,100,103,104,100,102,103,104,100,103,104,100,102,103,104,100,102,103,104,100,107,106,102,108,107,115,100,106,102,108,115,59,107,106,107,108,106,59,107,106,107,106,107,106,59,107,106,108,59,107,106,59,107,108,106,107,106,108,25,2,219,2,21,44,2,3,4,1,6,4,8,1,10,1,12,10,14,1,16,1,18,1,20,4,22,4,24,18,26,99,28,58,30,4,28,1,30,1,32,3,34,2,32,1,34,23,32,39,22,31,81,175,1,21,177,2,239,4,77,169,3,223,6,107,33,79,9,0,26,47,3,0,19,3,18,42,177,1,187,2,43,23,19,211,1,1,1,8,3,10,4,1,8,2,6,8,1,8,22,4,39,96,100,4,142,143,169,235] 25 | Doc.fromBytes(x, "asdf") 26 | 27 | })() 28 | 29 | 30 | // const {Console} = require('console') 31 | 32 | // const {Branch, OpLog} = import('./pkg/diamond_wasm.js') 33 | 34 | // console.log(Branch, OpLog) 35 | // 36 | // const ops = new OpLog() 37 | // let t = ops.ins(0, "hi there") 38 | // console.log(t) 39 | // let t2 = ops.del(3, 3) 40 | // 41 | // console.log("local branch", ops.getLocalVersion()) 42 | // console.log("frontier", ops.getFrontier()) 43 | // console.log("ops", ops.getOps()) 44 | // console.log("history", ops.txns()) 45 | // 46 | // console.log("bytes", ops.toBytes()) 47 | // 48 | // oplog2 = OpLog.fromBytes(ops.toBytes()) 49 | // console.log("ops", oplog2.getOps()) 50 | 51 | 52 | // const branch = new Branch() 53 | // branch.merge(ops, t) 54 | // console.log('branch', `"${branch.get()}"`) 55 | // console.log("branch branch", branch.getFrontier()) 56 | 57 | // const c2 = Checkout.all(ops) 58 | // console.log(c2.get()) 59 | 60 | 61 | 62 | // const ops2 = new OpLog() 63 | // ops2.ins(0, "aaa") 64 | // ops2.ins(0, "bbb", [-1]) 65 | // 66 | // const checkout2 = Checkout.all(ops2) 67 | // console.log(checkout2.get()) 68 | // console.log("checkout branch", checkout2.getBranch()) 69 | 70 | // console.log(ops2.getOps()) 71 | // console.log(ops2.txns()) 72 | 73 | -------------------------------------------------------------------------------- /crates/dt-wasm/src/utils.rs: -------------------------------------------------------------------------------- 1 | pub fn set_panic_hook() { 2 | // When the `console_error_panic_hook` feature is enabled, we can call the 3 | // `set_panic_hook` function at least once during initialization, and then 4 | // we will get better error messages if our code ever panics. 5 | // 6 | // For more details see 7 | // https://github.com/rustwasm/console_error_panic_hook#readme 8 | #[cfg(feature = "console_error_panic_hook")] 9 | console_error_panic_hook::set_once(); 10 | } 11 | -------------------------------------------------------------------------------- /crates/dt-wasm/tests/web.rs: -------------------------------------------------------------------------------- 1 | //! Test suite for the Web and headless browsers. 2 | 3 | #![cfg(target_arch = "wasm32")] 4 | 5 | extern crate wasm_bindgen_test; 6 | use wasm_bindgen_test::*; 7 | 8 | wasm_bindgen_test_configure!(run_in_browser); 9 | 10 | #[wasm_bindgen_test] 11 | fn pass() { 12 | assert_eq!(1 + 1, 2); 13 | } 14 | -------------------------------------------------------------------------------- /crates/rle/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rle" 3 | version = "0.2.0" 4 | authors = ["Seph Gentle "] 5 | edition = "2021" 6 | license = "ISC OR Apache-2.0" 7 | description = "Simple utilities for run-length encoded data" 8 | repository = "https://github.com/josephg/diamond-types" 9 | 10 | [dependencies] 11 | smallvec = { version = "2.0.0-alpha.6", optional = true } 12 | -------------------------------------------------------------------------------- /crates/rle/README.md: -------------------------------------------------------------------------------- 1 | # RLE (run length encoding) tools 2 | 3 | This small utility crate contains some common traits for interacting with run-length encoded content. The most important trait here is SplitableSpan, which is used extensively throughout [diamond types](https://github.com/josephg/diamond-types). -------------------------------------------------------------------------------- /crates/rle/src/append_rle.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "smallvec")] 2 | use smallvec::SmallVec; 3 | 4 | use crate::MergableSpan; 5 | 6 | pub trait AppendRle { 7 | /// Push a new item to this list-like object. If the passed item can be merged into the 8 | /// last item in the list, do so instead of inserting a new item. 9 | /// 10 | /// Returns true if the item was merged into the previous last item, false if it was inserted. 11 | fn push_rle(&mut self, item: T) -> bool; 12 | 13 | /// Push a new item to the end of this list-like object. If the passed object can be merged 14 | /// to the front of the previously last item, do so. This is useful for appending to a list 15 | /// which is sorted in reverse. 16 | fn push_reversed_rle(&mut self, item: T) -> bool; 17 | 18 | /// Extend the item by RLE-compacting the incoming iterator. 19 | fn extend_rle>(&mut self, iter: I) { 20 | // TODO: Consider only calling push_rle until we can't append any more. 21 | for item in iter { 22 | self.push_rle(item); 23 | } 24 | } 25 | } 26 | 27 | // Apparently the cleanest way to do this DRY is using macros. 28 | impl AppendRle for Vec { 29 | fn push_rle(&mut self, item: T) -> bool { 30 | if let Some(v) = self.last_mut() { 31 | if v.can_append(&item) { 32 | v.append(item); 33 | return true; 34 | } 35 | } 36 | 37 | self.push(item); 38 | false 39 | } 40 | 41 | fn push_reversed_rle(&mut self, item: T) -> bool { 42 | if let Some(v) = self.last_mut() { 43 | if item.can_append(v) { 44 | v.prepend(item); 45 | return true; 46 | } 47 | } 48 | 49 | self.push(item); 50 | false 51 | } 52 | } 53 | 54 | #[cfg(feature = "smallvec")] 55 | impl AppendRle for SmallVec where T: MergableSpan { 56 | fn push_rle(&mut self, item: T) -> bool { 57 | // debug_assert!(item.len() > 0); 58 | 59 | if let Some(v) = self.last_mut() { 60 | if v.can_append(&item) { 61 | v.append(item); 62 | return true; 63 | } 64 | } 65 | 66 | self.push(item); 67 | false 68 | } 69 | 70 | fn push_reversed_rle(&mut self, item: T) -> bool { 71 | // debug_assert!(item.len() > 0); 72 | 73 | if let Some(v) = self.last_mut() { 74 | if item.can_append(v) { 75 | v.prepend(item); 76 | return true; 77 | } 78 | } 79 | 80 | self.push(item); 81 | false 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /crates/rle/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Debug; 2 | 3 | pub use append_rle::AppendRle; 4 | pub use splitable_span::*; 5 | pub use merge_iter::*; 6 | use std::ops::Range; 7 | 8 | mod splitable_span; 9 | mod merge_iter; 10 | mod append_rle; 11 | pub mod zip; 12 | pub mod take_max_iter; 13 | pub mod intersect; 14 | pub mod rlerun; 15 | // mod gapbuffer; 16 | // pub mod iter_ctx; 17 | 18 | pub use rlerun::{RleRun, RleDRun}; 19 | 20 | pub trait Searchable { 21 | type Item: Copy + Debug; 22 | 23 | // This is strictly unnecessary given truncate(), but it makes some code cleaner. 24 | // fn truncate_keeping_right(&mut self, at: usize) -> Self; 25 | 26 | /// Checks if the entry contains the specified item. If it does, returns the offset into the 27 | /// item. 28 | fn get_offset(&self, loc: Self::Item) -> Option; 29 | 30 | // I'd use Index for this but the index trait returns a reference. 31 | // fn at_offset(&self, offset: usize) -> Self::Item; 32 | fn at_offset(&self, offset: usize) -> Self::Item; 33 | } 34 | 35 | pub trait HasRleKey { 36 | fn rle_key(&self) -> usize; 37 | } 38 | 39 | impl HasRleKey for &T where T: HasRleKey { 40 | fn rle_key(&self) -> usize { 41 | (*self).rle_key() 42 | } 43 | } 44 | 45 | impl HasRleKey for Range { 46 | fn rle_key(&self) -> usize { 47 | self.start 48 | } 49 | } 50 | 51 | impl HasRleKey for Range { 52 | fn rle_key(&self) -> usize { 53 | self.start as _ 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /crates/rle/src/merge_iter.rs: -------------------------------------------------------------------------------- 1 | use crate::MergableSpan; 2 | 3 | /// This is an iterator composer which wraps any iterator over a SplitableSpan to become an 4 | /// iterator over those same items in run-length order. 5 | 6 | #[derive(Debug, Clone)] 7 | pub struct MergeIter { 8 | next: Option, 9 | iter: I 10 | } 11 | 12 | pub fn merge_items(iter: I) -> MergeIter { 13 | MergeIter::new(iter) 14 | } 15 | pub fn merge_items_rev(iter: I) -> MergeIter { 16 | MergeIter::new(iter) 17 | } 18 | 19 | impl MergeIter { 20 | pub fn new(iter: I) -> Self { 21 | Self { 22 | next: None, 23 | iter 24 | } 25 | } 26 | 27 | pub fn into_inner(self) -> I { 28 | self.iter 29 | } 30 | } 31 | 32 | impl Iterator for MergeIter 33 | where 34 | I: Iterator, 35 | X: MergableSpan 36 | { 37 | type Item = X; 38 | 39 | fn next(&mut self) -> Option { 40 | let mut this_val = match self.next.take() { 41 | Some(val) => val, 42 | None => { 43 | self.iter.next()? 44 | } 45 | }; 46 | 47 | for val in &mut self.iter { 48 | if FWD && this_val.can_append(&val) { 49 | this_val.append(val); 50 | } else if !FWD && val.can_append(&this_val) { 51 | this_val.prepend(val); 52 | } else { 53 | self.next = Some(val); 54 | break; 55 | } 56 | } 57 | 58 | Some(this_val) 59 | } 60 | 61 | fn size_hint(&self) -> (usize, Option) { 62 | let (lower, upper) = self.iter.size_hint(); 63 | (lower.min(1), upper) 64 | } 65 | } 66 | 67 | pub trait MergeableIterator: Iterator where Self: Sized { 68 | fn merge_spans(self) -> MergeIter; 69 | fn merge_spans_rev(self) -> MergeIter; 70 | } 71 | 72 | impl MergeableIterator for I 73 | where I: Iterator, X: MergableSpan, Self: Sized 74 | { 75 | fn merge_spans(self) -> MergeIter { 76 | MergeIter::new(self) 77 | } 78 | fn merge_spans_rev(self) -> MergeIter { 79 | MergeIter::new(self) 80 | } 81 | } 82 | 83 | #[cfg(test)] 84 | mod test { 85 | use std::ops::Range; 86 | use super::merge_items; 87 | use crate::{merge_items_rev, RleRun}; 88 | 89 | #[test] 90 | fn test_merge_iter() { 91 | let empty: Vec> = vec![]; 92 | assert_eq!(merge_items(empty.into_iter()).collect::>(), vec![]); 93 | 94 | let one = vec![RleRun { val: 5, len: 1 }]; 95 | assert_eq!(merge_items(one.into_iter()).collect::>(), vec![RleRun { val: 5, len: 1 }]); 96 | 97 | let two_split = vec![2u32..3, 5..10]; 98 | assert_eq!(merge_items(two_split.iter().cloned()).collect::>(), two_split); 99 | 100 | let two_merged = vec![2u32..5, 5..10]; 101 | assert_eq!(merge_items(two_merged.iter().cloned()).collect::>(), vec![2..10]); 102 | } 103 | 104 | #[test] 105 | fn test_merge_iter_rev() { 106 | // TODO: This is a bit of a crap test because it doesn't actually 107 | let empty: Vec> = vec![]; 108 | assert_eq!(merge_items_rev(empty.into_iter()).collect::>(), vec![]); 109 | 110 | let one = vec![5..6]; 111 | assert_eq!(merge_items_rev(one.into_iter()).collect::>(), vec![5u32..6]); 112 | 113 | let two_split = vec![5u32..10, 2..3]; 114 | assert_eq!(merge_items_rev(two_split.iter().cloned()).collect::>(), two_split); 115 | 116 | let two_merged = vec![5u32..10, 2..5]; 117 | assert_eq!(merge_items_rev(two_merged.iter().cloned()).collect::>(), vec![2..10]); 118 | } 119 | } -------------------------------------------------------------------------------- /crates/rle/src/rlerun.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Range; 2 | use crate::{HasLength, MergableSpan, SplitableSpanHelpers}; 3 | 4 | /// A splitablespan which contains a single element repeated N times. This is used in some examples. 5 | #[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, Default)] 6 | pub struct RleRun { 7 | pub val: T, 8 | pub len: usize, 9 | } 10 | 11 | impl RleRun { 12 | pub fn new(val: T, len: usize) -> Self { 13 | Self { val, len } 14 | } 15 | 16 | pub fn single(val: T) -> Self { 17 | Self { val, len: 1 } 18 | } 19 | } 20 | 21 | impl HasLength for RleRun { 22 | fn len(&self) -> usize { self.len } 23 | } 24 | 25 | impl SplitableSpanHelpers for RleRun { 26 | fn truncate_h(&mut self, at: usize) -> Self { 27 | let remainder = self.len - at; 28 | self.len = at; 29 | Self { val: self.val.clone(), len: remainder } 30 | } 31 | } 32 | 33 | impl MergableSpan for RleRun { 34 | fn can_append(&self, other: &Self) -> bool { 35 | self.val == other.val || self.len == 0 36 | } 37 | 38 | fn append(&mut self, other: Self) { 39 | self.len += other.len; 40 | self.val = other.val; // Needed when we use default() - which gives it a length of 0. 41 | } 42 | } 43 | 44 | /// Distinct RLE run. Each distinct run expresses some value between each (start, end) pair. 45 | #[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, Default)] 46 | pub struct RleDRun { 47 | pub start: usize, 48 | pub end: usize, 49 | pub val: T, 50 | } 51 | 52 | impl RleDRun { 53 | pub fn new(range: Range, val: T) -> Self { 54 | Self { 55 | start: range.start, 56 | end: range.end, 57 | val, 58 | } 59 | } 60 | } 61 | 62 | impl HasLength for RleDRun { 63 | fn len(&self) -> usize { self.end - self.start } 64 | } 65 | 66 | impl SplitableSpanHelpers for RleDRun { 67 | fn truncate_h(&mut self, at: usize) -> Self { 68 | let split_point = self.start + at; 69 | debug_assert!(split_point < self.end); 70 | let remainder = Self { start: split_point, end: self.end, val: self.val.clone() }; 71 | self.end = split_point; 72 | remainder 73 | } 74 | } 75 | 76 | impl MergableSpan for RleDRun { 77 | fn can_append(&self, other: &Self) -> bool { 78 | self.end == other.start && self.val == other.val 79 | } 80 | 81 | fn append(&mut self, other: Self) { 82 | self.end = other.end; 83 | } 84 | } 85 | 86 | 87 | // impl Searchable for RleDRun { 88 | // type Item = T; 89 | // 90 | // fn get_offset(&self, _loc: Self::Item) -> Option { 91 | // unimplemented!() 92 | // } 93 | // 94 | // fn at_offset(&self, offset: usize) -> Self::Item { 95 | // Some( 96 | // } 97 | // } 98 | -------------------------------------------------------------------------------- /crates/trace-alloc/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "trace-alloc" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | #[dependencies] 9 | #serde = { version = "1.0.198", optional = true, features = ["serde_derive"] } 10 | 11 | [features] 12 | memusage = [] 13 | #serde = ["dep:serde"] -------------------------------------------------------------------------------- /crates/trace-alloc/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This module provides a replacement allocator for use in testing code, so we can trace & track 2 | //! memory allocations in tests. 3 | //! 4 | //! This only used when compiled with the `memusage` feature flag is enabled. 5 | //! 6 | //! This code is not part of the standard diamond types API surface. It will be removed at some 7 | //! point (or moved into testing code). DO NOT DEPEND ON THIS. 8 | //! 9 | //! TODO: Make this not public (or move it into a private module). 10 | 11 | use std::alloc::{GlobalAlloc, Layout, System}; 12 | use std::cell::RefCell; 13 | 14 | #[derive(Debug, Clone, Copy, Default)] 15 | pub struct AllocStats { 16 | pub num_allocations: usize, 17 | pub current_allocated_bytes: usize, 18 | pub peak_allocated_bytes: usize, 19 | } 20 | 21 | thread_local! { 22 | // Pair of (num allocations, total bytes allocated). 23 | static ALLOCATED: RefCell = RefCell::default(); 24 | } 25 | // pub static ALLOCATED: AtomicUsize = AtomicUsize::new(0); 26 | 27 | pub struct TracingAlloc; 28 | 29 | unsafe impl GlobalAlloc for TracingAlloc { 30 | unsafe fn alloc(&self, layout: Layout) -> *mut u8 { 31 | // println!("{}", std::backtrace::Backtrace::force_capture()); 32 | let ret = System.alloc(layout); 33 | if !ret.is_null() { 34 | // ALLOCATED.fetch_add(layout.size(), Ordering::AcqRel); 35 | ALLOCATED.with(|s| { 36 | let mut r = s.borrow_mut(); 37 | r.num_allocations += 1; 38 | r.current_allocated_bytes += layout.size(); 39 | r.peak_allocated_bytes = r.peak_allocated_bytes.max(r.current_allocated_bytes); 40 | }); 41 | } 42 | ret 43 | } 44 | 45 | unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { 46 | // ALLOCATED.fetch_sub(layout.size(), Ordering::AcqRel); 47 | ALLOCATED.with_borrow_mut(|r| { 48 | r.num_allocations -= 1; 49 | // It should be impossible to wrap, but since this is debugging code we'll silently 50 | // ignore if that happens. 51 | r.current_allocated_bytes = r.current_allocated_bytes.saturating_sub(layout.size()); 52 | }); 53 | System.dealloc(ptr, layout); 54 | } 55 | 56 | // Eh, would be better to implement this but it'd be easy to mess this up. 57 | // unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { 58 | // } 59 | } 60 | 61 | #[allow(unused)] 62 | pub fn get_thread_num_allocations() -> usize { 63 | ALLOCATED.with(|s| { 64 | s.borrow().num_allocations 65 | }) 66 | } 67 | 68 | #[allow(unused)] 69 | pub fn get_thread_memory_usage() -> usize { 70 | ALLOCATED.with(|s| { 71 | s.borrow().current_allocated_bytes 72 | }) 73 | } 74 | 75 | #[allow(unused)] 76 | pub fn get_peak_memory_usage() -> usize { 77 | ALLOCATED.with(|s| { 78 | s.borrow().peak_allocated_bytes 79 | }) 80 | } 81 | 82 | #[allow(unused)] 83 | pub fn reset_peak_memory_usage() { 84 | ALLOCATED.with_borrow_mut(|s| { 85 | s.peak_allocated_bytes = s.current_allocated_bytes 86 | }); 87 | } 88 | 89 | // #[derive(Debug, Clone, Copy, Serialize)] 90 | // #[derive(Debug, Clone, Copy)] 91 | // pub struct MemUsage { 92 | // steady_state: usize, 93 | // peak: usize, 94 | // } 95 | 96 | // Returns (peak memory, resulting memory usage, R). 97 | pub fn measure_memusage R, R>(f: F) -> (usize, usize, R) { 98 | let before = get_thread_memory_usage(); 99 | reset_peak_memory_usage(); 100 | 101 | let result = f(); 102 | 103 | ( 104 | get_peak_memory_usage() - before, 105 | get_thread_memory_usage() - before, 106 | result 107 | ) 108 | } 109 | 110 | #[cfg(any(test, feature = "memusage"))] 111 | mod trace_alloc { 112 | use super::TracingAlloc; 113 | 114 | #[global_allocator] 115 | static A: TracingAlloc = TracingAlloc; 116 | } -------------------------------------------------------------------------------- /examples/print.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused)] 2 | 3 | use std::env; 4 | use diamond_types::list::{ListOpLog, encoding::EncodeOptions}; 5 | use rle::zip::rle_zip; 6 | 7 | fn print_stats_for_file(name: &str) { 8 | let contents = std::fs::read(name).unwrap(); 9 | println!("\n\nLoaded testing data from {} ({} bytes)", name, contents.len()); 10 | 11 | let oplog = ListOpLog::load_from(&contents).unwrap(); 12 | 13 | println!("\nOperations:"); 14 | for op in oplog.iter_ops() { 15 | println!("{:?}", op); 16 | } 17 | 18 | println!("\nHistory:"); 19 | for hist in oplog.iter_history() { 20 | println!("{:?}", hist); 21 | } 22 | 23 | println!("\nAgent assignment mappings:"); 24 | for m in oplog.iter_remote_mappings() { 25 | println!("{:?}", m); 26 | } 27 | 28 | // println!("\nTransformed operations:"); 29 | // for (origin_op, (span, xf_op)) in rle_zip(oplog.iter(), oplog.get_all_transformed_operations()) { 30 | // if let Some(xf_op) = xf_op { 31 | // if origin_op.span != xf_op.span { 32 | // let amt = xf_op.start() as isize - origin_op.start() as isize; 33 | // println!("{:?} moved by {} ({:?} {:?})", span, amt, origin_op.tag, origin_op.content); 34 | // // println!("{:?} moved {:?} -> {:?} {:?} {:?}", span, origin_op.span, xf_op.span, origin_op.tag, origin_op.content); 35 | // } 36 | // } else { 37 | // println!("{:?} double deleted", span); 38 | // } 39 | // } 40 | 41 | // for (span, op) in oplog.get_all_transformed_operations() { 42 | // println!("{:?} {:?}", span, op); 43 | // } 44 | 45 | // for c in oplog. 46 | 47 | // for x in rle_zip3( 48 | // oplog.iter_mappings(), 49 | // oplog.iter_history(), 50 | // oplog.iter() 51 | // ) { 52 | // println!("{:?}", x); 53 | // } 54 | 55 | // for x in rle_zip( 56 | // oplog.iter_history(), 57 | // oplog.iter() 58 | // ) { 59 | // println!("{:?}", x); 60 | // } 61 | 62 | println!(); 63 | oplog.encode(&EncodeOptions::patch().store_deleted_content(true).verbose(true)); 64 | } 65 | 66 | 67 | fn main() { 68 | let args = env::args(); 69 | let filename = args.last().unwrap_or_else(|| "node_nodecc.dt".into()); 70 | print_stats_for_file(&filename); 71 | } -------------------------------------------------------------------------------- /examples/profile.rs: -------------------------------------------------------------------------------- 1 | use std::hint::black_box; 2 | use crdt_testdata::{load_testing_data, TestPatch, TestTxn}; 3 | use diamond_types::list::{ListCRDT, ListOpLog}; 4 | 5 | pub fn apply_edits_direct(doc: &mut ListCRDT, txns: &Vec) { 6 | let id = doc.get_or_create_agent_id("jeremy"); 7 | 8 | for (_i, txn) in txns.iter().enumerate() { 9 | for TestPatch(pos, del_span, ins_content) in &txn.patches { 10 | if *del_span > 0 { 11 | doc.delete_without_content(id, *pos .. *pos + *del_span); 12 | } 13 | 14 | if !ins_content.is_empty() { 15 | doc.insert(id, *pos, ins_content); 16 | } 17 | } 18 | } 19 | } 20 | 21 | // This is a dirty addition for profiling. 22 | #[allow(unused)] 23 | fn profile_direct_editing() { 24 | let filename = "benchmark_data/automerge-paper.json.gz"; 25 | let test_data = load_testing_data(&filename); 26 | 27 | for _i in 0..300 { 28 | let mut doc = ListCRDT::new(); 29 | apply_edits_direct(&mut doc, &test_data.txns); 30 | assert_eq!(doc.len(), test_data.end_content.chars().count()); 31 | } 32 | } 33 | 34 | #[allow(unused)] 35 | fn profile_merge(name: &str, n: usize) { 36 | let contents = std::fs::read(&format!("benchmark_data/{name}.dt")).unwrap(); 37 | let oplog = ListOpLog::load_from(&contents).unwrap(); 38 | 39 | for _i in 0..n { 40 | // black_box(oplog.checkout_tip_old()); 41 | black_box(oplog.checkout_tip()); 42 | } 43 | } 44 | 45 | #[allow(unused)] 46 | fn profile_make_plan(name: &str, n: usize) { 47 | let contents = std::fs::read(&format!("benchmark_data/{name}.dt")).unwrap(); 48 | let oplog = ListOpLog::load_from(&contents).unwrap(); 49 | 50 | for _i in 0..n { 51 | oplog.dbg_bench_make_plan(); 52 | } 53 | } 54 | 55 | // RUSTFLAGS="-Cforce-frame-pointers=yes" cargo build --profile profiling --example profile 56 | fn main() { 57 | profile_merge("clownschool", 500); 58 | // profile_make_plan("clownschool", 2); 59 | // profile_merge("git-makefile", 200); 60 | // profile_merge("git-makefile", 1); 61 | // profile_merge("node_nodecc", 1); 62 | // profile_merge("clownschool", 1); 63 | } -------------------------------------------------------------------------------- /js/.mocharc.json: -------------------------------------------------------------------------------- 1 | { 2 | "spec": [ 3 | "dist/tests/**/*.js" 4 | ], 5 | "watch-files": [ 6 | "dist/src" 7 | ] 8 | } -------------------------------------------------------------------------------- /js/example.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert/strict') 2 | const dt = require('./dist') 3 | 4 | const db = dt.createDb() 5 | 6 | dt.localMapInsert(db, ['seph', 0], dt.ROOT, 'yo', {type: 'primitive', val: 123}) 7 | assert.deepEqual(dt.get(db), {yo: 123}) 8 | 9 | // concurrent changes 10 | dt.applyRemoteOp(db, { 11 | id: ['mike', 0], 12 | globalParents: [], 13 | crdtId: dt.ROOT, 14 | action: {type: 'map', localParents: [], key: 'yo', val: {type: 'primitive', val: 321}}, 15 | }) 16 | assert.deepEqual(dt.get(db), {yo: 123}) 17 | 18 | dt.applyRemoteOp(db, { 19 | id: ['mike', 1], 20 | globalParents: [['mike', 0], ['seph', 0]], 21 | crdtId: dt.ROOT, 22 | action: {type: 'map', localParents: [['mike', 0], ['seph', 0]], key: 'yo', val: {type: 'primitive', val: 1000}}, 23 | }) 24 | assert.deepEqual(dt.get(db), {yo: 1000}) 25 | 26 | // Set a value in an inner map 27 | const inner = dt.localMapInsert(db, ['seph', 1], dt.ROOT, 'stuff', {type: 'crdt', crdtKind: 'map'}) 28 | dt.localMapInsert(db, ['seph', 2], inner.id, 'cool', {type: 'primitive', val: 'definitely'}) 29 | assert.deepEqual(dt.get(db), {yo: 1000, stuff: {cool: 'definitely'}}) 30 | 31 | 32 | // Insert a set 33 | const innerSet = dt.localMapInsert(db, ['seph', 2], dt.ROOT, 'a set', {type: 'crdt', crdtKind: 'set'}) 34 | dt.localSetInsert(db, ['seph', 3], innerSet.id, {type: 'primitive', val: 'whoa'}) 35 | dt.localSetInsert(db, ['seph', 4], innerSet.id, {type: 'crdt', crdtKind: 'map'}) 36 | 37 | console.log('db', dt.get(db)) 38 | console.log('db', db) 39 | 40 | 41 | assert.deepEqual(db, dt.fromJSON(dt.toJSON(db))) 42 | // console.log(JSON.stringify(dt.toJSON(db))) -------------------------------------------------------------------------------- /js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dt-js", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "license": "MIT", 6 | "scripts": { 7 | "build": "tsc && browserify dist/client.js -o public/bundle.js", 8 | "watch": "watchify dist/client.js -o public/bundle.js -v", 9 | "start": "node dist/server.js" 10 | }, 11 | "dependencies": { 12 | "@types/node": "^18.8.3", 13 | "@types/polka": "^0.5.4", 14 | "@types/priorityqueuejs": "^1.0.1", 15 | "@types/ws": "^8.5.3", 16 | "binary-search": "^1.3.6", 17 | "body-parser": "^1.20.0", 18 | "browserify": "^17.0.0", 19 | "map2": "^1.1.2", 20 | "polka": "^0.5.2", 21 | "priorityqueuejs": "^2.0.0", 22 | "sirv": "^2.0.2", 23 | "typescript": "^4.7.4", 24 | "watchify": "^4.0.0", 25 | "ws": "^8.8.1" 26 | }, 27 | "devDependencies": { 28 | "@types/mocha": "^9.1.1", 29 | "mocha": "^10.0.0", 30 | "ts-node": "^10.9.1" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /js/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | DTJS 3 |

4 | Diamond Types JS example 5 |

6 |
7 | Loading... 8 | 9 | 10 |
11 | 12 | 

13 | 
14 | 


--------------------------------------------------------------------------------
/js/src/client.ts:
--------------------------------------------------------------------------------
  1 | import * as dt from './simpledb.js'
  2 | import { WSServerClientMsg } from './msgs.js'
  3 | import { Operation, ROOT } from './types.js'
  4 | import { createAgent, versionInSummary } from './utils.js'
  5 | 
  6 | const agent = createAgent()
  7 | let db: null | dt.SimpleDB = null
  8 | 
  9 | let ws: WebSocket | null = null
 10 | 
 11 | let inflightOps: Operation[] = []
 12 | let pendingOps: Operation[] = []
 13 | 
 14 | const rerender = () => {
 15 |   const dbVal = dt.get(db!) as any
 16 |   const clockElem = document.getElementById('clock')
 17 |   clockElem!.textContent = dbVal.time
 18 | 
 19 |   const rawElem = document.getElementById('raw')
 20 |   rawElem!.innerText = `
 21 | RAW: ${JSON.stringify(dbVal, null, 2)}
 22 | 
 23 | Internal: ${JSON.stringify(dt.toSnapshot(db!), null, 2)}
 24 | `
 25 | }
 26 | 
 27 | const connect = () => {
 28 |   const loc = window.location
 29 |   const url = (loc.protocol === 'https:' ? 'wss://' : 'ws://')
 30 |     + loc.host
 31 |     + loc.pathname
 32 |     + 'ws'
 33 |   console.log('url', url)
 34 |   ws = new WebSocket(url)
 35 |   ws.onopen = (e) => {
 36 |     console.log('open', e)
 37 |     pendingOps = inflightOps.concat(pendingOps)
 38 |     inflightOps.length = 0
 39 |   }
 40 | 
 41 |   ws.onmessage = (e) => {
 42 |     // console.log('msg', e.data)
 43 | 
 44 |     const data = JSON.parse(e.data) as WSServerClientMsg
 45 |   
 46 |     // console.log('data', data)
 47 |   
 48 |     switch (data.type) {
 49 |       case 'snapshot': {
 50 |         let changed = true
 51 |         if (db == null) {
 52 |           db = dt.fromSnapshot(data.data)
 53 |         } else {
 54 |           changed = dt.mergeSnapshot(db, data.data, data.v, inflightOps.concat(pendingOps))
 55 |         }
 56 | 
 57 |         if (changed) {
 58 |           console.log('version changed. Rerendering.')
 59 |           rerender()
 60 |         }
 61 | 
 62 |         flush()
 63 |         break
 64 |       }
 65 |       case 'ops': {
 66 |         let anyChange = false
 67 |         for (const op of data.ops) {
 68 |           if (inflightOps.find(op2 => dt.versionEq(op.id, op2.id)) == null) {
 69 |             dt.applyRemoteOp(db!, op)
 70 |             anyChange = true
 71 |           }
 72 |         }
 73 | 
 74 |         // console.log(anyChange, 'if', inflightOps, 'pending', pendingOps)
 75 |         if (anyChange) rerender()
 76 | 
 77 |         break
 78 |       }
 79 |     }
 80 |   }
 81 |   
 82 |   ws.onclose = (e) => {
 83 |     console.log('WS closed', e)
 84 |     ws = null
 85 |     setTimeout(() => {
 86 |       connect()
 87 |     }, 3000)
 88 |   }
 89 | 
 90 |   ws.onerror = (e) => {
 91 |     console.error('WS error', e)
 92 |   }
 93 | }
 94 | 
 95 | connect()
 96 | 
 97 | const decrButton = document.getElementById('decrement')!
 98 | const incrButton = document.getElementById('increment')!
 99 | 
100 | let flushing = false
101 | const flush = () => {
102 |   if (!flushing && ws != null && inflightOps.length === 0 && pendingOps.length > 0) {
103 |     // console.log('flush!')
104 |     flushing = true
105 |     inflightOps.push(...pendingOps)
106 |     pendingOps.length = 0
107 | 
108 |     ;(async () => {
109 |       try {
110 |         // console.log('Sending ops')
111 |         const response = await fetch('/op', {
112 |           method: 'post',
113 |           cache: 'no-store',
114 |           headers: {
115 |             'content-type': 'application/json',
116 |           },
117 |           body: JSON.stringify(inflightOps)
118 |         })
119 |         // console.log('Got response', response.status, response.statusText)
120 |         flushing = false
121 | 
122 |         const status = response.status
123 |         if (status < 400) {
124 |           // console.log('ok')
125 |           inflightOps.length = 0
126 |           flush()
127 |         } else {
128 |           console.error('Could not submit op:', status, response.statusText)
129 |           setTimeout(flush, 3000)
130 |         }
131 |       } catch (e) {
132 |         console.log('Could not submit op:', e)
133 |         flushing = false
134 |         setTimeout(flush, 3000)
135 |       }
136 |     })()
137 |   }
138 | }
139 | 
140 | const editTime = (newVal: number) => {
141 |   const op = dt.setAtPath(db!, agent(), ['time'], {type: 'primitive', val: newVal})
142 |   pendingOps.push(op)
143 |   flush()
144 |   // ws?.send(JSON.stringify(msg))
145 | 
146 |   rerender()
147 | }
148 | 
149 | incrButton.onclick = () => {
150 |   const oldVal = dt.getAtPath(db!, ['time']) as number ?? 0
151 |   editTime(oldVal + 1)
152 | }
153 | decrButton.onclick = () => {
154 |   const oldVal = dt.getAtPath(db!, ['time']) as number ?? 0
155 |   editTime(oldVal - 1)
156 | }
157 | 


--------------------------------------------------------------------------------
/js/src/fancydb/rle.ts:
--------------------------------------------------------------------------------
 1 | import { LVRange } from "../types"
 2 | 
 3 | export const pushRLEList = (list: T[], newItem: T, tryAppend: (a: T, b: T) => boolean) => {
 4 |   if (list.length > 0) {
 5 |     if (tryAppend(list[list.length - 1], newItem)) return
 6 |   }
 7 |   list.push(newItem)
 8 | }
 9 | 
10 | export const tryRangeAppend = (r1: LVRange, r2: LVRange): boolean => {
11 |   if (r1[1] === r2[0]) {
12 |     r1[1] = r2[1]
13 |     return true
14 |   } else return false
15 | }
16 | 
17 | export const tryRevRangeAppend = (r1: LVRange, r2: LVRange): boolean => {
18 |   if (r1[0] === r2[1]) {
19 |     r1[0] = r2[0]
20 |     return true
21 |   } else return false
22 | }


--------------------------------------------------------------------------------
/js/src/msgs.ts:
--------------------------------------------------------------------------------
 1 | import { DBSnapshot, Operation, VersionSummary } from "./types.js"
 2 | 
 3 | export type WSServerClientMsg = {
 4 |   type: 'snapshot',
 5 |   data: DBSnapshot,
 6 |   v: VersionSummary
 7 | } | {
 8 |   type: 'ops',
 9 |   ops: Operation[]
10 | }
11 | 


--------------------------------------------------------------------------------
/js/src/server.ts:
--------------------------------------------------------------------------------
  1 | import * as dt from './fancydb'
  2 | import polka from 'polka'
  3 | import * as bodyParser from 'body-parser'
  4 | import sirv from 'sirv'
  5 | import {WebSocket, WebSocketServer} from 'ws'
  6 | import * as http from 'http'
  7 | import { WSServerClientMsg } from './msgs.js'
  8 | import { Operation, ROOT_LV } from './types.js'
  9 | import { createAgent, rateLimit } from './utils.js'
 10 | import fs from 'fs'
 11 | import { hasVersion, summarizeVersion } from './fancydb/causal-graph'
 12 | 
 13 | const app = polka()
 14 | .use(sirv('public', {
 15 |   dev: true
 16 | }))
 17 | 
 18 | const DB_FILE = process.env['DB_FILE'] || 'db.dtjson'
 19 | 
 20 | const db = (() => {
 21 |   try {
 22 |     const bytes = fs.readFileSync(DB_FILE, 'utf8')
 23 |     const json = JSON.parse(bytes)
 24 |     return dt.fromSerialized(json)
 25 |   } catch (e: any) {
 26 |     if (e.code !== 'ENOENT') throw e
 27 | 
 28 |     console.log('Using new database file')
 29 |     return dt.createDb()
 30 |   }
 31 | })()
 32 | 
 33 | console.dir(dt.get(db), {depth: null})
 34 | 
 35 | const saveDb = rateLimit(100, () => {
 36 |   // console.log('saving')
 37 |   const json = dt.serialize(db)
 38 |   const bytes = JSON.stringify(json, null, 2)
 39 |   // return fs.promises.writeFile(DB_FILE, bytes)
 40 |   return fs.writeFileSync(DB_FILE, bytes)
 41 | })
 42 | 
 43 | db.onop = (db, op) => saveDb()
 44 | 
 45 | process.on('exit', () => {
 46 |   saveDb.flushSync()
 47 | })
 48 | 
 49 | process.on('SIGINT', () => {
 50 |   // Catching this to make sure we save!
 51 |   // console.log('SIGINT!')
 52 |   process.exit(1)
 53 | })
 54 | 
 55 | const clients = new Set()
 56 | 
 57 | const broadcastOp = (ops: Operation[], exclude?: any) => {
 58 |   console.log('broadcast', ops)
 59 |   const msg: WSServerClientMsg = {
 60 |     type: 'ops',
 61 |     ops
 62 |   }
 63 | 
 64 |   const msgStr = JSON.stringify(msg)
 65 |   for (const c of clients) {
 66 |     // if (c !== exclude) {
 67 |     c.send(msgStr)
 68 |     // }
 69 |   }
 70 | }
 71 | 
 72 | if (dt.get(db).time == null) {
 73 |   console.log('Setting time = 0')
 74 |   const serverAgent = createAgent()
 75 |   dt.localMapInsert(db, serverAgent(), ROOT_LV, 'time', {type: 'primitive', val: 0})
 76 | }
 77 | 
 78 | app.post('/op', bodyParser.json(), (req, res, next) => {
 79 |   let ops = req.body as Operation[]
 80 |   console.log(`Got ${ops.length} from client`)
 81 | 
 82 |   ops = ops.filter(op => !hasVersion(db.cg, op.id[0], op.id[1]))
 83 |   ops.forEach(op => dt.applyRemoteOp(db, op))
 84 |   broadcastOp(ops)
 85 | 
 86 |   res.end()
 87 | })
 88 | 
 89 | const server = http.createServer(app.handler as any)
 90 | const wss = new WebSocketServer({server})
 91 | 
 92 | wss.on('connection', ws => {
 93 |   // console.dir(dt.toJSON(db), {depth: null})
 94 |   const msg: WSServerClientMsg = {
 95 |     type: 'snapshot',
 96 |     data: dt.toSnapshot(db),
 97 |     v: summarizeVersion(db.cg),
 98 |   }
 99 |   ws.send(JSON.stringify(msg))
100 |   clients.add(ws)
101 | 
102 |   ws.on('message', (msgBytes) => {
103 |     // const rawJSON = msgBytes.toString('utf-8')
104 |     // const msg = JSON.parse(rawJSON) as WSClientServerMsg
105 |     // // console.log('msg', msg)
106 |     // switch (msg.type) {
107 |     //   case 'op': {
108 |     //     console.log(msg)
109 |     //     msg.ops.forEach(op => dt.applyRemoteOp(db, op))
110 |     //     broadcastOp(msg.ops, ws)
111 |     //     break
112 |     //   }
113 |     // }
114 |   })
115 | 
116 |   ws.on('close', () => {
117 |     console.log('client closed')
118 |     clients.delete(ws)
119 |   })
120 | })
121 | 
122 | server.listen(3003, () => {
123 |   console.log('listening on port 3003')
124 | })
125 | 


--------------------------------------------------------------------------------
/js/src/text/text.ts:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/js/src/text/text.ts


--------------------------------------------------------------------------------
/js/src/types.ts:
--------------------------------------------------------------------------------
 1 | import Map2 from "map2"
 2 | 
 3 | export type RawVersion = [agent: string, seq: number]
 4 | 
 5 | export const ROOT: RawVersion = ['ROOT', 0]
 6 | 
 7 | /** Local version */
 8 | export type LV = number
 9 | 
10 | /** Local version range. Range is [start, end). */
11 | export type LVRange = [start: number, end: number]
12 | 
13 | export const ROOT_LV: LV = -1
14 | 
15 | export type Primitive = null
16 |   | boolean
17 |   | string
18 |   | number
19 |   | Primitive[]
20 |   | {[k: string]: Primitive}
21 | 
22 | export type CreateValue = {type: 'primitive', val: Primitive}
23 |   | {type: 'crdt', crdtKind: 'map' | 'set' | 'register'}
24 | 
25 | export type Action =
26 | { type: 'map', key: string, localParents: RawVersion[], val: CreateValue }
27 | | { type: 'registerSet', localParents: RawVersion[], val: CreateValue }
28 | | { type: 'setInsert', val: CreateValue }
29 | | { type: 'setDelete', target: RawVersion }
30 | 
31 | export interface Operation {
32 |   id: RawVersion,
33 |   globalParents: RawVersion[],
34 |   crdtId: RawVersion,
35 |   action: Action,
36 | }
37 | 
38 | export type DBValue = null
39 |   | boolean
40 |   | string
41 |   | number
42 |   | DBValue[]
43 |   | {[k: string]: DBValue} // Map
44 |   | Map2 // Set.
45 | 
46 | 
47 | /** Helper type for a list with at least 1 entry in it. */
48 | export type AtLeast1 = [T, ...T[]]
49 | 
50 | 
51 | export type SnapRegisterValue = {type: 'primitive', val: Primitive}
52 |   | {type: 'crdt', id: RawVersion}
53 | export type SnapMVRegister = [RawVersion, SnapRegisterValue][]
54 | export type SnapCRDTInfo = {
55 |   type: 'map',
56 |   registers: {[k: string]: SnapMVRegister},
57 | } | {
58 |   type: 'set',
59 |   values: [string, number, SnapRegisterValue][],
60 | } | {
61 |   type: 'register',
62 |   value: SnapMVRegister,
63 | }
64 | 
65 | export interface DBSnapshot {
66 |   version: RawVersion[],
67 |   crdts: [string, number, SnapCRDTInfo][]
68 | }
69 | 
70 | export interface VersionSummary {[agent: string]: [number, number][]}
71 | 


--------------------------------------------------------------------------------
/js/src/utils.ts:
--------------------------------------------------------------------------------
 1 | import { RawVersion, VersionSummary } from "./types"
 2 | 
 3 | export function createAgent(): () => RawVersion {
 4 |   const agent = Math.random().toString(36).slice(2)
 5 |   let seq = 0
 6 |   return () => ([agent, seq++])
 7 | }
 8 | 
 9 | type RateLimit = {
10 |   flushSync(): void,
11 |   (): void,
12 | }
13 | 
14 | export function rateLimit(min_delay: number, fn: () => void): RateLimit {
15 |   let next_call = 0
16 |   let timer: NodeJS.Timeout | null = null
17 | 
18 |   const rl = () => {
19 |     let now = Date.now()
20 | 
21 |     if (next_call <= now) {
22 |       // Just call the function.
23 |       next_call = now + min_delay
24 | 
25 |       if (timer != null) {
26 |         clearTimeout(timer)
27 |         timer = null
28 |       }
29 |       fn()
30 |     } else {
31 |       // Queue the function call.
32 |       if (timer == null) {
33 |         timer = setTimeout(() => {
34 |           timer = null
35 |           next_call = Date.now() + min_delay
36 |           fn()
37 |         }, next_call - now)
38 |       } // Otherwise its already queued.
39 |     }
40 |   }
41 | 
42 |   rl.flushSync = () => {
43 |     if (timer != null) {
44 |       clearTimeout(timer)
45 |       timer = null
46 |       fn()
47 |     }
48 |   }
49 | 
50 |   return rl
51 | }
52 | 
53 | export const versionInSummary = (vs: VersionSummary, [agent, seq]: RawVersion): boolean => {
54 |   const ranges = vs[agent]
55 |   if (ranges == null) return false
56 |   // This could be implemented using a binary search, but thats probably fine here.
57 |   return ranges.find(([from, to]) => seq >= from && seq < to) !== undefined
58 | }
59 | 


--------------------------------------------------------------------------------
/js/tests/causal-graph.ts:
--------------------------------------------------------------------------------
 1 | import 'mocha'
 2 | import * as causalGraph from '../src/fancydb/causal-graph'
 3 | import assert from 'assert/strict'
 4 | 
 5 | describe('causal graph', () => {
 6 |   it('smoke test', () => {
 7 |     const cg = causalGraph.create()
 8 | 
 9 |     causalGraph.add(cg, 'seph', 10, 20, [])
10 |     causalGraph.add(cg, 'mike', 10, 20, [])
11 | 
12 |     causalGraph.assignLocal(cg, 'seph', 5)
13 | 
14 |     const serialized = causalGraph.serialize(cg)
15 |     const deserialized = causalGraph.fromSerialized(serialized)
16 |     assert.deepEqual(cg, deserialized)
17 |   })
18 | })
19 | 


--------------------------------------------------------------------------------
/js/tests/cg-tools.ts:
--------------------------------------------------------------------------------
  1 | import 'mocha'
  2 | import fs from 'fs'
  3 | import * as causalGraph from '../src/fancydb/causal-graph'
  4 | import assert from 'assert/strict'
  5 | import { LV, LVRange } from '../src/types'
  6 | import { pushRLEList, tryRevRangeAppend } from '../src/fancydb/rle'
  7 | 
  8 | type HistItem = {
  9 |   span: LVRange,
 10 |   parents: LV[]
 11 | }
 12 | 
 13 | const histToCG = (hist: HistItem[]): causalGraph.CausalGraph => {
 14 |   const cg = causalGraph.create()
 15 | 
 16 |   for (const h of hist) {
 17 |     causalGraph.add(cg, 'testUser', h.span[0], h.span[1], h.parents)
 18 |   }
 19 | 
 20 | 
 21 |   return cg
 22 | }
 23 | 
 24 | const readJSONFile = (filename: string): T[] => (
 25 |   fs.readFileSync(filename, 'utf-8')
 26 |     .split('\n')
 27 |     .filter(s => s.length > 0)
 28 |     .map(s => JSON.parse(s))
 29 | )
 30 | 
 31 | describe('causal graph utilities', () => {
 32 |   it('version contains time', () => {
 33 |     type VersionContainsTimeTest = {
 34 |       hist: HistItem[],
 35 |       frontier: number[],
 36 |       target: number,
 37 |       expected: boolean
 38 |     }
 39 | 
 40 |     const data = readJSONFile('../test_data/causal_graph/version_contains.json')
 41 | 
 42 |     for (const {hist, frontier, target, expected} of data) {
 43 |       // console.log(hist)
 44 |       const cg = histToCG(hist)
 45 | 
 46 |       const actual = causalGraph.versionContainsTime(cg, frontier, target)
 47 |       if (expected !== actual) {
 48 |         console.dir(cg, {depth: null})
 49 |         console.dir(frontier, {depth: null})
 50 |         console.dir(target, {depth: null})
 51 |         console.log('expected:', expected, 'actual:', actual)
 52 |       }
 53 |       assert.equal(expected, actual)
 54 |     }
 55 |   })
 56 | 
 57 |   it('diff', () => {
 58 |     type DiffTest = {
 59 |       hist: HistItem[],
 60 |       a: number[],
 61 |       b: number[],
 62 |       expect_a: Range[],
 63 |       expect_b: Range[],
 64 |     }
 65 | 
 66 |     const data = readJSONFile('../test_data/causal_graph/diff.json')
 67 | 
 68 | 
 69 |     for (const {hist, a, b, expect_a, expect_b} of data) {
 70 |       // console.log(hist)
 71 |       const cg = histToCG(hist)
 72 |       expect_a.reverse()
 73 |       expect_b.reverse()
 74 | 
 75 |       const {aOnly, bOnly} = causalGraph.diff(cg, a, b)
 76 |       // console.log(aOnly, expect_a)
 77 |       assert.deepEqual(aOnly, expect_a)
 78 |       assert.deepEqual(bOnly, expect_b)
 79 |     }
 80 |   })
 81 | 
 82 |   it('find conflicting', () => {
 83 |     type DiffFlagStr = 'OnlyA' | 'OnlyB' | 'Shared'
 84 |     type ConflictTest = {
 85 |       hist: HistItem[],
 86 |       a: number[],
 87 |       b: number[],
 88 |       expect_spans: [range: {start: number, end: number}, flag: DiffFlagStr][],
 89 |       expect_common: number[],
 90 |     }
 91 | 
 92 |     const flagToStr = (f: causalGraph.DiffFlag): DiffFlagStr => (
 93 |         f === causalGraph.DiffFlag.A ? 'OnlyA'
 94 |           : f === causalGraph.DiffFlag.B ? 'OnlyB'
 95 |           : 'Shared'
 96 |     )
 97 | 
 98 |     const data = readJSONFile('../test_data/causal_graph/conflicting.json')
 99 | 
100 |     const test = ({hist, a, b, expect_spans, expect_common}: ConflictTest) => {
101 |       const expectSpans = expect_spans.map(([{start, end}, flagStr]) => [[start, end], flagStr] as [LVRange, DiffFlagStr])
102 | 
103 |       const cg = histToCG(hist)
104 | 
105 |       const actualSpans: [LVRange, DiffFlagStr][] = []
106 |       const actualCommon = causalGraph.findConflicting(cg, a, b, (range, flag) => {
107 |         // console.log('emit', range, flag)
108 | 
109 |         // The 'actualSpans' list is in reverse order. We need to RLE style merge it all together.
110 |         pushRLEList<[LVRange, DiffFlagStr]>(actualSpans, [range, flagToStr(flag)], (a, b) => (
111 |           a[1] === b[1] && tryRevRangeAppend(a[0], b[0])
112 |         ))
113 |       })
114 |       actualSpans.reverse()
115 |       // console.log('actual', actualSpans, 'expect', expectSpans)
116 |       assert.deepEqual(expectSpans, actualSpans)
117 |       assert.deepEqual(expect_common, actualCommon)
118 |     }
119 | 
120 |     // test(data[19])
121 |     for (let i = 0; i < data.length; i++) {
122 |       // console.log(`======== ${i} =======`)
123 |       test(data[i])
124 |     }
125 | 
126 |   })
127 | })


--------------------------------------------------------------------------------
/npm-pkg-isomorphic/README.md:
--------------------------------------------------------------------------------
1 | # Isomorphic diamond-types
2 | 
3 | This package is a placeholder for an isomorphic build of diamond-types. Currently diamond types is split into diamond-types-web and diamond-types-node. This package should work in both contexts, when I make it.
4 | 


--------------------------------------------------------------------------------
/npm-pkg-isomorphic/index.js:
--------------------------------------------------------------------------------
1 | throw Error('This package is currently a stub.')
2 | 


--------------------------------------------------------------------------------
/npm-pkg-isomorphic/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "diamond-types",
 3 |   "version": "0.0.1",
 4 |   "description": "Diamond types isomorphic package",
 5 |   "main": "index.js",
 6 |   "scripts": {
 7 |     "test": "echo \"Error: no test specified\" && exit 1"
 8 |   },
 9 |   "author": "",
10 |   "license": "ISC"
11 | }
12 | 


--------------------------------------------------------------------------------
/prev_oplog/operation.rs:
--------------------------------------------------------------------------------
 1 | use rle::{HasLength, MergableSpan, SplitableSpanCtx};
 2 | use crate::{CRDTKind, ListOperationCtx, Op, OpContents};
 3 | 
 4 | impl HasLength for OpContents {
 5 |     fn len(&self) -> usize {
 6 |         match self {
 7 |             OpContents::Text(metrics) => metrics.len(),
 8 |             _ => 1,
 9 |         }
10 |     }
11 | }
12 | 
13 | impl MergableSpan for OpContents {
14 |     fn can_append(&self, other: &Self) -> bool {
15 |         match (self, other) {
16 |             (OpContents::Text(a), OpContents::Text(b)) => a.can_append(b),
17 |             _ => false,
18 |         }
19 |     }
20 | 
21 |     fn append(&mut self, other: Self) {
22 |         match (self, other) {
23 |             (OpContents::Text(a), OpContents::Text(b)) => a.append(b),
24 |             _ => panic!("Cannot append"),
25 |         }
26 |     }
27 | }
28 | 
29 | impl SplitableSpanCtx for OpContents {
30 |     type Ctx = ListOperationCtx;
31 | 
32 |     fn truncate_ctx(&mut self, at: usize, ctx: &Self::Ctx) -> Self {
33 |         match self {
34 |             OpContents::Text(metrics) => {
35 |                 let remainder = metrics.truncate_ctx(at, ctx);
36 |                 OpContents::Text(remainder)
37 |             }
38 |             _ => {
39 |                 panic!("Cannot truncate op");
40 |             }
41 |         }
42 |     }
43 | }
44 | 
45 | impl OpContents {
46 |     pub fn kind(&self) -> CRDTKind {
47 |         match self {
48 |             OpContents::RegisterSet(_) => CRDTKind::Register,
49 |             OpContents::MapSet(_, _) | OpContents::MapDelete(_) => CRDTKind::Map,
50 |             OpContents::Collection(_) => CRDTKind::Collection,
51 |             OpContents::Text(_) => CRDTKind::Text,
52 |         }
53 |     }
54 | }
55 | 
56 | impl HasLength for Op {
57 |     fn len(&self) -> usize { self.contents.len() }
58 | }
59 | 
60 | impl MergableSpan for Op {
61 |     fn can_append(&self, other: &Self) -> bool {
62 |         self.target_id == other.target_id && self.contents.can_append(&other.contents)
63 |     }
64 | 
65 |     fn append(&mut self, other: Self) {
66 |         self.contents.append(other.contents)
67 |     }
68 | }
69 | 
70 | impl SplitableSpanCtx for Op {
71 |     type Ctx = ListOperationCtx;
72 | 
73 |     fn truncate_ctx(&mut self, at: usize, ctx: &Self::Ctx) -> Self {
74 |         let remainder = self.contents.truncate_ctx(at, ctx);
75 |         Self {
76 |             target_id: self.target_id,
77 |             contents: remainder
78 |         }
79 |     }
80 | }


--------------------------------------------------------------------------------
/prev_oplog/path.rs:
--------------------------------------------------------------------------------
 1 | use crate::Branch;
 2 | 
 3 | /// The path API provides a simple way to traverse in and modify values
 4 | 
 5 | #[derive(Debug, Clone, Eq, PartialEq)]
 6 | pub enum PathComponent<'a> {
 7 |     Inside,
 8 |     Key(&'a str),
 9 | }
10 | 
11 | // pub type PathRef<'a> = &'a [PathComponent];
12 | // pub type Path = SmallVec<[PathComponent; 4]>;
13 | pub type Path<'a> = &'a [PathComponent<'a>];
14 | // pub type Path<'a> = &'a [&'a str];
15 | 
16 | impl Branch {
17 |     // pub fn item_at_path_mut(&mut self, path: Path) -> Value {
18 |     //     let mut current_kind = CRDTKind::Map;
19 |     //     let mut current_value = Value::InnerCRDT(ROOT_MAP);
20 |     //     let mut current_key: Option = None;
21 |     //
22 |     //     for p in path {
23 |     //         match (current_kind, p) {
24 |     //             // (Value::Primitive(_), _) => {
25 |     //             //     panic!("Cannot traverse inside primitive value");
26 |     //             // }
27 |     //             // (Value::Map(_), PathComponent::Inside) => {
28 |     //             //     panic!("Cannot traverse inside map");
29 |     //             // }
30 |     //             (CRDTKind::Map, PathComponent::Key(k)) => {
31 |     //                 self.get_map_value(current_id, *k);
32 |     //                 current_key = Some(k.into());
33 |     //                 // current = Value::InnerCRDT(self.get_or_create_map_child(map_id, (*k).into()));
34 |     //             }
35 |     //             // (Value::InnerCRDT(item_id), PathComponent::Inside) => {
36 |     //             //     current = self.get_value_of_register(item_id, version).unwrap();
37 |     //             // }
38 |     //             // (Value::InnerCRDT(item_id), PathComponent::Key(k)) => {
39 |     //             //     let map_id = self.autoexpand_until_map(item_id, version).unwrap();
40 |     //             //     current = Value::InnerCRDT(self.get_or_create_map_child(map_id, (*k).into()));
41 |     //             // }
42 |     //         }
43 |     //     }
44 |     //     current
45 |     // }
46 | 
47 |     // fn autoexpand_until_map(&self, mut item_id: CRDTItemId, version: &[Time]) -> Option {
48 |     //     loop {
49 |     //         let info = &self.known_crdts[item_id];
50 |     //
51 |     //         if info.kind == CRDTKind::LWWRegister {
52 |     //             // Unwrap and loop
53 |     //             let value = self.get_value_of_register(item_id, version)?;
54 |     //             match value {
55 |     //                 Value::Primitive(_) => {
56 |     //                     return None;
57 |     //                 }
58 |     //                 Value::Map(map_id) => {
59 |     //                     return Some(map_id);
60 |     //                 }
61 |     //                 Value::InnerCRDT(inner) => {
62 |     //                     item_id = inner;
63 |     //                     // And recurse.
64 |     //                 }
65 |     //             }
66 |     //         } else {
67 |     //             return None;
68 |     //         }
69 |     //     }
70 |     // }
71 |     //
72 |     // // pub(crate) fn append_create_inner_crdt(&mut self, agent_id: AgentId, parents: &[Time], parent_crdt_id: ScopeId, kind: CRDTKind) -> (Time, ScopeId) {
73 |     // // fn create_at_path(&mut self, agent_id: AgentId, parents: &[Time], path: Path, kind: CRDTKind)
74 |     // pub fn create_crdt_at_path(&mut self, agent_id: AgentId, path: Path, kind: CRDTKind) -> Time {
75 |     //     let v = self.now(); // I hate this.
76 |     //     let scope = self.item_at_path_mut(path, &v).unwrap_crdt();
77 |     //     self.append_create_inner_crdt(agent_id, &v, scope, kind).0
78 |     // }
79 |     //
80 |     // pub fn create_map_at_path(&mut self, agent_id: AgentId, path: Path) -> Time {
81 |     //     let v = self.now(); // :/
82 |     //     let scope = self.item_at_path_mut(path, &v).unwrap_crdt();
83 |     //     self.append_set_new_map(agent_id, &v, scope).0
84 |     // }
85 |     //
86 |     // pub fn set_at_path(&mut self, agent_id: AgentId, path: Path, value: Primitive) -> Time {
87 |     //     let v = self.now(); // :(
88 |     //     let scope = self.item_at_path_mut(path, &v).unwrap_crdt();
89 |     //     self.append_set(agent_id, &v, scope, value)
90 |     // }
91 | }


--------------------------------------------------------------------------------
/profile.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | set -e
3 | 
4 | RUSTFLAGS="-Cforce-frame-pointers=yes" cargo build --profile profiling --example profile
5 | perf record -g -F 9999 --call-graph fp target/profiling/examples/profile
6 | perf script -F +pid > /tmp/test.perf
7 | 
8 | echo "Perf data in perf.data and script in /tmp/test.perf"
9 | 


--------------------------------------------------------------------------------
/src/causalgraph/agent_span.rs:
--------------------------------------------------------------------------------
 1 | // TODO: Consider moving me into agent_assignment/.
 2 | 
 3 | use std::ops::Range;
 4 | // use content_tree::ContentLength;
 5 | use rle::{HasLength, MergableSpan, Searchable, SplitableSpan, SplitableSpanHelpers};
 6 | use crate::AgentId;
 7 | use crate::dtrange::DTRange;
 8 | 
 9 | /// (agent_id, seq) pair. The agent ID is an integer which maps to a local string via causal graph.
10 | pub type AgentVersion = (AgentId, usize);
11 | 
12 | /// An AgentSpan represents a sequential span of (agent, seq) versions.
13 | #[derive(Debug, Copy, Clone, Eq, PartialEq)]
14 | pub struct AgentSpan {
15 |     pub agent: AgentId,
16 |     pub seq_range: DTRange,
17 | }
18 | 
19 | impl From<(AgentId, DTRange)> for AgentSpan {
20 |     fn from((agent, seq_range): (AgentId, DTRange)) -> Self {
21 |         AgentSpan { agent, seq_range }
22 |     }
23 | }
24 | 
25 | impl From<(AgentId, Range)> for AgentSpan {
26 |     fn from((agent, seq_range): (AgentId, Range)) -> Self {
27 |         AgentSpan { agent, seq_range: seq_range.into() }
28 |     }
29 | }
30 | 
31 | impl From for AgentSpan {
32 |     fn from((agent, seq): AgentVersion) -> Self {
33 |         AgentSpan { agent, seq_range: seq.into() }
34 |     }
35 | }
36 | 
37 | impl Searchable for AgentSpan {
38 |     type Item = AgentVersion;
39 | 
40 |     fn get_offset(&self, (agent, seq): AgentVersion) -> Option {
41 |         // let r = self.loc.seq .. self.loc.seq + (self.len.abs() as usize);
42 |         // self.loc.agent == loc.agent && entry.get_seq_range().contains(&loc.seq)
43 |         if self.agent == agent {
44 |             self.seq_range.get_offset(seq)
45 |         } else { None }
46 |     }
47 | 
48 |     fn at_offset(&self, offset: usize) -> AgentVersion {
49 |         assert!(offset < self.len());
50 |         (self.agent, self.seq_range.start + offset)
51 |     }
52 | }
53 | 
54 | impl HasLength for AgentSpan {
55 |     /// this length refers to the length that we'll use when we call truncate(). So this does count
56 |     /// deletes.
57 |     fn len(&self) -> usize {
58 |         self.seq_range.len()
59 |     }
60 | }
61 | impl SplitableSpanHelpers for AgentSpan {
62 |     fn truncate_h(&mut self, at: usize) -> Self {
63 |         AgentSpan {
64 |             agent: self.agent,
65 |             seq_range: self.seq_range.truncate(at)
66 |         }
67 |     }
68 | 
69 |     fn truncate_keeping_right_h(&mut self, at: usize) -> Self {
70 |         AgentSpan {
71 |             agent: self.agent,
72 |             seq_range: self.seq_range.truncate_keeping_right(at)
73 |         }
74 |     }
75 | }
76 | impl MergableSpan for AgentSpan {
77 |     fn can_append(&self, other: &Self) -> bool {
78 |         self.agent == other.agent
79 |             && self.seq_range.end == other.seq_range.start
80 |     }
81 | 
82 |     fn append(&mut self, other: Self) {
83 |         self.seq_range.end = other.seq_range.end;
84 |     }
85 | 
86 |     fn prepend(&mut self, other: Self) {
87 |         self.seq_range.start = other.seq_range.start;
88 |     }
89 | }
90 | 


--------------------------------------------------------------------------------
/src/causalgraph/check.rs:
--------------------------------------------------------------------------------
 1 | use crate::CausalGraph;
 2 | use crate::causalgraph::agent_assignment::AgentAssignment;
 3 | 
 4 | impl AgentAssignment {
 5 |     #[allow(unused)]
 6 |     pub fn dbg_check(&self, deep: bool) {
 7 |         // The client_with_localtime should match with the corresponding items in client_data
 8 |         self.client_with_lv.check_packed();
 9 | 
10 |         for pair in self.client_with_lv.iter() {
11 |             let expected_range = pair.range();
12 | 
13 |             let span = pair.1;
14 |             let client = &self.client_data[span.agent as usize];
15 |             let actual_range = client.lv_for_seq.find_packed_and_split(span.seq_range);
16 | 
17 |             assert_eq!(actual_range.1, expected_range);
18 |         }
19 | 
20 |         if deep {
21 |             // Also check the other way around.
22 |             for (agent, client) in self.client_data.iter().enumerate() {
23 |                 for range in client.lv_for_seq.iter() {
24 |                     let actual = self.client_with_lv.find_packed_and_split(range.1);
25 |                     assert_eq!(actual.1.agent as usize, agent);
26 |                 }
27 |             }
28 |         }
29 |     }
30 | }
31 | 
32 | impl CausalGraph {
33 |     #[allow(unused)]
34 |     pub fn dbg_check(&self, deep: bool) {
35 |         if deep {
36 |             self.graph.dbg_check(deep);
37 |         }
38 | 
39 |         self.agent_assignment.dbg_check(deep);
40 | 
41 |         assert_eq!(self.version, self.graph.dbg_get_frontier_inefficiently());
42 |     }
43 | }
44 | 


--------------------------------------------------------------------------------
/src/causalgraph/dot.rs:
--------------------------------------------------------------------------------
  1 | use std::collections::HashSet;
  2 | use std::ffi::OsString;
  3 | use std::fs::File;
  4 | use std::io::Write;
  5 | use std::fmt::Write as _;
  6 | use std::path::{Path, PathBuf};
  7 | use std::process::{Command, Stdio};
  8 | use rle::HasLength;
  9 | use crate::{CausalGraph, LV};
 10 | 
 11 | #[derive(Debug, Clone, Copy)]
 12 | #[allow(unused)]
 13 | pub enum DotColor {
 14 |     Red, Green, Blue, Grey, Black
 15 | }
 16 | 
 17 | impl ToString for DotColor {
 18 |     fn to_string(&self) -> String {
 19 |         match self {
 20 |             DotColor::Red => "red".into(),
 21 |             DotColor::Green => "\"#98ea79\"".into(),
 22 |             DotColor::Blue => "\"#036ffc\"".into(),
 23 |             DotColor::Grey => "\"#eeeeee\"".into(),
 24 |             DotColor::Black => "black".into(),
 25 |         }
 26 |     }
 27 | }
 28 | 
 29 | impl CausalGraph {
 30 |     pub fn to_dot_graph(&self, v: Option<&[LV]>) -> String {
 31 |         // Same as above, but each merge creates a new dot item.
 32 |         let mut merges_touched = HashSet::new();
 33 | 
 34 |         fn key_for_parents(p: &[LV]) -> String {
 35 |             p.iter().map(|t| format!("{t}"))
 36 |                 .collect::>().join("0")
 37 |         }
 38 | 
 39 |         let mut out = String::new();
 40 |         out.push_str("strict digraph {\n");
 41 |         out.push_str("\trankdir=\"BT\"\n");
 42 |         // out.write_fmt(format_args!("\tlabel='{}'>\n", starting_content));
 43 |         out.push_str("\tlabelloc=\"t\"\n");
 44 |         out.push_str("\tnode [shape=box style=filled]\n");
 45 |         out.push_str("\tedge [color=\"#333333\" dir=none]\n");
 46 | 
 47 |         write!(&mut out, "\tROOT [fillcolor={} label=]\n", DotColor::Red.to_string()).unwrap();
 48 |         let subgraph = self.graph.make_simple_graph(v.unwrap_or(self.version.as_ref()));
 49 | 
 50 |         for txn in subgraph.iter() {
 51 |             // dbg!(txn);
 52 |             let range = txn.span;
 53 | 
 54 |             let parent_item = match txn.parents.len() {
 55 |                 0 => "ROOT".to_string(),
 56 |                 1 => format!("{}", txn.parents[0]),
 57 |                 _ => {
 58 |                     let key = key_for_parents(txn.parents.as_ref());
 59 |                     if merges_touched.insert(key.clone()) {
 60 |                         // Emit the merge item.
 61 |                         write!(&mut out, "\t{key} [fillcolor={} label=\"\" shape=point]\n", DotColor::Blue.to_string()).unwrap();
 62 |                         for &p in txn.parents.iter() {
 63 |                             write!(&mut out, "\t{key} -> {} [label={} color={}]\n", p, p, DotColor::Blue.to_string()).unwrap();
 64 |                         }
 65 |                     }
 66 | 
 67 |                     key
 68 |                 }
 69 |             };
 70 | 
 71 |             write!(&mut out, "\t{} [label=<{} (Len {})>]\n", range.last(), range.start, range.len()).unwrap();
 72 |             write!(&mut out, "\t{} -> {}\n", range.last(), parent_item).unwrap();
 73 |         }
 74 | 
 75 |         out.push_str("}\n");
 76 | 
 77 |         out
 78 |     }
 79 | 
 80 |     pub(crate) fn generate_dot_svg>(&self, out_filename: P, v: Option<&[LV]>) {
 81 |         render_dot_string(self.to_dot_graph(v), out_filename.as_ref());
 82 |     }
 83 | }
 84 | 
 85 | // This is for debugging.
 86 | pub(crate) fn render_dot_string(dot_content: String, out_filename: &Path) {
 87 |     let out_file = File::create(&out_filename).expect("Could not create output file");
 88 |     let dot_path = "dot";
 89 |     let mut child = Command::new(dot_path)
 90 |         // .arg("-Tpng")
 91 |         .arg("-Tsvg")
 92 |         .stdin(Stdio::piped())
 93 |         .stdout(out_file)
 94 |         .stderr(Stdio::piped())
 95 |         .spawn()
 96 |         .expect("Could not run dot");
 97 | 
 98 |     let mut stdin = child.stdin.take().unwrap();
 99 |     // Spawn is needed here to prevent a potential deadlock. See:
100 |     // https://doc.rust-lang.org/std/process/index.html#handling-io
101 |     std::thread::spawn(move || {
102 |         stdin.write_all(dot_content.as_bytes()).unwrap();
103 |     });
104 | 
105 |     let out = child.wait_with_output().unwrap();
106 | 
107 |     // Pipe stderr.
108 |     std::io::stderr().write_all(&out.stderr).unwrap();
109 | 
110 |     println!("Wrote DOT output to {}", out_filename.display());
111 | }
112 | 


--------------------------------------------------------------------------------
/src/causalgraph/enc_fuzzer.rs:
--------------------------------------------------------------------------------
 1 | use rand::prelude::*;
 2 | use crate::{CausalGraph, Frontier};
 3 | use crate::encoding::bufparser::BufParser;
 4 | use crate::encoding::cg_entry::{read_cg_entry_into_cg, write_cg_entry_iter};
 5 | use crate::encoding::map::{ReadMap, WriteMap};
 6 | use crate::list_fuzzer_tools::{choose_2, fuzz_multithreaded};
 7 | 
 8 | fn merge_changes(from_cg: &CausalGraph, into_cg: &mut CausalGraph, from_root: bool) {
 9 |     let from_frontier = if from_root {
10 |         Frontier::root()
11 |     } else {
12 |         let into_summary = into_cg.agent_assignment.summarize_versions_flat();
13 |         // dbg!(&a_summary);
14 |         let (frontier, _remainder) = from_cg.intersect_with_flat_summary(&into_summary, &[]);
15 |         frontier
16 |     };
17 | 
18 |     // Serialize the changes from from_frontier.
19 |     let msg = from_cg.serialize_changes_since(from_frontier.as_ref());
20 | 
21 |     // And merge them in!
22 |     into_cg.merge_serialized_changes(&msg).unwrap();
23 | }
24 | 
25 | /// This fuzzer variant creates linear timelines from 3 different user agents. We still end up with
26 | /// a complex entwined graph, but `(agent, x)` always directly precedes `(agent, x+1)`.
27 | fn fuzz_cg_flat(seed: u64, verbose: bool) {
28 |     let mut rng = SmallRng::seed_from_u64(seed);
29 | 
30 |     let mut cgs = [CausalGraph::new(), CausalGraph::new(), CausalGraph::new()];
31 |     let agents = ["a", "b", "c"];
32 | 
33 |     for c in &mut cgs {
34 |         for a in &agents {
35 |             c.get_or_create_agent_id(*a);
36 |         }
37 |     }
38 | 
39 |     for _i in 0..50 {
40 |         if verbose { println!("\n\ni {}", _i); }
41 | 
42 |         // Generate some operations
43 |         for _j in 0..3 {
44 |             // for _j in 0..5 {
45 |             let idx = rng.gen_range(0..cgs.len());
46 |             let cg = &mut cgs[idx];
47 | 
48 |             let agent_id = cg.get_or_create_agent_id(agents[idx]);
49 |             let num = rng.gen_range(1..10);
50 |             cg.assign_local_op(agent_id, num);
51 |         }
52 | 
53 |         // And merge 2 random causal graphs
54 |         let (_a_idx, a, _b_idx, b) = choose_2(&mut cgs, &mut rng);
55 | 
56 |         merge_changes(a, b, rng.gen_bool(0.04));
57 |         // println!("--\n\n---");
58 |         merge_changes(b, a, rng.gen_bool(0.04));
59 | 
60 |         assert_eq!(a, b);
61 |     }
62 | 
63 |     for cg in cgs {
64 |         cg.dbg_check(true);
65 |     }
66 | }
67 | 
68 | #[test]
69 | fn fuzz_cg_once() {
70 |     fuzz_cg_flat(123, true);
71 | }
72 | 
73 | #[test]
74 | fn fuzz_cg() {
75 |     for k in 0..70 {
76 |         // println!("{k}...");
77 |         fuzz_cg_flat(k, false);
78 |     }
79 | }
80 | 
81 | #[test]
82 | #[ignore]
83 | fn fuzz_cg_forever() {
84 |     fuzz_multithreaded(u64::MAX, |seed| {
85 |         if seed % 1000 == 0 {
86 |             println!("Iteration {}", seed);
87 |         }
88 |         fuzz_cg_flat(seed, false);
89 |     })
90 | }


--------------------------------------------------------------------------------
/src/causalgraph/entry.rs:
--------------------------------------------------------------------------------
 1 | use rle::{HasLength, MergableSpan, SplitableSpan, SplitableSpanHelpers};
 2 | use crate::{DTRange, Frontier, LV};
 3 | use crate::causalgraph::agent_span::AgentSpan;
 4 | 
 5 | #[derive(Clone, Debug, Eq, PartialEq)]
 6 | pub struct CGEntry {
 7 |     pub start: LV,
 8 |     pub parents: Frontier,
 9 |     pub span: AgentSpan,
10 | }
11 | 
12 | impl Default for CGEntry {
13 |     fn default() -> Self {
14 |         CGEntry {
15 |             start: 0,
16 |             parents: Default::default(),
17 |             span: AgentSpan {
18 |                 agent: 0,
19 |                 seq_range: (0..0).into()
20 |             }
21 |         }
22 |     }
23 | }
24 | 
25 | impl HasLength for CGEntry {
26 |     fn len(&self) -> usize {
27 |         self.span.len()
28 |     }
29 | }
30 | 
31 | impl MergableSpan for CGEntry {
32 |     fn can_append(&self, other: &Self) -> bool {
33 |         let end = self.start + self.len();
34 |         (end == other.start)
35 |             && other.parents_are_trivial()
36 |             && self.span.can_append(&other.span)
37 |     }
38 | 
39 |     fn append(&mut self, other: Self) {
40 |         self.span.append(other.span)
41 |         // Other parents don't matter.
42 |     }
43 | }
44 | 
45 | impl CGEntry {
46 |     pub fn parents_are_trivial(&self) -> bool {
47 |         self.parents.len() == 1
48 |             && self.parents[0] == self.start - 1
49 |     }
50 | 
51 |     pub fn time_span(&self) -> DTRange {
52 |         (self.start..self.start + self.len()).into()
53 |     }
54 | 
55 |     pub fn clear(&mut self) {
56 |         self.span.seq_range.clear()
57 |     }
58 | }
59 | 
60 | impl SplitableSpanHelpers for CGEntry {
61 |     #[inline]
62 |     fn truncate_h(&mut self, at: usize) -> Self {
63 |         let other_span = self.span.truncate(at);
64 | 
65 |         Self {
66 |             start: self.start + at,
67 |             parents: Frontier::new_1(self.start + at - 1),
68 |             span: other_span
69 |         }
70 |     }
71 | }


--------------------------------------------------------------------------------
/src/causalgraph/graph/random_graphs.rs:
--------------------------------------------------------------------------------
 1 | //! This file contains a fuzzer-style generator of random causal graphs used to test various
 2 | //! CG functions.
 3 | 
 4 | use std::path::Path;
 5 | use rand::rngs::SmallRng;
 6 | use rand::{Rng, SeedableRng};
 7 | use crate::causalgraph::graph::Graph;
 8 | use crate::{AgentId, CausalGraph, DTRange, Frontier};
 9 | use crate::list_fuzzer_tools::choose_2;
10 | 
11 | pub(crate) fn with_random_cgs(seed: u64, iterations: (usize, usize), mut f: F) {
12 |     for outer in 0..iterations.0 {
13 |         let seed_here = seed + outer as u64;
14 |         let mut rng = SmallRng::seed_from_u64(seed_here);
15 |         // println!("seed {seed_here}");
16 |         let mut frontiers = [Frontier::root(), Frontier::root(), Frontier::root()];
17 |         let mut cg = CausalGraph::new();
18 | 
19 |         let agents = ["a", "b", "c"];
20 |         // Agent IDs 0, 1 and 2.
21 |         for a in agents { cg.get_or_create_agent_id(a); }
22 | 
23 |         // for _i in 0..300 {
24 |         for i in 0..iterations.1 {
25 |             // Generate some "operations" from the peers.
26 |             for _j in 0..2 {
27 |                 let idx = rng.gen_range(0..frontiers.len());
28 |                 let frontier = &mut frontiers[idx];
29 | 
30 |                 let first_change = cg.len();
31 |                 // let span: DTRange = (first_change..first_change + rng.gen_range(1..5)).into();
32 |                 let span: DTRange = (first_change..first_change + 1).into();
33 |                 cg.assign_span(idx as AgentId, frontier.as_ref(), span);
34 | 
35 |                 frontier.replace_with_1(span.last());
36 |             }
37 | 
38 |             // Now randomly merge some frontiers into other frontiers.
39 |             for _j in 0..5 {
40 |                 let (_a_idx, a, _b_idx, b) = choose_2(&mut frontiers, &mut rng);
41 | 
42 |                 *a = cg.graph.find_dominators_2(a.as_ref(), b.as_ref());
43 |             }
44 | 
45 |             f((outer, i), &cg, &frontiers);
46 |         }
47 |     }
48 | }
49 | 
50 | // This generates some graphs to the graphs/ folder.
51 | #[test]
52 | #[ignore]
53 | #[cfg(feature = "dot_export")]
54 | fn generate_some_graphs() {
55 |     with_random_cgs(123, (1, 10), |(_, i), cg, _frontiers| {
56 |         // dbg!(&cg.graph);
57 |         cg.generate_dot_svg(Path::new(&format!("graphs/{i}.svg")));
58 |     });
59 | }


--------------------------------------------------------------------------------
/src/causalgraph/graph/simple.rs:
--------------------------------------------------------------------------------
  1 | use std::collections::BinaryHeap;
  2 | use crate::causalgraph::graph::{Graph, GraphEntrySimple};
  3 | use crate::{DTRange, Frontier, LV};
  4 | use crate::rle::{RleKeyedAndSplitable, RleVec};
  5 | 
  6 | impl Graph {
  7 |     /// This method returns the graph, but split up so parents always refer to the last entry of an
  8 |     /// item. This is useful for debugging, exporting the causal graph and for printing the causal
  9 |     /// graph using DOT.
 10 |     ///
 11 |     /// I'm using RleVec here because the list is packed and sorted, but the entries aren't actually
 12 |     /// fully merged.
 13 |     pub(crate) fn make_simple_graph(&self, frontier: &[LV]) -> RleVec {
 14 |         let mut result = vec![];
 15 | 
 16 |         let mut queue = frontier.iter().copied().collect::>();
 17 | 
 18 |         while let Some(v) = queue.pop() {
 19 |             // println!("Popped {v}");
 20 | 
 21 |             let e = self.entries.find_packed(v);
 22 |             // We could use the entry's end here, but if the frontier is partial it'll end up wrong.
 23 |             let mut span_remaining: DTRange = (e.span.start..v+1).into();
 24 |             // let mut last = v;
 25 | 
 26 |             while let Some(&peek_v) = queue.peek() {
 27 |                 if peek_v < span_remaining.start { break; }
 28 | 
 29 |                 queue.pop();
 30 |                 if peek_v == span_remaining.last() { continue; } // Ignore duplicates.
 31 |                 // println!("- Peeked {peek_v}");
 32 | 
 33 |                 // Emit peek_v+1..=v.
 34 |                 let emit_here = span_remaining.truncate_from(peek_v + 1);
 35 |                 debug_assert!(!emit_here.is_empty());
 36 |                 result.push(GraphEntrySimple {
 37 |                     span: emit_here,
 38 |                     parents: Frontier::new_1(peek_v),
 39 |                 });
 40 |             }
 41 | 
 42 |             debug_assert!(!span_remaining.is_empty());
 43 |             result.push(GraphEntrySimple {
 44 |                 span: span_remaining,
 45 |                 parents: e.parents.clone(),
 46 |             });
 47 | 
 48 |             // Add parents.
 49 |             queue.extend(e.parents.iter().copied());
 50 |             // dbg!(&queue);
 51 |         }
 52 | 
 53 |         result.reverse();
 54 |         RleVec(result)
 55 |     }
 56 | }
 57 | 
 58 | #[cfg(test)]
 59 | mod test {
 60 |     use crate::causalgraph::graph::GraphEntrySimple;
 61 |     use crate::causalgraph::graph::tools::test::fancy_graph;
 62 |     use crate::LV;
 63 | 
 64 |     fn check_simple_graph(g: &[GraphEntrySimple]) {
 65 |         let mut last = usize::MAX;
 66 |         for e in g {
 67 |             assert!(last == usize::MAX || e.span.start > last);
 68 |             last = e.span.last();
 69 | 
 70 |             assert!(!e.span.is_empty());
 71 | 
 72 |             for &p in e.parents.iter() {
 73 |                 assert!(p < e.span.start);
 74 |             }
 75 | 
 76 |             // And the big one: All items which reference this item through their parents must
 77 |             // reference the last entry of our span.
 78 |             for ee in g {
 79 |                 for &p in ee.parents.iter() {
 80 |                     assert!(p < e.span.start || p >= e.span.last(), "Parent points inside this entry");
 81 |                 }
 82 |             }
 83 |         }
 84 |     }
 85 | 
 86 |     #[test]
 87 |     fn fancy_graph_as_simple() {
 88 |         let g = fancy_graph();
 89 | 
 90 |         let check = |f: &[LV]| {
 91 |             let simple_graph = g.make_simple_graph(f);
 92 |             check_simple_graph(&simple_graph.0);
 93 |         };
 94 | 
 95 |         check(&[]);
 96 |         check(&[0]);
 97 |         check(&[3]);
 98 |         check(&[6]);
 99 |         check(&[0, 3]);
100 |         check(&[10]);
101 |         check(&[5, 10]);
102 | 
103 |         // for e in r {
104 |         //     println!("{:?}", e);
105 |         // }
106 |     }
107 | }
108 | 


--------------------------------------------------------------------------------
/src/causalgraph/mod.rs:
--------------------------------------------------------------------------------
 1 | // #![warn(unused)]
 2 | 
 3 | use crate::{DTRange, Frontier, KVPair, Graph};
 4 | use crate::causalgraph::agent_assignment::AgentAssignment;
 5 | 
 6 | pub(crate) mod storage;
 7 | mod causalgraph;
 8 | mod check;
 9 | pub mod graph;
10 | mod eq;
11 | pub mod entry;
12 | pub mod summary;
13 | pub mod agent_span;
14 | pub mod agent_assignment;
15 | 
16 | #[cfg(test)]
17 | mod enc_fuzzer;
18 | #[cfg(feature = "dot_export")]
19 | pub mod dot;
20 | 
21 | #[derive(Clone, Debug, Default)]
22 | pub struct CausalGraph {
23 |     pub agent_assignment: AgentAssignment,
24 | 
25 |     /// Transaction metadata (succeeds, parents) for all operations on this document. This is used
26 |     /// for `diff` and `branchContainsVersion` calls on the document, which is necessary to merge
27 |     /// remote changes.
28 |     ///
29 |     /// At its core, this data set compactly stores the list of parents for every operation.
30 |     pub graph: Graph,
31 | 
32 |     /// This is the version you get if you load the entire causal graph
33 |     pub version: Frontier,
34 | }
35 | 


--------------------------------------------------------------------------------
/src/check.rs:
--------------------------------------------------------------------------------
 1 | // use crate::Branch;
 2 | // use crate::OpLog;
 3 | //
 4 | // impl OpLog {
 5 | //     /// Check the internal state of the oplog. This is only exported for integration testing. You
 6 | //     /// shouldn't have any reason to call this method.
 7 | //     ///
 8 | //     /// This method is public, but do not depend on it as part of the DT API. It could be removed at
 9 | //     /// any time.
10 | //     #[allow(unused)]
11 | //     pub fn dbg_check(&self, deep: bool) {
12 | //         self.cg.dbg_check(deep);
13 | //
14 | //         // for map in self.maps.iter() {
15 | //         //     for (key, item) in map.children.iter() {
16 | //         //         // Each child of a map must be a LWWRegister.
17 | //         //         let child_item = &self.known_crdts[*item];
18 | //         //         assert_eq!(child_item.kind, CRDTKind::LWWRegister);
19 | //         //         assert_eq!(child_item.history.created_at, map.created_at);
20 | //         //     }
21 | //         // }
22 | //
23 | //         // TODO: Check all owned CRDT objects exists in overlay.
24 | //     }
25 | // }
26 | //
27 | // impl Branch {
28 | //     #[allow(unused)]
29 | //     pub fn dbg_check(&self, deep: bool) {
30 | //         if deep {
31 | //             let mut num_invalid_entries = 0;
32 | //
33 | //             todo!();
34 | //             // for (time, value) in &self.overlay {
35 | //             //     match value {
36 | //             //         OverlayValue::LWW(lwwval) => {
37 | //             //
38 | //             //         }
39 | //             //         OverlayValue::Map(_) => {}
40 | //             //         OverlayValue::Set(_) => {}
41 | //             //     }
42 | //             // }
43 | //
44 | //             assert_eq!(num_invalid_entries, self.num_invalid);
45 | //         }
46 | //     }
47 | // }


--------------------------------------------------------------------------------
/src/encoding/chunk_reader.rs:
--------------------------------------------------------------------------------
 1 | use crate::encoding::bufparser::BufParser;
 2 | use crate::encoding::ChunkType;
 3 | use crate::encoding::parseerror::ParseError;
 4 | 
 5 | /// A ChunkReader is a wrapper around some bytes which just contain a series of chunks.
 6 | #[derive(Debug, Clone)]
 7 | pub(crate) struct ChunkReader<'a>(pub BufParser<'a>);
 8 | 
 9 | impl<'a> Iterator for ChunkReader<'a> {
10 |     type Item = Result<(ChunkType, BufParser<'a>), ParseError>;
11 | 
12 |     fn next(&mut self) -> Option {
13 |         if self.0.is_empty() {
14 |             None
15 |         } else {
16 |             Some(self.next_chunk())
17 |         }
18 |     }
19 | }
20 | 
21 | impl<'a> ChunkReader<'a> {
22 |     pub fn is_empty(&self) -> bool {
23 |         self.0.is_empty()
24 |     }
25 | 
26 |     pub fn expect_empty(&self) -> Result<(), ParseError> {
27 |         self.0.expect_empty()
28 |     }
29 | 
30 |     fn next_chunk_raw(&mut self) -> Result<(ChunkType, BufParser<'a>), ParseError> {
31 |         let chunk_type = ChunkType::try_from(self.0.next_u32()?)
32 |             .map_err(|_| ParseError::UnknownChunk);
33 | 
34 |         // This in no way guarantees we're good.
35 |         let len = self.0.next_usize()?;
36 |         if len > self.0.len() {
37 |             return Err(ParseError::InvalidLength);
38 |         }
39 | 
40 |         let reader = BufParser(self.0.next_n_bytes(len)?);
41 | 
42 |         // Note we're try-ing chunk_type here so we still read all the bytes if we can, even if
43 |         // the chunk type is unknown.
44 |         Ok((chunk_type?, reader))
45 |     }
46 | 
47 |     /// Read the next chunk, skipping unknown chunks for forwards compatibility.
48 |     pub fn next_chunk(&mut self) -> Result<(ChunkType, BufParser<'a>), ParseError> {
49 |         loop {
50 |             let c = self.next_chunk_raw();
51 |             match c {
52 |                 Err(ParseError::UnknownChunk) => {}, // Keep scanning.
53 |                 _ => { return c; }
54 |             }
55 |         }
56 |     }
57 | 
58 |     /// Read a chunk with the named type. Returns None if the next chunk isn't the specified type,
59 |     /// or we hit EOF.
60 |     pub fn read_chunk_if_eq(&mut self, expect_chunk_type: ChunkType) -> Result>, ParseError> {
61 |         if let Some(actual_chunk_type) = self.0.peek_u32()? {
62 |             if actual_chunk_type != (expect_chunk_type as u32) {
63 |                 // Chunk doesn't match requested type.
64 |                 return Ok(None);
65 |             }
66 |             self.next_chunk().map(|(_type, c)| Some(c))
67 |         } else {
68 |             // EOF.
69 |             Ok(None)
70 |         }
71 |     }
72 | 
73 |     #[inline]
74 |     pub fn expect_chunk_pred
(&mut self, pred: P, err_type: ChunkType) -> Result<(ChunkType, BufParser<'a>), ParseError>
75 |         where P: FnOnce(ChunkType) -> bool
76 |     {
77 |         let (actual_chunk_type, r) = self.next_chunk()?;
78 | 
79 |         if pred(actual_chunk_type) {
80 |             // dbg!(expect_chunk_type, actual_chunk_type);
81 |             Ok((actual_chunk_type, r))
82 |         } else {
83 |             Err(ParseError::MissingChunk(err_type as _))
84 |         }
85 |     }
86 | 
87 |     pub fn expect_chunk(&mut self, expect_chunk_type: ChunkType) -> Result, ParseError> {
88 |         self.expect_chunk_pred(|c| c == expect_chunk_type, expect_chunk_type)
89 |             .map(|(_c, r)| r)
90 |     }
91 | }


--------------------------------------------------------------------------------
/src/encoding/mod.rs:
--------------------------------------------------------------------------------
  1 | use std::marker::PhantomData;
  2 | use std::mem::replace;
  3 | use rle::MergableSpan;
  4 | use num_enum::TryFromPrimitive;
  5 | 
  6 | /// The encoding module converts the internal data structures to and from a lossless compact binary
  7 | /// data format.
  8 | ///
  9 | /// This is modelled after the run-length encoding in Automerge and Yjs.
 10 | 
 11 | // Notes for next time I break compatibility:
 12 | // - Version in encode::write_local_version - skip second 0 if its ROOT.
 13 | pub mod varint;
 14 | pub(crate) mod bufparser;
 15 | pub(crate) mod parseerror;
 16 | pub(crate) mod tools;
 17 | pub(crate) mod parents;
 18 | pub(crate) mod op_contents;
 19 | pub(crate) mod cg_entry;
 20 | pub(crate) mod op;
 21 | pub(crate) mod chunk_reader;
 22 | pub(crate) mod map;
 23 | // mod agent_assignment;
 24 | 
 25 | 
 26 | #[derive(Debug, PartialEq, Eq, Copy, Clone, TryFromPrimitive)]
 27 | #[repr(u32)]
 28 | pub(crate) enum ChunkType {
 29 |     /// Packed bytes storing any data compressed in later parts of the file.
 30 |     // CompressedFieldsLZ4 = 5,
 31 | 
 32 |     /// FileInfo contains optional UserData and AgentNames.
 33 |     FileInfo = 1,
 34 |     DbId = 2,
 35 |     // AgentNames = 3,
 36 |     UserData = 4,
 37 | 
 38 |     /// The StartBranch chunk describes the state of the document before included patches have been
 39 |     /// applied.
 40 |     StartBranch = 10,
 41 |     Version = 12,
 42 |     // /// StartBranch content is optional.
 43 |     // TextContent = 13,
 44 |     // TextContentCompressed = 14, // Might make more sense to have a generic compression tag for chunks.
 45 | 
 46 |     SetContent = 15,
 47 |     SetContentCompressed = 16,
 48 | 
 49 |     CausalGraph = 21,
 50 |     Operations = 20,
 51 |     // OpTypeAndPosition = 22,
 52 | 
 53 |     // PatchContent = 24,
 54 |     // /// ContentKnown is a RLE expressing which ranges of patches have known content
 55 |     // ContentIsKnown = 25,
 56 | 
 57 |     // TransformedPositions = 27, // Currently unused
 58 | 
 59 |     // Crc = 100,
 60 | }
 61 | 
 62 | #[derive(Clone)]
 63 | pub(super) struct Merger {
 64 |     last: Option,
 65 |     f: F,
 66 |     _ctx: PhantomData // Its pretty silly that this is needed.
 67 | }
 68 | 
 69 | impl Merger {
 70 |     pub fn new(f: F) -> Self {
 71 |         Self { last: None, f, _ctx: PhantomData }
 72 |     }
 73 | 
 74 |     pub fn push2(&mut self, span: S, ctx: &mut Ctx) {
 75 |         if let Some(last) = self.last.as_mut() {
 76 |             if last.can_append(&span) {
 77 |                 last.append(span);
 78 |             } else {
 79 |                 let old = replace(last, span);
 80 |                 (self.f)(old, ctx);
 81 |             }
 82 |         } else {
 83 |             self.last = Some(span);
 84 |         }
 85 |     }
 86 | 
 87 |     pub fn flush2(mut self, ctx: &mut Ctx) {
 88 |         if let Some(span) = self.last.take() {
 89 |             (self.f)(span, ctx);
 90 |         }
 91 |     }
 92 | 
 93 |     pub fn flush_iter2>(mut self, iter: I, ctx: &mut Ctx) {
 94 |         for span in iter {
 95 |             self.push2(span, ctx);
 96 |         }
 97 |         self.flush2(ctx);
 98 |     }
 99 | }
100 | 
101 | impl Merger {
102 |     pub fn push(&mut self, span: S) {
103 |         self.push2(span, &mut ());
104 |     }
105 |     pub fn flush(self) {
106 |         self.flush2(&mut ());
107 |     }
108 |     pub fn flush_iter>(self, iter: I) {
109 |         self.flush_iter2(iter, &mut ());
110 |     }
111 | }
112 | 
113 | impl Drop for Merger {
114 |     fn drop(&mut self) {
115 |         if self.last.is_some() && !std::thread::panicking() {
116 |             panic!("Merger dropped with unprocessed data");
117 |         }
118 |     }
119 | }
120 | 
121 | // #[derive(Debug, Clone)]
122 | // struct EncodingContext {
123 | //     agent_map: AgentMappingEnc,
124 | //     txn_map: TxnMap,
125 | // }
126 | 
127 | 
128 | 
129 | 
130 | #[cfg(test)]
131 | mod test {
132 | 
133 | }
134 | 


--------------------------------------------------------------------------------
/src/encoding/op_contents.rs:
--------------------------------------------------------------------------------
 1 | // use bumpalo::Bump;
 2 | // use bumpalo::collections::vec::Vec as BumpVec;
 3 | // use crate::{OpLog, Primitive, SnapshotValue};
 4 | // use num_enum::TryFromPrimitive;
 5 | // use crate::encoding::tools::{push_str, push_u32, push_u64};
 6 | // use crate::encoding::varint::num_encode_zigzag_i64;
 7 | //
 8 | // #[derive(Debug, PartialEq, Eq, Copy, Clone, TryFromPrimitive)]
 9 | // #[repr(u32)]
10 | // enum ValueType {
11 | //     // TODO: Assign numbers!
12 | //     PrimFalse,
13 | //     PrimTrue,
14 | //
15 | //     PrimSInt,
16 | //     PrimUInt,
17 | //
18 | //     PrimFloat,
19 | //     PrimDouble,
20 | //
21 | //     PrimStr,
22 | //
23 | //     LWWRegister,
24 | //     MVRegister,
25 | //     Map,
26 | //     Set,
27 | //     Text,
28 | //     // TODO: Arbitrary shifty list
29 | // }
30 | //
31 | // pub fn encode_op_contents<'a, 'b: 'a, I: Iterator>(bump: &'a Bump, iter: I, _oplog: &OpLog) -> BumpVec<'a, u8> {
32 | //     let mut result = BumpVec::new_in(bump);
33 | //
34 | //     for val in iter {
35 | //         match val {
36 | //             SnapshotValue::Primitive(Primitive::I64(n)) => {
37 | //                 push_u32(&mut result, ValueType::PrimSInt as u32);
38 | //                 push_u64(&mut result, num_encode_zigzag_i64(*n));
39 | //             }
40 | //             SnapshotValue::Primitive(Primitive::Str(s)) => {
41 | //                 push_u32(&mut result, ValueType::PrimStr as u32);
42 | //                 push_str(&mut result, s);
43 | //             }
44 | //             SnapshotValue::Primitive(_) => {
45 | //                 todo!()
46 | //             }
47 | //             // Value::Map(_) => {
48 | //             //     push_u32(&mut result, ValueType::Map as u32);
49 | //             // }
50 | //             SnapshotValue::InnerCRDT(_crdt_id) => {
51 | //                 todo!();
52 | //                 // let kind = oplog.get_kind(*crdt_id);
53 | //                 // let kind_value = match kind {
54 | //                 //     CRDTKind::LWW => ValueType::LWWRegister,
55 | //                 //     CRDTKind::Map => ValueType::Map,
56 | //                 // };
57 | //                 // push_u32(&mut result, kind_value as u32);
58 | //             }
59 | //         }
60 | //     }
61 | //
62 | //     result
63 | // }
64 | 


--------------------------------------------------------------------------------
/src/encoding/parseerror.rs:
--------------------------------------------------------------------------------
 1 | use std::error::Error;
 2 | use std::fmt::{Display, Formatter};
 3 | use crate::causalgraph::agent_assignment::remote_ids::VersionConversionError;
 4 | 
 5 | 
 6 | // #[derive(Debug)]
 7 | // pub enum ParseError {
 8 | //     GenericInvalidBytes,
 9 | //     InvalidLength,
10 | //     UnexpectedEOF,
11 | //     InvalidVarInt,
12 | // }
13 | 
14 | #[derive(Debug, Eq, PartialEq, Clone, Copy)]
15 | #[non_exhaustive]
16 | pub enum ParseError {
17 |     InvalidMagic,
18 |     UnsupportedProtocolVersion,
19 |     DocIdMismatch,
20 |     BaseVersionUnknown,
21 |     UnknownChunk,
22 |     LZ4DecoderNeeded,
23 |     LZ4DecompressionError, // I'd wrap it but lz4_flex errors don't implement any traits
24 |     // LZ4DecompressionError(lz4_flex::block::DecompressError),
25 |     CompressedDataMissing,
26 |     InvalidChunkHeader,
27 |     MissingChunk(u32),
28 |     // UnexpectedChunk {
29 |     //     // I could use Chunk here, but I'd rather not expose them publicly.
30 |     //     // expected: Chunk,
31 |     //     // actual: Chunk,
32 |     //     expected: u32,
33 |     //     actual: u32,
34 |     // },
35 |     InvalidLength,
36 |     UnexpectedEOF,
37 |     // TODO: Consider elidiing the details here to keep the wasm binary small.
38 |     // InvalidUTF8(Utf8Error),
39 |     InvalidUTF8,
40 |     InvalidRemoteID(VersionConversionError),
41 |     InvalidVarInt,
42 |     InvalidContent,
43 | 
44 |     GenericInvalidData,
45 | 
46 |     ChecksumFailed,
47 | 
48 |     /// This error is interesting. We're loading a chunk but missing some of the data. In the future
49 |     /// I'd like to explicitly support this case, and allow the oplog to contain a somewhat- sparse
50 |     /// set of data, and load more as needed.
51 |     DataMissing,
52 | }
53 | 
54 | impl Display for ParseError {
55 |     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
56 |         write!(f, "ParseError {:?}", self)
57 |     }
58 | }
59 | 
60 | impl Error for ParseError {}
61 | 


--------------------------------------------------------------------------------
/src/fuzzer.rs:
--------------------------------------------------------------------------------
 1 | // use rand::prelude::*;
 2 | // use crate::OpLog;
 3 | //
 4 | // fn merge_fuzz(seed: u64, verbose: bool) {
 5 | //     // A parachute so if the fuzzer crashes we can recover the seed.
 6 | //     let mut rng = SmallRng::seed_from_u64(seed);
 7 | //
 8 | //     let mut oplogs = [OpLog::new(), OpLog::new(), OpLog::new()];
 9 | //     let agents = ["a", "b", "c"];
10 | //
11 | //     for _i in 0..300 {
12 | //         if verbose { println!("\n\ni {}", _i); }
13 | //
14 | //         // Generate some operations
15 | //         for _j in 0..2 {
16 | //             // for _j in 0..5 {
17 | //             let idx = rng.gen_range(0..oplogs.len());
18 | //             let oplog = &mut oplogs[idx];
19 | //
20 | //
21 | //         }
22 | //     }
23 | // }


--------------------------------------------------------------------------------
/src/list/buffered_iter.rs:
--------------------------------------------------------------------------------
 1 | use std::ops::Deref;
 2 | 
 3 | /// This is a simple iterator wrapper which has a buffer, and allows an item to be "put back" on
 4 | /// the iterator.
 5 | 
 6 | #[derive(Debug, Clone)]
 7 | pub(crate) struct BufferedIter {
 8 |     inner: Iter,
 9 |     buffer: Option,
10 | }
11 | 
12 | impl Iterator for BufferedIter {
13 |     type Item = Iter::Item;
14 | 
15 |     fn next(&mut self) -> Option {
16 |         let buffered = self.buffer.take();
17 |         if buffered.is_some() {
18 |             buffered
19 |         } else {
20 |             self.inner.next()
21 |         }
22 |     }
23 | }
24 | 
25 | impl BufferedIter {
26 |     pub fn new(inner: Iter) -> Self {
27 |         Self {
28 |             inner,
29 |             buffer: None
30 |         }
31 |     }
32 | 
33 |     pub fn push_back(&mut self, item: Iter::Item) {
34 |         assert!(self.buffer.is_none());
35 |         self.buffer = Some(item);
36 |     }
37 | }
38 | 
39 | impl From for BufferedIter {
40 |     fn from(iter: Iter) -> Self {
41 |         Self::new(iter)
42 |     }
43 | }
44 | 
45 | impl Deref for BufferedIter {
46 |     type Target = Iter;
47 | 
48 |     fn deref(&self) -> &Self::Target {
49 |         &self.inner
50 |     }
51 | }
52 | 
53 | pub(crate) trait Buffered: Iterator + Sized {
54 |     fn buffered(self) -> BufferedIter {
55 |         self.into()
56 |     }
57 | }
58 | 
59 | impl Buffered for T {}


--------------------------------------------------------------------------------
/src/list/check.rs:
--------------------------------------------------------------------------------
 1 | use jumprope::JumpRope;
 2 | use crate::list::{ListBranch, ListCRDT, ListOpLog};
 3 | 
 4 | /// This file contains debugging assertions to validate the document's internal state.
 5 | ///
 6 | /// This is used during fuzzing to make sure everything is working properly, and if not, find bugs
 7 | /// as early as possible.
 8 | 
 9 | impl ListBranch {
10 |     #[allow(unused)]
11 |     pub fn dbg_assert_content_eq_rope(&self, expected_content: &JumpRope) {
12 |         assert_eq!(&self.content, expected_content);
13 |     }
14 | 
15 | 
16 | }
17 | 
18 | impl ListOpLog {
19 |     /// Check the internal state of the diamond types list. This is only exported for integration
20 |     /// testing.
21 |     ///
22 |     /// You shouldn't have any reason to call this method.
23 |     ///
24 |     /// This method is public, but do not depend on it as part of the DT API. It could be removed at
25 |     /// any time.
26 |     #[allow(unused)]
27 |     pub fn dbg_check(&self, deep: bool) {
28 |         self.cg.dbg_check(deep);
29 |     }
30 | 
31 |     #[allow(unused)]
32 |     pub(crate) fn check_all_changes_rle_merged(&self) {
33 |         assert_eq!(self.cg.agent_assignment.client_data[0].lv_for_seq.num_entries(), 1);
34 |         // .. And operation log.
35 |         assert_eq!(self.cg.graph.entries.num_entries(), 1);
36 |     }
37 | }
38 | 
39 | impl ListCRDT {
40 |     /// Check the internal state of the diamond types document. This is only exported for
41 |     /// integration testing.
42 |     ///
43 |     /// You shouldn't have any reason to call this method.
44 |     ///
45 |     /// This method is public, but do not depend on it as part of the DT API. It could be removed at
46 |     /// any time.
47 |     #[allow(unused)]
48 |     pub fn dbg_check(&self, deep: bool) {
49 |         self.oplog.dbg_check(deep);
50 |     }
51 | }


--------------------------------------------------------------------------------
/src/list/encoding/encode_options.rs:
--------------------------------------------------------------------------------
  1 | use crate::list::ListOpLog;
  2 | use crate::LV;
  3 | 
  4 | // TODO: Make a builder API for this
  5 | #[derive(Debug, Clone)]
  6 | pub struct EncodeOptions<'a> {
  7 |     pub(crate) user_data: Option<&'a [u8]>,
  8 | 
  9 |     // NYI.
 10 |     // from_version: LocalVersion,
 11 | 
 12 |     pub(crate) store_start_branch_content: bool,
 13 | 
 14 |     /// Experimental.
 15 |     pub(crate) store_end_branch_content: bool,
 16 | 
 17 |     pub(crate) store_inserted_content: bool,
 18 |     pub(crate) store_deleted_content: bool,
 19 | 
 20 |     pub(crate) compress_content: bool,
 21 | 
 22 |     pub(crate) verbose: bool,
 23 | 
 24 |     pub(crate) store_xf: bool,
 25 |     pub(crate) sort: bool,
 26 | }
 27 | 
 28 | 
 29 | pub const ENCODE_PATCH: EncodeOptions = EncodeOptions {
 30 |     user_data: None,
 31 |     store_start_branch_content: false,
 32 |     store_end_branch_content: false,
 33 |     store_inserted_content: true,
 34 |     store_deleted_content: false,
 35 |     compress_content: true,
 36 |     verbose: false,
 37 |     // sort_events:
 38 |     store_xf: false,
 39 |     sort: false,
 40 | };
 41 | 
 42 | pub const ENCODE_FULL: EncodeOptions = EncodeOptions {
 43 |     user_data: None,
 44 |     store_start_branch_content: true,
 45 |     store_end_branch_content: false,
 46 |     store_inserted_content: true,
 47 |     store_deleted_content: false, // ?? Not sure about this one!
 48 |     compress_content: true,
 49 |     verbose: false,
 50 |     store_xf: false,
 51 |     sort: false,
 52 | };
 53 | 
 54 | impl<'a> Default for EncodeOptions<'a> {
 55 |     fn default() -> Self {
 56 |         ENCODE_FULL
 57 |     }
 58 | }
 59 | 
 60 | // pub struct EncodeOptionsBuilder<'a>(EncodeOptions<'a>);
 61 | 
 62 | impl<'a> EncodeOptions<'a> {
 63 |     pub fn patch() -> Self {
 64 |         ENCODE_PATCH.clone()
 65 |     }
 66 |     pub fn full() -> Self {
 67 |         ENCODE_FULL.clone()
 68 |     }
 69 | 
 70 |     pub fn encode_from(&self, oplog: &ListOpLog, from_version: &[LV]) -> Vec {
 71 |         oplog.encode_from(self, from_version)
 72 |     }
 73 | 
 74 |     pub fn user_data(mut self, data: &'a [u8]) -> Self {
 75 |         self.user_data = Some(data);
 76 |         self
 77 |     }
 78 | 
 79 |     pub fn store_start_branch_content(mut self, store_start_branch_content: bool) -> Self {
 80 |         self.store_start_branch_content = store_start_branch_content;
 81 |         self
 82 |     }
 83 | 
 84 |     pub fn experimentally_store_end_branch_content(mut self, store: bool) -> Self {
 85 |         self.store_end_branch_content = store;
 86 |         self
 87 |     }
 88 | 
 89 |     pub fn store_inserted_content(mut self, store_inserted_content: bool) -> Self {
 90 |         self.store_inserted_content = store_inserted_content;
 91 |         self
 92 |     }
 93 | 
 94 |     pub fn store_deleted_content(mut self, store_deleted_content: bool) -> Self {
 95 |         self.store_deleted_content = store_deleted_content;
 96 |         self
 97 |     }
 98 | 
 99 |     pub fn compress_content(mut self, compress_content: bool) -> Self {
100 |         self.compress_content = compress_content;
101 |         self
102 |     }
103 | 
104 |     pub fn verbose(mut self, verbose: bool) -> Self {
105 |         self.verbose = verbose;
106 |         self
107 |     }
108 | 
109 |     pub fn sort_operations(mut self, sort: bool) -> Self {
110 |         self.sort = sort;
111 |         self
112 |     }
113 |     
114 |     /// Implies sorting
115 |     pub fn store_xf(mut self, store_xf: bool) -> Self {
116 |         self.store_xf = store_xf;
117 |         self.sort = true;
118 |         self
119 |     }
120 | 
121 |     pub fn build(self) -> EncodeOptions<'a> {
122 |         self
123 |     }
124 | }
125 | 
126 | pub type EncodeOptionsBuilder<'a> = EncodeOptions<'a>;


--------------------------------------------------------------------------------
/src/list/encoding/encode_tools.rs:
--------------------------------------------------------------------------------
  1 | use std::mem::{replace, size_of};
  2 | use rle::{MergableSpan, RleRun};
  3 | use std::marker::PhantomData;
  4 | use crate::list::encoding::ListChunkType;
  5 | use crate::encoding::varint::mix_bit_usize;
  6 | 
  7 | #[cfg(feature = "serde")]
  8 | use serde::Serialize;
  9 | use crate::list::encoding::leb::{encode_leb_u32, encode_leb_u64};
 10 | 
 11 | pub(super) fn push_leb_u32(into: &mut Vec, val: u32) {
 12 |     let mut buf = [0u8; 5];
 13 |     let pos = encode_leb_u32(val, &mut buf);
 14 |     into.extend_from_slice(&buf[..pos]);
 15 | }
 16 | 
 17 | pub(super) fn push_leb_u64(into: &mut Vec, val: u64) {
 18 |     let mut buf = [0u8; 10];
 19 |     let pos = encode_leb_u64(val, &mut buf);
 20 |     into.extend_from_slice(&buf[..pos]);
 21 | }
 22 | 
 23 | pub(super) fn push_leb_usize(into: &mut Vec, val: usize) {
 24 |     if size_of::() <= size_of::() {
 25 |         push_leb_u32(into, val as u32);
 26 |     } else if size_of::() == size_of::() {
 27 |         push_leb_u64(into, val as u64);
 28 |     } else {
 29 |         panic!("usize larger than u64 is not supported");
 30 |     }
 31 | }
 32 | 
 33 | pub(super) fn push_leb_str(into: &mut Vec, val: &str) {
 34 |     let bytes = val.as_bytes();
 35 |     push_leb_usize(into, bytes.len());
 36 |     into.extend_from_slice(bytes);
 37 | }
 38 | 
 39 | pub(super) fn push_u32_le(into: &mut Vec, val: u32) {
 40 |     // This is used for the checksum. Using LE because varint is LE.
 41 |     let bytes = val.to_le_bytes();
 42 |     into.extend_from_slice(&bytes);
 43 | }
 44 | 
 45 | fn push_leb_chunk_header(into: &mut Vec, chunk_type: ListChunkType, len: usize) {
 46 |     push_leb_u32(into, chunk_type as u32);
 47 |     push_leb_usize(into, len);
 48 | }
 49 | 
 50 | pub(super) fn push_leb_chunk(into: &mut Vec, chunk_type: ListChunkType, data: &[u8], verbose: bool) {
 51 |     if verbose {
 52 |         println!("Chunk {:?} - size {}", chunk_type, data.len());
 53 |     }
 54 |     push_leb_chunk_header(into, chunk_type, data.len());
 55 |     into.extend_from_slice(data);
 56 | }
 57 | 
 58 | pub(super) fn write_leb_bit_run(run: RleRun, into: &mut Vec) {
 59 |     // dbg!(run);
 60 |     let mut n = run.len;
 61 |     n = mix_bit_usize(n, run.val);
 62 |     push_leb_usize(into, n);
 63 | }
 64 | 
 65 | #[derive(Clone)]
 66 | pub(super) struct Merger {
 67 |     pub(super) last: Option,
 68 |     f: F,
 69 |     _ctx: PhantomData // This is awful.
 70 | }
 71 | 
 72 | impl Merger {
 73 |     pub fn new(f: F) -> Self {
 74 |         Self { last: None, f, _ctx: PhantomData }
 75 |     }
 76 | 
 77 |     pub fn push2(&mut self, span: S, ctx: &mut Ctx) {
 78 |         if let Some(last) = self.last.as_mut() {
 79 |             if last.can_append(&span) {
 80 |                 last.append(span);
 81 |             } else {
 82 |                 let old = replace(last, span);
 83 |                 (self.f)(old, ctx);
 84 |             }
 85 |         } else {
 86 |             self.last = Some(span);
 87 |         }
 88 |     }
 89 | 
 90 |     pub fn flush2(mut self, ctx: &mut Ctx) {
 91 |         if let Some(span) = self.last.take() {
 92 |             (self.f)(span, ctx);
 93 |         }
 94 |     }
 95 | }
 96 | 
 97 | // I hate this.
 98 | impl Merger {
 99 |     pub fn push(&mut self, span: S) {
100 |         self.push2(span, &mut ());
101 |     }
102 |     pub fn flush(self) {
103 |         self.flush2(&mut ());
104 |     }
105 | }
106 | 
107 | impl Drop for Merger {
108 |     fn drop(&mut self) {
109 |         if self.last.is_some() && !std::thread::panicking() {
110 |             panic!("Merger dropped with unprocessed data");
111 |         }
112 |     }
113 | }
114 | 


--------------------------------------------------------------------------------
/src/list/encoding/fuzzer.rs:
--------------------------------------------------------------------------------
  1 | use rand::prelude::*;
  2 | use crate::list::{ListCRDT, ListOpLog};
  3 | use crate::list::encoding::EncodeOptions;
  4 | use crate::list::old_fuzzer_tools::old_make_random_change;
  5 | use crate::list_fuzzer_tools::{choose_2, make_random_change};
  6 | use crate::listmerge::simple_oplog::{SimpleBranch, SimpleOpLog};
  7 | 
  8 | // This fuzzer will make an oplog, spam it with random changes from a single peer. Then save & load
  9 | // it back to make sure the result doesn't change.
 10 | fn fuzz_encode_decode_once(seed: u64) {
 11 |     let mut doc = ListCRDT::new();
 12 |     doc.get_or_create_agent_id("a"); // 0
 13 |     doc.get_or_create_agent_id("b"); // 1
 14 |     doc.get_or_create_agent_id("c"); // 2
 15 | 
 16 |     let mut rng = SmallRng::seed_from_u64(seed);
 17 | 
 18 |     for _i in 0..300 {
 19 |         // println!("\n\nIteration {i}");
 20 |         let agent = rng.gen_range(0..3);
 21 |         for _k in 0..rng.gen_range(1..=3) {
 22 |             old_make_random_change(&mut doc, None, agent, &mut rng, true);
 23 |         }
 24 | 
 25 |         let bytes = doc.oplog.encode(&EncodeOptions::full().store_deleted_content(true));
 26 | 
 27 |         let decoded = ListOpLog::load_from(&bytes).unwrap();
 28 |         if doc.oplog != decoded {
 29 |             // eprintln!("Original doc {:#?}", &doc.ops);
 30 |             // eprintln!("Loaded doc {:#?}", &decoded);
 31 |             panic!("Docs do not match!");
 32 |         }
 33 |         // assert_eq!(decoded, doc.ops);
 34 |     }
 35 | }
 36 | 
 37 | #[test]
 38 | fn encode_decode_fuzz_once() {
 39 |     fuzz_encode_decode_once(2);
 40 | }
 41 | 
 42 | #[test]
 43 | #[ignore]
 44 | fn encode_decode_fuzz_forever() {
 45 |     for seed in 0.. {
 46 |         if seed % 10 == 0 { println!("seed {seed}"); }
 47 |         fuzz_encode_decode_once(seed);
 48 |     }
 49 | }
 50 | 
 51 | fn agent_name(i: usize) -> String {
 52 |     format!("agent {}", i)
 53 | }
 54 | 
 55 | // This fuzzer makes 3 oplogs, and merges patches between them.
 56 | fn fuzz_encode_decode_multi(seed: u64, verbose: bool) {
 57 |     let mut rng = SmallRng::seed_from_u64(seed);
 58 |     let mut docs = [ListCRDT::new(), ListCRDT::new(), ListCRDT::new()];
 59 | 
 60 |     for i in 0..docs.len() {
 61 |         // for a in 0..3 {
 62 |         //     docs[i].get_or_create_agent_id(agent_name(a).as_str());
 63 |         // }
 64 |         docs[i].get_or_create_agent_id(agent_name(i).as_str());
 65 |     }
 66 | 
 67 |     for _i in 0..50 {
 68 |         if verbose { println!("\n\ni {}", _i); }
 69 |         // Generate some operations
 70 |         for _j in 0..2 {
 71 |             // for _j in 0..5 {
 72 |             let idx = rng.gen_range(0..docs.len());
 73 |             let doc = &mut docs[idx];
 74 | 
 75 |             // make_random_change(doc, None, idx as AgentId, &mut rng);
 76 |             old_make_random_change(doc, None, 0, &mut rng, true);
 77 |         }
 78 | 
 79 |         let (a_idx, a, b_idx, b) = choose_2(&mut docs, &mut rng);
 80 | 
 81 |         // Merge by applying patches
 82 |         // let b_agent = a.get_or_create_agent_id(agent_name(b_idx).as_str());
 83 | 
 84 |         let encode_opts = EncodeOptions::full().store_deleted_content(true);
 85 |         let a_data = a.oplog.encode(&encode_opts);
 86 |         b.merge_data_and_ff(&a_data).unwrap();
 87 | 
 88 |         let b_data = b.oplog.encode(&encode_opts);
 89 |         a.merge_data_and_ff(&b_data).unwrap();
 90 | 
 91 |         if a.oplog != b.oplog {
 92 |             println!("Docs {} and {} after {} iterations:", a_idx, b_idx, _i);
 93 |             dbg!(&a);
 94 |             dbg!(&b);
 95 |             panic!("Documents do not match");
 96 |         } else {
 97 |             if verbose {
 98 |                 println!("Merge {:?} -> '{}'", &a.oplog.cg.version, &a.branch.content);
 99 |             }
100 |         }
101 |     }
102 | }
103 | 
104 | 
105 | #[test]
106 | fn encode_decode_multi_fuzz_once() {
107 |     fuzz_encode_decode_multi(10, false);
108 | }
109 | 
110 | #[test]
111 | #[ignore]
112 | fn encode_decode_multi_fuzz_forever() {
113 |     for seed in 0.. {
114 |         if seed % 20 == 0 { println!("seed {seed}"); }
115 |         fuzz_encode_decode_multi(seed, false);
116 |     }
117 | }
118 | 


--------------------------------------------------------------------------------
/src/list/encoding/mod.rs:
--------------------------------------------------------------------------------
 1 | /// This module contains the old (original) text encoding format code. This code is used by
 2 | /// some early applications written using diamond types. It will probably be moved behind a feature
 3 | /// flag once the new encoding code gets written.
 4 | 
 5 | mod encode_oplog;
 6 | mod decode_oplog;
 7 | 
 8 | #[cfg(test)]
 9 | mod tests;
10 | #[cfg(test)]
11 | mod fuzzer;
12 | pub mod encode_tools;
13 | mod decode_tools;
14 | pub mod save_transformed;
15 | pub(crate) mod leb;
16 | pub(crate) mod txn_trace;
17 | mod encode_options;
18 | 
19 | use rle::MergableSpan;
20 | use crate::encoding::varint::*;
21 | use num_enum::TryFromPrimitive;
22 | pub use encode_options::{EncodeOptions, EncodeOptionsBuilder, ENCODE_FULL, ENCODE_PATCH};
23 | 
24 | const MAGIC_BYTES: [u8; 8] = *b"DMNDTYPS";
25 | 
26 | const PROTOCOL_VERSION: usize = 0;
27 | 
28 | // #[derive(Debug, PartialEq, Eq, Copy, Clone)]
29 | #[derive(Debug, PartialEq, Eq, Copy, Clone, TryFromPrimitive)]
30 | #[repr(u32)]
31 | enum ListChunkType {
32 |     /// Packed bytes storing any data compressed in later parts of the file.
33 |     CompressedFieldsLZ4 = 5,
34 | 
35 |     /// FileInfo contains optional UserData and AgentNames.
36 |     FileInfo = 1,
37 |     DocId = 2,
38 |     AgentNames = 3,
39 |     UserData = 4,
40 | 
41 |     /// The StartBranch chunk describes the state of the document before included patches have been
42 |     /// applied.
43 |     StartBranch = 10,
44 |     ExperimentalEndBranch = 11,
45 |     Version = 12,
46 |     /// StartBranch content is optional.
47 |     Content = 13,
48 |     ContentCompressed = 14, // Might make more sense to have a generic compression tag for chunks.
49 | 
50 |     Patches = 20,
51 |     OpVersions = 21,
52 |     OpTypeAndPosition = 22,
53 |     OpParents = 23,
54 | 
55 |     PatchContent = 24,
56 |     /// ContentKnown is a RLE expressing which ranges of patches have known content
57 |     ContentIsKnown = 25,
58 | 
59 |     /// A chunk specifying which operations are cancelled when the data is transformed
60 |     TransformedCancelsOps = 27,
61 |     /// A chunk specifying the position deltas for operations when transformed in the stored order
62 |     TransformedPositions = 28,
63 | 
64 |     Crc = 100,
65 | }
66 | 
67 | #[derive(Debug, PartialEq, Eq, Copy, Clone, TryFromPrimitive)]
68 | #[repr(u32)]
69 | enum DataType {
70 |     Bool = 1,
71 |     VarUInt = 2,
72 |     VarInt = 3,
73 |     PlainText = 4,
74 | }
75 | 
76 | #[derive(Debug, PartialEq, Eq, Copy, Clone, TryFromPrimitive)]
77 | #[repr(u32)]
78 | enum CompressionFormat {
79 |     // Just for future proofing, ya know?
80 |     LZ4 = 1,
81 | }
82 | 


--------------------------------------------------------------------------------
/src/list/encoding/save_transformed.rs:
--------------------------------------------------------------------------------
  1 | use rle::{AppendRle, HasLength, RleRun};
  2 | use crate::encoding::Merger;
  3 | use crate::list::encoding::leb::num_encode_zigzag_isize_old;
  4 | use crate::list::encoding::encode_tools::{push_leb_usize, write_leb_bit_run};
  5 | use crate::list::ListOpLog;
  6 | use crate::listmerge::merge::TransformedResultRaw;
  7 | use crate::LV;
  8 | 
  9 | /// *** This is EXPERIMENTAL work-in-progress code to save transformed positions ***
 10 | 
 11 | /// This feature isn't implemented yet, but I wanted to get some benchmarking figures for my blog
 12 | /// post.
 13 | 
 14 | 
 15 | #[derive(Debug, Eq, PartialEq, Clone, Copy)]
 16 | enum XFState {
 17 |     Cancelled,
 18 |     XFBy(isize),
 19 | }
 20 | 
 21 | impl ListOpLog {
 22 |     pub fn bench_writing_xf_since(&self, from_version: &[LV]) {
 23 |         let mut tn_ops: Vec> = vec![];
 24 | 
 25 |         // for r in self.get_xf_operations_full_raw(from_version, self.cg.version.as_ref()) {
 26 |         //     tn_ops.push_rle(match r {
 27 |         //         TransformedResultRaw::FF(range) => {
 28 |         //             RleRun::new(XFState::XFBy(0), range.len())
 29 |         //         },
 30 |         //         TransformedResultRaw::Apply { xf_pos, op } => {
 31 |         //             RleRun::new(XFState::XFBy(xf_pos as isize - op.1.start() as isize), op.len())
 32 |         //         },
 33 |         //         TransformedResultRaw::DeleteAlreadyHappened(range) => {
 34 |         //             RleRun::new(XFState::Cancelled, range.len())
 35 |         //         },
 36 |         //     });
 37 |         // }
 38 | 
 39 |         
 40 |         for op in self.get_xf_operations_full(from_version, self.cg.version.as_ref()) {
 41 |             let (val, len) = match op {
 42 |                 TransformedResultRaw::FF(range) => {
 43 |                     (XFState::XFBy(0), range.len())
 44 |                 }
 45 |                 TransformedResultRaw::Apply { xf_pos, op } => {
 46 |                     let origin_pos = op.1.start() as isize;
 47 |                     (XFState::XFBy(xf_pos as isize - origin_pos), op.len())
 48 |                 }
 49 |                 TransformedResultRaw::DeleteAlreadyHappened(range) => {
 50 |                     (XFState::Cancelled, range.len())
 51 |                 }
 52 |             };
 53 |             
 54 |             tn_ops.push_rle(RleRun { val, len });
 55 |         }
 56 | 
 57 |         // for (_, op, xf) in self.get_xf_operations_full_old(from_version, self.cg.version.as_ref()) {
 58 |         //     let val = match xf {
 59 |         //         TransformedResult::BaseMoved(xf_pos) => {
 60 |         //             let origin_pos = op.start() as isize;
 61 |         //             XFState::XFBy(xf_pos as isize - origin_pos)
 62 |         //         },
 63 |         //         TransformedResult::DeleteAlreadyHappened => XFState::Cancelled,
 64 |         //     };
 65 |         // 
 66 |         //     tn_ops.push_rle(RleRun {
 67 |         //         val,
 68 |         //         len: op.len()
 69 |         //     });
 70 |         // }
 71 | 
 72 |         dbg!(&tn_ops.len());
 73 | 
 74 |         // First pass: just write it.
 75 | 
 76 |         let mut buf = Vec::new();
 77 |         let mut buf2 = Vec::new();
 78 |         // let mut last: isize = 0;
 79 | 
 80 |         // let mut onoff = Vec::new();
 81 | 
 82 |         let mut w = Merger::new(write_leb_bit_run);
 83 | 
 84 |         for e in tn_ops.iter() {
 85 |             // onoff.push_rle(RleRun::new(e.val == XFState::Cancelled, e.len));
 86 |             w.push2(RleRun::new(e.val == XFState::Cancelled, e.len), &mut buf2);
 87 | 
 88 |             // if e.len > 10000 {
 89 |             //     dbg!(e);
 90 |             // }
 91 |             match e.val {
 92 |                 XFState::Cancelled => {}
 93 |                 XFState::XFBy(dist) => {
 94 |                     let n = num_encode_zigzag_isize_old(dist);
 95 |                     push_leb_usize(&mut buf, n);
 96 |                     push_leb_usize(&mut buf, e.len);
 97 |                 }
 98 |             }
 99 |         }
100 | 
101 |         w.flush2(&mut buf2);
102 | 
103 |         dbg!(buf2.len() + buf.len());
104 |         dbg!(buf2.len());
105 |         // dbg!(onoff.len());
106 | 
107 |         // dbg!(&onoff);
108 | 
109 | 
110 |         // 2.65kb.
111 |         // let mut buf = Vec::new();
112 |         // for e in tn_ops.iter() {
113 |         //     match e.val {
114 |         //         XFState::Cancelled => {}
115 |         //         XFState::XFBy(dist) => {
116 |         //             let n = num_encode_zigzag_isize(dist);
117 |         //             push_usize(&mut buf, n);
118 |         //             push_usize(&mut buf, e.len);
119 |         //         }
120 |         //     }
121 |         // }
122 | 
123 |         dbg!(buf.len());
124 |     }
125 | }
126 | 


--------------------------------------------------------------------------------
/src/list/gen_random.rs:
--------------------------------------------------------------------------------
 1 | use rand::prelude::*;
 2 | use crate::AgentId;
 3 | use crate::list::{ListBranch, ListOpLog};
 4 | use crate::list::old_fuzzer_tools::old_make_random_change_raw;
 5 | use crate::list_fuzzer_tools::choose_2;
 6 | 
 7 | /// This is a simple generator for random test data used for conformance tests.
 8 | pub fn gen_oplog(seed: u64, steps: usize, use_unicode: bool, interleave_agents: bool) -> ListOpLog {
 9 |     let verbose = false;
10 | 
11 |     let mut rng = SmallRng::seed_from_u64(seed);
12 |     let mut oplog = ListOpLog::new();
13 |     let mut branches = [
14 |         ListBranch::new(), ListBranch::new(), ListBranch::new(),
15 |         // ListBranch::new(), ListBranch::new(), ListBranch::new(),
16 |     ];
17 | 
18 |     let agents = ["a", "b", "c"];
19 |     // let agents = ["a", "b", "c", "d", "e", "f"];
20 |     for a in agents {
21 |         // This makes agent 'a' have agent_id 0, 'b' = 1, 'c' = 2, ...
22 |         oplog.get_or_create_agent_id(a);
23 |     }
24 | 
25 |     for _i in 0..steps {
26 |         if verbose { println!("\n\ni {}", _i); }
27 |         // Generate some operations
28 |         // for _j in 0..2 {
29 |         for _j in 0..5 {
30 |             let idx = rng.gen_range(0..branches.len());
31 |             let branch = &mut branches[idx];
32 | 
33 |             let agent = if interleave_agents {
34 |                 // Pick a random agent to use with the new operation(s).
35 |                 rng.gen_range(0..agents.len())
36 |             } else {
37 |                 // Just use the agent corresponding to the branch.
38 |                 idx
39 |             } as AgentId;
40 | 
41 |             // This should + does also work if we set idx=0 and use the same agent for all changes.
42 |             let v = old_make_random_change_raw(&mut oplog, branch, None, agent, &mut rng, use_unicode);
43 |             branch.merge(&oplog, &[v]);
44 |             // println!("branch {} content '{}'", idx, &branch.content);
45 |         }
46 | 
47 |         // Then merge 2 branches at random
48 |         // TODO: Rewrite this to use choose_2.
49 |         let (_a_idx, a, _b_idx, b) = choose_2(&mut branches, &mut rng);
50 | 
51 |         a.merge(&oplog, b.version.as_ref());
52 |         b.merge(&oplog, a.version.as_ref());
53 | 
54 |         // Our frontier should contain everything in the document.
55 |         debug_assert_eq!(a, b);
56 | 
57 |         if _i % 50 == 0 {
58 |             // Every little while, merge everything. This has 2 purposes:
59 |             // 1. It stops the fuzzer being n^2. (Its really unfortunate we need this)
60 |             // And 2. It makes sure n-way merging also works correctly.
61 |             for branch in branches.iter_mut() {
62 |                 branch.merge(&oplog, oplog.local_frontier_ref());
63 |                 // oplog.merge_all(branch);
64 |             }
65 |             for w in branches.windows(2) {
66 |                 debug_assert_eq!(w[0].content, w[1].content);
67 |             }
68 |         }
69 |     }
70 | 
71 |     // let result = oplog.checkout_tip().content.to_string();
72 |     // (oplog, result)
73 |     oplog
74 | }
75 | 
76 | #[test]
77 | fn generates_simple_oplog() {
78 |     let _oplog = gen_oplog(123, 10, false, false);
79 |     // dbg!(oplog);
80 | }


--------------------------------------------------------------------------------
/src/list/oplog_merge_fuzzer.rs:
--------------------------------------------------------------------------------
 1 | use rand::prelude::*;
 2 | 
 3 | use crate::list::ListCRDT;
 4 | use crate::list::old_fuzzer_tools::old_make_random_change;
 5 | use crate::list_fuzzer_tools::choose_2;
 6 | 
 7 | fn oplog_merge_fuzz(seed: u64, n: usize, verbose: bool) {
 8 |     let mut rng = SmallRng::seed_from_u64(seed);
 9 |     let mut docs = [ListCRDT::new(), ListCRDT::new(), ListCRDT::new()];
10 | 
11 |     for i in 0..docs.len() {
12 |         // docs[i].get_or_create_agent_id(format!("agent {}", i).as_str());
13 |         for a in 0..docs.len() {
14 |             docs[i].get_or_create_agent_id(format!("agent {}", a).as_str());
15 |         }
16 |     }
17 | 
18 |     for _i in 0..n {
19 |         if verbose { println!("\n\ni {}", _i); }
20 | 
21 |         // for (idx, d) in docs.iter().enumerate() {
22 |         //     println!("doc {idx} length {}", d.ops.len());
23 |         // }
24 | 
25 |         // Generate some operations
26 |         for _j in 0..2 {
27 |             let idx = rng.gen_range(0..docs.len());
28 | 
29 |             // This should + does also work if we set idx=0 and use the same agent for all changes.
30 |             // make_random_change(&mut docs[idx], None, 0, &mut rng);
31 |             old_make_random_change(&mut docs[idx], None, idx as _, &mut rng, false);
32 |         }
33 | 
34 |         // for (idx, d) in docs.iter().enumerate() {
35 |         //     println!("with changes {idx} length {}", d.ops.len());
36 |         // }
37 | 
38 |         let (_a_idx, a, _b_idx, b) = choose_2(&mut docs, &mut rng);
39 | 
40 |         // a.ops.dbg_print_assignments_and_ops();
41 |         // println!("\n");
42 |         // b.ops.dbg_print_assignments_and_ops();
43 | 
44 |         // dbg!((&a.ops, &b.ops));
45 |         a.oplog.add_missing_operations_from(&b.oplog);
46 |         // a.check(true);
47 |         // println!("->c {_a_idx} length {}", a.ops.len());
48 | 
49 |         b.oplog.add_missing_operations_from(&a.oplog);
50 |         // b.check(true);
51 |         // println!("->c {_b_idx} length {}", b.ops.len());
52 | 
53 | 
54 |         // dbg!((&a.ops, &b.ops));
55 | 
56 |         assert_eq!(a.oplog, b.oplog);
57 | 
58 |         a.branch.merge(&a.oplog, a.oplog.cg.version.as_ref());
59 |         b.branch.merge(&b.oplog, b.oplog.cg.version.as_ref());
60 |         // assert_eq!(a.branch.content.to_string(), b.branch.content.to_string());
61 |         assert_eq!(a.branch.content, b.branch.content);
62 | 
63 | 
64 |         // let mut new_oplog = ListOpLog::new();
65 |         // for (op, graph, agent_span) in a.oplog.iter_full() {
66 |         //     // I'm going to ignore the agent span and just let it extend naturally.
67 |         //     let agent = new_oplog.get_or_create_agent_id(agent_span.0);
68 |         //     new_oplog.add_operations_at(agent, graph.parents.as_ref(), &[op]);
69 |         // }
70 |         //
71 |         // assert_eq!(new_oplog, a.oplog);
72 |     }
73 | 
74 |     for doc in &docs {
75 |         doc.dbg_check(true);
76 |     }
77 | }
78 | 
79 | #[test]
80 | fn oplog_merge_fuzz_once() {
81 |     oplog_merge_fuzz(1000139, 100, true);
82 | }
83 | 
84 | #[test]
85 | #[ignore]
86 | fn oplog_merge_fuzz_forever() {
87 |     for seed in 0.. {
88 |         if seed % 10 == 0 { println!("seed {seed}"); }
89 |         oplog_merge_fuzz(seed, 100, false);
90 |     }
91 | }


--------------------------------------------------------------------------------
/src/list/stochastic_summary.rs:
--------------------------------------------------------------------------------
 1 | // WIP.
 2 | 
 3 | use crate::causalgraph::agent_span::AgentVersion;
 4 | use crate::list::ListOpLog;
 5 | use crate::LV;
 6 | 
 7 | impl ListOpLog {
 8 | 
 9 |     /// Sooo, when 2 peers love each other very much...
10 |     ///
11 |     /// They connect together. And they need to find the shared point in time from which they should
12 |     /// send changes.
13 |     ///
14 |     /// Over the network this problem fundamentally pits round-trip time against bandwidth overhead.
15 |     /// The algorithmic approach which would result in the fewest round-trips would just be for both
16 |     /// peers to send their entire histories immediately. But this would waste bandwidth. And the
17 |     /// approach using the least bandwidth would have peers essentially do a distributed binary
18 |     /// search to find a common point in time. But this would take log(n) round-trips, and over long
19 |     /// network distances this is really slow.
20 |     ///
21 |     /// In practice this is usually mostly unnecessary - usually one peer's version is a direct
22 |     /// ancestor of the other peer's version. (Eg, I'm modifying a document and you're just
23 |     /// observing it.)
24 |     ///
25 |     /// Ny design here is a hybrid approach. I'm going to construct a fixed-sized chunk of known
26 |     /// versions we can send to our remote peer. (And the remote peer can do the same with us). The
27 |     /// chunk will contain exponentially less information the further back in time we scan; so the
28 |     /// more time which has passed since we have a common ancestor, the more wasted bytes of changes
29 |     /// we'll send to the remote peer. But this approach will always only need 1RTT to sync.
30 |     ///
31 |     /// Its not perfect, but it'll do donkey. It'll do.
32 |     #[allow(unused)]
33 |     fn get_stochastic_version(&self, target_count: usize) -> Vec {
34 |         // TODO: WIP.
35 |         let target_count = target_count.max(self.cg.version.len());
36 |         let mut result = Vec::with_capacity(target_count + 10);
37 | 
38 |         let time_len = self.len();
39 | 
40 |         // If we have no changes, just return the empty set. Descending from ROOT is implied anyway.
41 |         if time_len == 0 { return result; }
42 | 
43 |         let mut push_time = |t: LV| {
44 |             result.push(self.cg.agent_assignment.local_to_agent_version(t));
45 |         };
46 | 
47 |         // No matter what, we'll send the current frontier:
48 |         for t in self.cg.version.iter() {
49 |             push_time(*t);
50 |         }
51 | 
52 |         // So we want about target_count items. I'm assuming there's an exponentially decaying
53 |         // probability of syncing as we go further back in time. This is a big assumption - and
54 |         // probably not true in practice. But it'll do. (TODO: Quadratic might be better?)
55 |         //
56 |         // Given factor, the approx number of operations we'll return is log_f(|ops|).
57 |         // Solving for f gives f = |ops|^(1/target).
58 |         if target_count > self.cg.version.len() {
59 |             // Note I'm using n_ops here rather than time, since this easily scales time by the
60 |             // approximate size of the transmitted operations. TODO: This might be a faulty
61 |             // assumption given we're probably sending inserted content? Hm!
62 |             let remaining_count = target_count - self.cg.version.len();
63 |             let n_ops = self.operations.0.len();
64 |             let mut factor = f32::powf(n_ops as f32, 1f32 / (remaining_count) as f32);
65 |             factor = factor.max(1.1);
66 | 
67 |             let mut t_inv = 1f32;
68 |             while t_inv < time_len as f32 {
69 |                 dbg!(t_inv);
70 |                 push_time(time_len - (t_inv as usize));
71 |                 t_inv *= factor;
72 |             }
73 |         }
74 | 
75 |         result
76 |     }
77 | }
78 | 
79 | #[cfg(test)]
80 | mod tests {
81 |     use crate::list::ListOpLog;
82 | 
83 |     #[test]
84 |     fn test_versions_since() {
85 |         let mut oplog = ListOpLog::new();
86 |         // Should be an empty set
87 |         assert_eq!(oplog.get_stochastic_version(10), &[]);
88 | 
89 |         oplog.get_or_create_agent_id("seph");
90 |         oplog.add_insert(0, 0, "a");
91 |         oplog.add_insert(0, 0, "a");
92 |         oplog.add_insert(0, 0, "a");
93 |         oplog.add_insert(0, 0, "a");
94 |         dbg!(oplog.get_stochastic_version(10));
95 |     }
96 | }


--------------------------------------------------------------------------------
/src/listmerge/README.md:
--------------------------------------------------------------------------------
 1 | # Merging code
 2 | 
 3 | This internal module contains the code for merging changes into a branch. This is a difficult problem because sometimes changes are concurrent.
 4 | 
 5 | For example, consider a document with 3 inserts producing the document "abc", but where the `a` and `b` characters were typed concurrently. We end up with a time dag like this:
 6 | 
 7 | ```
 8 |  ROOT
 9 |  / \
10 | a   |
11 | |   b
12 |  \ /
13 |   c
14 | ```
15 | 
16 | Internally, the oplog stores this information as a table:
17 | 
18 | | Time | Content | Position | Parents |
19 | |------|---------|----------|---------|
20 | | 0    | `a`     | 0        | ROOT    |
21 | | 1    | `b`     | **0**    | ROOT    |
22 | | 2    | `c`     | 2        | 0 + 1   |
23 | 
24 | > *Aside:* This is simplified - DT stores a few more fields, including author (agentID), sequence number and operation type (insert or delete).
25 | 
26 | This table is interesting:
27 | 
28 | - Both the `a` and `b` characters have a position of 0. If we replayed this editing trace naively, we'd end up with `bac` instead of `abc`.
29 | - We can detect that those two inserts were concurrent because neither operation is an ancestor of the other change.
30 | 
31 | The problem when replaying this editing history is figuring out the "merged position". Ie, when replaying these inserts, where does each insert go? We can think of this as needing to fill in another column on this table:
32 | 
33 | | Time | Content | Position | Parents | **Merge Pos** |
34 | |------|---------|----------|---------|---------------|
35 | | 0    | `a`     | 0        | ROOT    | ???           |
36 | | 1    | `b`     | **0**    | ROOT    | ???           |
37 | | 2    | `c`     | 2        | 0 + 1   | ???           |
38 | 
39 | Some of these merge position values are easy to fill in. When the parents set contains all previous edits, the merge position is the same as the origin position:
40 | 
41 | | Time | Content | Position | Parents | Merge Pos |
42 | |------|---------|----------|---------|-----------|
43 | | 0    | `a`     | 0        | ROOT    | **0**     |
44 | | 1    | `b`     | **0**    | ROOT    | ???       |
45 | | 2    | `c`     | 2        | 0 + 1   | **2**     |
46 | 
47 | But figuring out the merge position of the `b` character, we need to do a lot more work!
48 | 
49 | This module contains one algorithm to solve this problem.
50 | 
51 | ## The algorithm
52 | 
53 | There's a few different algorithmic ideas I've considered while working on diamond types. This code contains the first correct algorithm I've implemented, which works as follows.
54 | 
55 | First, the algorithm depends on a special data structure called a *tracker*. A tracker stores a set of operations (inserts or deletes) in *document order*. That is, in the order in which the changes appear in the final document. The tracker has two magic tricks:
56 | 
57 | 1. Tracked operations can be toggled on and off.
58 | 2. The tracker can map positions from the current state (ignoring operations which are toggled off) to the state when all operations are enabled. 
59 | 
60 | 
61 | The merging algorithm then essentially does the following steps:
62 | 
63 | 1. Create an empty tracker
64 | 2. For each operation, toggle operations in the tracker such that the tracker represents the state of the document when that operation was performed
65 | 3. Map the operation's location to the merged location using the tracker.
66 | 4. Use the mapped location to modify the document
67 | 
68 | The tracker is a b-tree of [`YjsSpan`](yjsspan.rs) items. Its implemented using the b-tree implementation in [`content-tree`](../../../../content-tree), with a custom index found in [`metrics.rs`](metrics.rs).
69 | 
70 | The code in [`txn_trace.rs`](../list/encoding/txn_trace.rs) implements an iterator over all changes in the document. Iteration order is somewhat complicated in order to avoid a few pathological cases.
71 | 
72 | The merging algorithm itself is in [`merge.rs`](merge.rs).


--------------------------------------------------------------------------------
/src/listmerge/mod.rs:
--------------------------------------------------------------------------------
 1 | //! This module is the second implementation for handling positional updates. Instead of generating
 2 | //! a series of patches and merging them, this code applies patches by making a positional map and
 3 | //! moving backwards in time.
 4 | //!
 5 | //! There's a bunch of ways this code could be written:
 6 | //!
 7 | //! 1. We could store the content tree + position map in the same structure or separately (as in
 8 | //! PositionMap in dt-crdt)
 9 | //! 2. When moving around, we could either scan the list and rewrite it (activating and deactivating
10 | //! entries as we go). Or we could figure it out by walking the txns forwards and backwards through
11 | //! time.
12 | 
13 | use crate::listmerge::markers::Marker;
14 | use crate::listmerge::yjsspan::CRDTSpan;
15 | use crate::ost::content_tree::ContentTree;
16 | use crate::ost::IndexTree;
17 | 
18 | pub(crate) mod yjsspan;
19 | pub(crate) mod merge;
20 | pub(crate) mod markers;
21 | mod advance_retreat;
22 | // pub(crate) mod txn_trace;
23 | #[cfg(test)]
24 | pub mod fuzzer;
25 | #[cfg(feature = "dot_export")]
26 | mod dot;
27 | 
28 | #[cfg(any(test, feature = "gen_test_data"))]
29 | pub(crate) mod simple_oplog;
30 | pub(crate) mod plan;
31 | 
32 | pub(crate) mod xf_old;
33 | 
34 | type Index = IndexTree;
35 | 
36 | #[derive(Debug)]
37 | struct M2Tracker {
38 |     /// The index is used for 2 things:
39 |     ///
40 |     /// - For inserts, this contains a pointer to the node in range_tree which contains this version
41 |     /// - For deletes, this names the time at which the delete happened.
42 |     index: Index,
43 |     
44 |     range_tree: ContentTree,
45 | 
46 |     #[cfg(feature = "merge_conflict_checks")]
47 |     concurrent_inserts_collide: bool,
48 | }
49 | 


--------------------------------------------------------------------------------
/src/listmerge/simple_oplog.rs:
--------------------------------------------------------------------------------
  1 | use std::ops::Range;
  2 | use jumprope::JumpRopeBuf;
  3 | use smartstring::SmartString;
  4 | use rle::HasLength;
  5 | use crate::list::operation::TextOperation;
  6 | use crate::{CausalGraph, Frontier, LV};
  7 | use crate::causalgraph::graph::Graph;
  8 | use crate::textinfo::TextInfo;
  9 | use crate::list::op_iter::{OpMetricsWithContent, OpMetricsIter};
 10 | use crate::unicount::count_chars;
 11 | 
 12 | #[derive(Debug, Default)]
 13 | pub(crate) struct SimpleOpLog {
 14 |     pub cg: CausalGraph,
 15 |     pub info: TextInfo,
 16 | }
 17 | 
 18 | #[derive(Debug, Default, Clone, Eq, PartialEq)]
 19 | pub(crate) struct SimpleBranch {
 20 |     pub content: JumpRopeBuf,
 21 | 
 22 |     // Always points to a version in the subgraph.
 23 |     pub version: Frontier,
 24 | }
 25 | 
 26 | impl SimpleOpLog {
 27 |     pub(crate) fn new() -> Self {
 28 |         Self::default()
 29 |     }
 30 | 
 31 |     pub(crate) fn goop(&mut self, n: usize) -> LV {
 32 |         // Just going to use agent 0 here.
 33 |         if self.cg.agent_assignment.client_data.is_empty() {
 34 |             self.cg.get_or_create_agent_id("goopy");
 35 |         }
 36 | 
 37 |         self.cg.assign_local_op(0, n).last()
 38 |     }
 39 | 
 40 |     pub(crate) fn add_operation(&mut self, agent_name: &str, op: TextOperation) -> LV  {
 41 |         let agent = self.cg.get_or_create_agent_id(agent_name);
 42 |         let len = op.len();
 43 |         let range = self.cg.assign_local_op(agent, len);
 44 |         self.info.local_push_op(op, range);
 45 |         range.last()
 46 |     }
 47 | 
 48 |     pub(crate) fn add_operation_at(&mut self, agent_name: &str, parents: &[LV], op: TextOperation) -> LV  {
 49 |         let agent = self.cg.get_or_create_agent_id(agent_name);
 50 |         let len = op.len();
 51 |         let range = self.cg.assign_local_op_with_parents(parents, agent, len);
 52 |         self.info.remote_push_op(op, range, parents, &self.cg.graph);
 53 |         range.last()
 54 |     }
 55 | 
 56 |     pub(crate) fn add_insert_at(&mut self, agent_name: &str, parents: &[LV], pos: usize, content: &str) -> LV {
 57 |         self.add_operation_at(agent_name, parents, TextOperation::new_insert(pos, content))
 58 |     }
 59 | 
 60 |     pub(crate) fn add_insert(&mut self, agent_name: &str, pos: usize, content: &str) -> LV {
 61 |         self.add_operation(agent_name, TextOperation::new_insert(pos, content))
 62 |     }
 63 | 
 64 |     pub(crate) fn add_delete_at(&mut self, agent_name: &str, parents: &[LV], del_range: Range) -> LV {
 65 |         self.add_operation_at(agent_name, parents, TextOperation::new_delete(del_range))
 66 |     }
 67 | 
 68 |     pub(crate) fn add_delete(&mut self, agent_name: &str, del_range: Range) -> LV {
 69 |         self.add_operation(agent_name, TextOperation::new_delete(del_range))
 70 |     }
 71 | 
 72 |     pub(crate) fn to_string(&self) -> String {
 73 |         let mut result = JumpRopeBuf::new();
 74 |         self.info.merge_into(&mut result, &self.cg, &[], self.cg.version.as_ref());
 75 |         result.to_string()
 76 |     }
 77 | 
 78 |     pub(crate) fn merge_raw(&self, into: &mut JumpRopeBuf, from: &[LV], to: &[LV]) -> Frontier {
 79 |         self.info.merge_into(into, &self.cg, from, to)
 80 |     }
 81 | 
 82 |     pub(crate) fn merge_all(&self, into: &mut SimpleBranch) {
 83 |         into.version = self.merge_raw(&mut into.content, into.version.as_ref(), self.cg.version.as_ref());
 84 |     }
 85 | 
 86 |     pub(crate) fn merge_to_version(&self, into: &mut SimpleBranch, to_version: &[LV]) {
 87 |         into.version = self.merge_raw(&mut into.content, into.version.as_ref(), to_version);
 88 |     }
 89 | 
 90 |     pub(crate) fn dbg_check(&self, deep: bool) {
 91 |         // TODO: Check the op ctx makes sense I guess?
 92 |         self.cg.dbg_check(deep);
 93 |     }
 94 | }
 95 | 
 96 | impl SimpleBranch {
 97 |     pub fn new() -> Self {
 98 |         Default::default()
 99 |     }
100 | 
101 |     pub fn len(&self) -> usize {
102 |         self.content.len_chars()
103 |     }
104 | 
105 |     pub fn is_empty(&self) -> bool {
106 |         self.content.is_empty()
107 |     }
108 | 
109 |     pub fn make_delete_op(&self, loc: Range) -> TextOperation {
110 |         assert!(loc.end <= self.content.len_chars());
111 |         let mut s = SmartString::new();
112 |         s.extend(self.content.borrow().slice_chars(loc.clone()));
113 |         TextOperation::new_delete_with_content_range(loc, s)
114 |     }
115 | }
116 | 


--------------------------------------------------------------------------------
/src/listmerge2/README.md:
--------------------------------------------------------------------------------
1 | # Merge algorithm 2
2 | 
3 | This is a separate, alternate sequence CRDT merge algorithm I've been working on in diamond types. Its not yet ready, and not yet enabled.


--------------------------------------------------------------------------------
/src/listmerge2/mod.rs:
--------------------------------------------------------------------------------
 1 | mod action_plan;
 2 | mod test_conversion;
 3 | 
 4 | // #[cfg(feature = "dot_export")]
 5 | mod dot;
 6 | mod index_gap_buffer;
 7 | mod yjsspan;
 8 | 
 9 | use std::cmp::Ordering;
10 | use std::collections::BinaryHeap;
11 | use smallvec::{SmallVec, smallvec};
12 | use rle::SplitableSpan;
13 | use crate::{DTRange, Frontier, LV};
14 | use crate::causalgraph::graph::tools::DiffFlag;
15 | 
16 | type Index = usize;
17 | 
18 | 
19 | 
20 | // #[test]
21 | // fn foo() {
22 | //     let a = RevSortFrontier::from(1);
23 | //     let b = RevSortFrontier::from([0usize, 1].as_slice());
24 | //     dbg!(a.cmp(&b));
25 | // }
26 | 
27 | // fn peek_when_matches bool>(heap: &BinaryHeap, pred: F) -> Option<&T> {
28 | //     if let Some(peeked) = heap.peek() {
29 | //         if pred(peeked) {
30 | //             return Some(peeked);
31 | //         }
32 | //     }
33 | //     None
34 | // }
35 | 


--------------------------------------------------------------------------------
/src/simple_checkout.rs:
--------------------------------------------------------------------------------
 1 | use std::collections::BTreeMap;
 2 | use crate::{Branch, CRDTKind, LV, Primitive, RegisterValue, ROOT_CRDT_ID};
 3 | use smartstring::alias::String as SmartString;
 4 | 
 5 | #[derive(Debug, Clone)]
 6 | pub enum SimpleVal {
 7 |     Text(String),
 8 |     Map(BTreeMap>),
 9 |     Primitive(Primitive),
10 | }
11 | 
12 | impl Branch {
13 |     fn simple_val_at(&self, key: LV, kind: CRDTKind) -> SimpleVal {
14 |         match kind {
15 |             CRDTKind::Map => {
16 |                 let mut map = BTreeMap::new();
17 |                 for (key, state) in self.maps.get(&key).unwrap() {
18 |                     // TODO: Rewrite this as an iterator map then collect().
19 |                     map.insert(key.clone(), Box::new(match &state.value {
20 |                         RegisterValue::Primitive(primitive) => {
21 |                             SimpleVal::Primitive(primitive.clone())
22 |                         }
23 |                         RegisterValue::OwnedCRDT(inner_kind, inner_key) => {
24 |                             self.simple_val_at(*inner_key, *inner_kind)
25 |                         }
26 |                     }));
27 |                 }
28 |                 SimpleVal::Map(map)
29 |             }
30 |             CRDTKind::Register => {
31 |                 // TODO
32 |                 SimpleVal::Primitive(Primitive::Nil)
33 |             }
34 |             CRDTKind::Collection => {
35 |                 // todo!();
36 |                 SimpleVal::Primitive(Primitive::Nil)
37 |             }
38 |             CRDTKind::Text => {
39 |                 SimpleVal::Text(self.texts.get(&key).unwrap().to_string())
40 |             }
41 |         }
42 |     }
43 | 
44 |     pub fn simple_val(&self) -> SimpleVal {
45 |         self.simple_val_at(ROOT_CRDT_ID, CRDTKind::Map)
46 |     }
47 | }


--------------------------------------------------------------------------------
/src/stats.rs:
--------------------------------------------------------------------------------
 1 | #![allow(unused)]
 2 | 
 3 | #[cfg(feature = "stats")]
 4 | use std::cell::RefCell;
 5 | 
 6 | #[cfg(feature = "stats")]
 7 | thread_local! {
 8 |     static CACHE_HITS: RefCell = RefCell::default();
 9 |     static CACHE_MISSES: RefCell = RefCell::default();
10 |     static AS: RefCell = RefCell::default();
11 |     static BS: RefCell = RefCell::default();
12 |     static CS: RefCell = RefCell::default();
13 | }
14 | 
15 | pub(crate) fn cache_hit() {
16 |     #[cfg(feature = "stats")] {
17 |         let old_val = CACHE_HITS.take();
18 |         CACHE_HITS.set(old_val + 1);
19 |     }
20 | }
21 | 
22 | pub(crate) fn cache_miss() {
23 |     #[cfg(feature = "stats")] {
24 |         let old_val = CACHE_MISSES.take();
25 |         CACHE_MISSES.set(old_val + 1);
26 |     }
27 | }
28 | 
29 | pub(crate) fn marker_a() {
30 |     #[cfg(feature = "stats")] {
31 |         let old_val = AS.take();
32 |         AS.set(old_val + 1);
33 |     }
34 | }
35 | pub(crate) fn marker_b() {
36 |     #[cfg(feature = "stats")] {
37 |         let old_val = BS.take();
38 |         BS.set(old_val + 1);
39 |     }
40 | }
41 | pub(crate) fn marker_c() {
42 |     #[cfg(feature = "stats")] {
43 |         let old_val = CS.take();
44 |         CS.set(old_val + 1);
45 |     }
46 | }
47 | 
48 | /// Returns (cache hits, cache misses).
49 | pub fn take_stats() -> (usize, usize) {
50 |     #[cfg(feature = "stats")] {
51 |         let (a, b, c) = (AS.take(), BS.take(), CS.take());
52 |         if a != 0 || b != 0 || c != 0 {
53 |             println!("A: {a} / B: {b} / C: {c}");
54 |         }
55 |         
56 |         (CACHE_HITS.take(), CACHE_MISSES.take())
57 |     }
58 | 
59 |     #[cfg(not(feature = "stats"))] {
60 |         (0, 0)
61 |     }
62 | }


--------------------------------------------------------------------------------
/src/storage/README.md:
--------------------------------------------------------------------------------
 1 | # Storage engine
 2 | 
 3 | This is a simple storage engine which can incrementally persist changes to DT documents on disk.
 4 | 
 5 | The storage engine is:
 6 | 
 7 | - Atomic (writes have either happened or they haven't)
 8 | - Incremental (when data changes, we don't need to re-save the entire history of a document)
 9 | 
10 | It does not yet support:
11 | 
12 | - Reads in `log(n)` time
13 | - Pruning
14 | 
15 | Each DT document has its oplog saved as a single file on disk.
16 | 
17 | 
18 | ## On disk layout
19 | 
20 | The file on disk is made up of a bunch of fixed size (4k) blocks. Every write overwrites an entire block - which sounds wasteful, but this plays nicely with modern NVMe block devices.
21 | 
22 | The data set is made of a bunch of data types, each encoded as a separate column. The columns are:
23 | 
24 | - Agent IDs
25 | - Causal graph (agent assignment & parents information)
26 | - Operations (???)


--------------------------------------------------------------------------------
/src/textinfo.rs:
--------------------------------------------------------------------------------
 1 | use jumprope::JumpRopeBuf;
 2 | use rle::HasLength;
 3 | use crate::causalgraph::graph::Graph;
 4 | use crate::dtrange::DTRange;
 5 | use crate::frontier::Frontier;
 6 | use crate::list::ListOpLog;
 7 | use crate::list::op_iter::{OpMetricsWithContent, OpMetricsIter};
 8 | use crate::list::op_metrics::{ListOperationCtx, ListOpMetrics};
 9 | use crate::list::operation::{ListOpKind, TextOperation};
10 | use crate::listmerge::merge::reverse_str;
11 | use crate::LV;
12 | use crate::rle::KVPair;
13 | use crate::rle::rle_vec::RleVec;
14 | 
15 | #[derive(Debug, Clone, Default)]
16 | pub(crate) struct TextInfo {
17 |     pub(crate) ctx: ListOperationCtx,
18 |     pub(crate) ops: RleVec>,
19 |     pub(crate) frontier: Frontier,
20 | }
21 | 
22 | impl TextInfo {
23 |     pub fn iter_metrics_range(&self, range: DTRange) -> OpMetricsIter {
24 |         OpMetricsIter::new(&self.ops, &self.ctx, range)
25 |     }
26 |     pub fn iter_metrics(&self) -> OpMetricsIter {
27 |         OpMetricsIter::new(&self.ops, &self.ctx, (0..self.ops.end()).into())
28 |     }
29 | 
30 |     pub fn iter_fast(&self) -> OpMetricsWithContent {
31 |         self.iter_metrics().into()
32 |     }
33 | 
34 |     pub fn iter(&self) -> impl Iterator + '_ {
35 |         self.iter_fast().map(|pair| (pair.0.1, pair.1).into())
36 |     }
37 | 
38 |     fn push_op_internal(&mut self, op: TextOperation, v_range: DTRange) {
39 |         debug_assert_eq!(v_range.len(), op.len());
40 | 
41 |         let content_pos = op.content.as_ref().map(|content| {
42 |             self.ctx.push_str(op.kind, content)
43 |         });
44 | 
45 |         self.ops.push(KVPair(v_range.start, ListOpMetrics {
46 |             loc: op.loc,
47 |             kind: op.kind,
48 |             content_pos
49 |         }));
50 |     }
51 | 
52 |     pub fn remote_push_op(&mut self, op: TextOperation, v_range: DTRange, parents: &[LV], graph: &Graph) {
53 |         self.push_op_internal(op, v_range);
54 |         // // TODO: Its probably simpler to just call advance_sparse() here.
55 |         // let local_parents = graph.project_onto_subgraph_raw(
56 |         //     subgraph_rev_iter(&self.ops),
57 |         //     parents
58 |         // );
59 |         // self.frontier.advance_by_known_run(local_parents.as_ref(), v_range);
60 |         self.frontier.advance_sparse_known_run(graph, parents, v_range);
61 |     }
62 | 
63 |     pub fn remote_push_op_unknown_parents(&mut self, op: TextOperation, v_range: DTRange, graph: &Graph) {
64 |         self.push_op_internal(op, v_range);
65 |         self.frontier.advance_sparse(graph, v_range);
66 |     }
67 | 
68 |     pub fn local_push_op(&mut self, op: TextOperation, v_range: DTRange) {
69 |         self.push_op_internal(op, v_range);
70 |         self.frontier.replace_with_1(v_range.last());
71 |     }
72 | 
73 |     #[inline(always)]
74 |     pub(crate) fn apply_op_to(&self, op: ListOpMetrics, dest: &mut JumpRopeBuf) {
75 |         // let xf_pos = op.loc.span.start;
76 |         match op.kind {
77 |             ListOpKind::Ins => {
78 |                 let content = self.ctx.get_str(ListOpKind::Ins, op.content_pos.unwrap());
79 |                 // assert!(pos <= self.content.len_chars());
80 |                 if op.loc.fwd {
81 |                     dest.insert(op.loc.span.start, content);
82 |                 } else {
83 |                     // We need to insert the content in reverse order.
84 |                     let c = reverse_str(content);
85 |                     dest.insert(op.loc.span.start, &c);
86 |                 }
87 |             }
88 |             ListOpKind::Del => {
89 |                 dest.remove(op.loc.span.into());
90 |             }
91 |         }
92 |     }
93 | 
94 | }
95 | 


--------------------------------------------------------------------------------
/src/unicount.rs:
--------------------------------------------------------------------------------
  1 | /// This is a tiny library to convert from codepoint offsets in a utf-8 string to byte offsets, and
  2 | /// back.
  3 | ///
  4 | /// Its super weird that rust doesn't have anything like this in the standard library (as far as I
  5 | /// can tell). You can fake it with char_indices().nth()... but the resulting generated code is
  6 | /// *awful*.
  7 | 
  8 | pub fn chars_to_bytes(s: &str, char_pos: usize) -> usize {
  9 |     // For all that my implementation above is correct and tight, ropey's char_to_byte_idx is
 10 |     // already being pulled in anyway by ropey, and its faster. Just use that.
 11 |     str_indices::chars::to_byte_idx(s, char_pos)
 12 | }
 13 | 
 14 | pub fn split_at_char(s: &str, char_pos: usize) -> (&str, &str) {
 15 |     s.split_at(chars_to_bytes(s, char_pos))
 16 | }
 17 | 
 18 | #[inline]
 19 | #[allow(unused)]
 20 | pub fn consume_chars<'a>(content: &mut &'a str, len: usize) -> &'a str {
 21 |     let (here, remaining) = split_at_char(content, len);
 22 |     *content = remaining;
 23 |     here
 24 | }
 25 | 
 26 | #[inline]
 27 | #[allow(unused)]
 28 | pub fn bytes_to_chars(s: &str, byte_pos: usize) -> usize {
 29 |     str_indices::chars::from_byte_idx(s, byte_pos)
 30 | }
 31 | 
 32 | pub fn count_chars(s: &str) -> usize {
 33 |     str_indices::chars::count(s)
 34 | }
 35 | 
 36 | #[cfg(test)]
 37 | mod test {
 38 |     use crate::unicount::*;
 39 | 
 40 |     fn std_chars_to_bytes(s: &str, char_pos: usize) -> usize {
 41 |         s.char_indices().nth(char_pos).map_or_else(
 42 |             || s.len(),
 43 |             |(i, _)| i
 44 |         )
 45 |     }
 46 | 
 47 |     pub fn std_bytes_to_chars(s: &str, byte_pos: usize) -> usize {
 48 |         s[..byte_pos].chars().count()
 49 |     }
 50 | 
 51 |     const TRICKY_CHARS: &[&str] = &[
 52 |         "a", "b", "c", "1", "2", "3", " ", "\n", // ASCII
 53 |         "©", "¥", "½", // The Latin-1 suppliment (U+80 - U+ff)
 54 |         "Ύ", "Δ", "δ", "Ϡ", // Greek (U+0370 - U+03FF)
 55 |         "←", "↯", "↻", "⇈", // Arrows (U+2190 – U+21FF)
 56 |         "𐆐", "𐆔", "𐆘", "𐆚", // Ancient roman symbols (U+10190 – U+101CF)
 57 |     ];
 58 | 
 59 |     fn check_matches(s: &str) {
 60 |         let char_len = s.chars().count();
 61 |         for i in 0..=char_len {
 62 |             let actual_bytes = std_chars_to_bytes(s, i);
 63 |             let ropey_bytes = str_indices::chars::to_byte_idx(s, i);
 64 |             // dbg!(expected, actual);
 65 |             assert_eq!(ropey_bytes, actual_bytes);
 66 | 
 67 |             let std_chars = std_bytes_to_chars(s, actual_bytes);
 68 |             let ropey_chars = bytes_to_chars(s, actual_bytes);
 69 | 
 70 |             assert_eq!(std_chars, i);
 71 |             assert_eq!(ropey_chars, i);
 72 |         }
 73 |     }
 74 | 
 75 |     #[test]
 76 |     fn str_pos_works() {
 77 |         check_matches("hi");
 78 |         check_matches("");
 79 |         for s in TRICKY_CHARS {
 80 |             check_matches(*s);
 81 |         }
 82 | 
 83 |         // And throw them all in a big string.
 84 |         let mut big_str = String::new();
 85 |         for s in TRICKY_CHARS {
 86 |             big_str.push_str(*s);
 87 |         }
 88 |         check_matches(big_str.as_str());
 89 |     }
 90 | 
 91 |     #[test]
 92 |     fn test_split_at_char() {
 93 |         assert_eq!(split_at_char("", 0), ("", ""));
 94 |         assert_eq!(split_at_char("hi", 0), ("", "hi"));
 95 |         assert_eq!(split_at_char("hi", 1), ("h", "i"));
 96 |         assert_eq!(split_at_char("hi", 2), ("hi", ""));
 97 | 
 98 |         assert_eq!(split_at_char("日本語", 0), ("", "日本語"));
 99 |         assert_eq!(split_at_char("日本語", 1), ("日", "本語"));
100 |         assert_eq!(split_at_char("日本語", 2), ("日本", "語"));
101 |         assert_eq!(split_at_char("日本語", 3), ("日本語", ""));
102 |     }
103 | }
104 | 
105 | 


--------------------------------------------------------------------------------
/test_data/causal_graph/diff.json:
--------------------------------------------------------------------------------
 1 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[0,1]},{"span":[3,4],"parents":[0,1]}],"a":[2],"b":[3],"expect_a":[[2,3]],"expect_b":[[3,4]]}
 2 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[0]}],"a":[2],"b":[],"expect_a":[[2,3],[0,1]],"expect_b":[]}
 3 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[0]}],"a":[2],"b":[1],"expect_a":[[2,3],[0,1]],"expect_b":[[1,2]]}
 4 | {"hist":[{"span":[0,3],"parents":[]},{"span":[3,5],"parents":[]},{"span":[5,6],"parents":[2,4]}],"a":[4],"b":[5],"expect_a":[],"expect_b":[[5,6],[0,3]]}
 5 | {"hist":[{"span":[0,3],"parents":[]},{"span":[3,5],"parents":[]},{"span":[5,6],"parents":[2,4]}],"a":[4],"b":[],"expect_a":[[3,5]],"expect_b":[]}
 6 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[]}],"a":[0],"b":[0,1],"expect_a":[],"expect_b":[[1,2]]}
 7 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[]}],"a":[0],"b":[],"expect_a":[[0,1]],"expect_b":[]}
 8 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[]}],"a":[],"b":[0],"expect_a":[],"expect_b":[[0,1]]}
 9 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[]}],"a":[1],"b":[],"expect_a":[[1,2]],"expect_b":[]}
10 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[]}],"a":[],"b":[1],"expect_a":[],"expect_b":[[1,2]]}
11 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[]}],"a":[2],"b":[],"expect_a":[[2,3]],"expect_b":[]}
12 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[]}],"a":[],"b":[2],"expect_a":[],"expect_b":[[2,3]]}
13 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[]}],"a":[],"b":[0,1],"expect_a":[],"expect_b":[[0,2]]}
14 | {"hist":[{"span":[0,1],"parents":[]},{"span":[1,2],"parents":[]},{"span":[2,3],"parents":[]}],"a":[0],"b":[1],"expect_a":[[0,1]],"expect_b":[[1,2]]}
15 | 


--------------------------------------------------------------------------------
/test_data/conformance.json.br:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/josephg/diamond-types/00f722d6ebdc9f3aec94d4406d9856b385939eae/test_data/conformance.json.br


--------------------------------------------------------------------------------
/vis/build.sh:
--------------------------------------------------------------------------------
1 | RUSTFLAGS="" npx snowpack build
2 | cp build/dist/diamond-wasm/diamond_wasm_bg.wasm build/dist/index_bg.wasm
3 | brotli build/dist/index_bg.wasm
4 | scp -r build/* tvbox:public/diamond-vis/
5 | 


--------------------------------------------------------------------------------
/vis/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "scripts": {
 3 |     "start": "snowpack dev",
 4 |     "build": "snowpack build"
 5 |   },
 6 |   "dependencies": {
 7 |     "svelte": "^3.49.0"
 8 |   },
 9 |   "devDependencies": {
10 |     "@emily-curry/snowpack-plugin-wasm-pack": "^1.1.4",
11 |     "@snowpack/plugin-dotenv": "^2.2.0",
12 |     "@snowpack/plugin-svelte": "^3.6.1",
13 |     "@snowpack/plugin-typescript": "^1.2.1",
14 |     "@tsconfig/svelte": "^2.0.1",
15 |     "@types/snowpack-env": "^2.3.3",
16 |     "snowpack": "^3.8.7",
17 |     "svelte-preprocess": "^4.7.2",
18 |     "typescript": "^4.3.4"
19 |   }
20 | }
21 | 


--------------------------------------------------------------------------------
/vis/public/index.html:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 |   
 4 |     
 5 |     
 6 |     
 7 |     Diamond types demo
 8 |     
36 | 
37 |   
38 |   
39 |     
40 |     
41 |     
51 |   
52 | 
53 | 


--------------------------------------------------------------------------------
/vis/snowpack.config.mjs:
--------------------------------------------------------------------------------
 1 | /** @type {import("snowpack").SnowpackUserConfig } */
 2 | export default {
 3 |   mount: {
 4 |     public: {url: '/', static: true},
 5 |     src: {url: '/dist'},
 6 |   },
 7 |   plugins: [
 8 |     '@snowpack/plugin-svelte',
 9 |     '@snowpack/plugin-dotenv',
10 |     [
11 |       '@snowpack/plugin-typescript',
12 |       {
13 |         /* Yarn PnP workaround: see https://www.npmjs.com/package/@snowpack/plugin-typescript */
14 |         ...(process.versions.pnp ? {tsc: 'yarn pnpify tsc'} : {}),
15 |       },
16 |     ],
17 |     [
18 |       '@emily-curry/snowpack-plugin-wasm-pack',
19 |       {
20 |         projectPath: '../crates/dt-wasm'
21 |       }
22 |     ]
23 |   ],
24 |   routes: [
25 |     /* Enable an SPA Fallback in development: */
26 |     // {"match": "routes", "src": ".*", "dest": "/index.html"},
27 |   ],
28 |   optimize: {
29 |     /* Example: Bundle your final build: */
30 |     "bundle": true,
31 |   },
32 |   packageOptions: {
33 | 
34 |   },
35 |   devOptions: {
36 | 
37 |   },
38 |   buildOptions: {
39 |     baseUrl: '.'
40 |   },
41 | };
42 | 


--------------------------------------------------------------------------------
/vis/src/App.svelte:
--------------------------------------------------------------------------------
 1 | 
24 | 
25 | 

26 |   
27 |   
28 |   
32 | 
33 |   
34 |   
35 |   
36 | 

37 | 
38 | 
89 | 
90 | 


--------------------------------------------------------------------------------
/vis/src/Editor.svelte:
--------------------------------------------------------------------------------
  1 | 
 75 | 
 76 | 

 77 |   

 78 |     {#each entries as entry (entry.order)}
 79 |       {#if !entry.isDeleted}
 80 |         {entry.content}
 81 |       {/if}
 82 |     {/each}
 83 |   

 84 |   
 90 | 

 91 | 
 92 | 
 93 | 


--------------------------------------------------------------------------------
/vis/src/index.ts:
--------------------------------------------------------------------------------
 1 | import App from './App.svelte';
 2 | import init from 'diamond-wasm'
 3 | 
 4 | 
 5 | // export default app;
 6 | 
 7 | ;(async () => {
 8 |   await init()
 9 | 
10 |   var app = new App({
11 |     target: document.body,
12 |   });
13 | 
14 |   // Hot Module Replacement (HMR) - Remove this snippet to remove HMR.
15 |   // Learn more: https://www.snowpack.dev/concepts/hot-module-replacement
16 |   if (import.meta.hot) {
17 |     import.meta.hot.accept();
18 |     import.meta.hot.dispose(() => {
19 |       app.$destroy();
20 |     });
21 |   }
22 | })()
23 | 
24 | 
25 | 


--------------------------------------------------------------------------------
/vis/src/utils.ts:
--------------------------------------------------------------------------------
  1 | import type { Doc } from "diamond-wasm";
  2 | 
  3 | export type DiffResult = {pos: number, del: number, ins: string}
  4 | 
  5 | export const calcDiff = (oldval: string, newval: string): DiffResult => {
  6 |   // Strings are immutable and have reference equality. I think this test is O(1), so its worth doing.
  7 |   if (oldval === newval) return {pos: 0, del: 0, ins: ''}
  8 | 
  9 |   let oldChars = [...oldval]
 10 |   let newChars = [...newval]
 11 | 
 12 |   var commonStart = 0;
 13 |   while (oldChars[commonStart] === newChars[commonStart]) {
 14 |     commonStart++;
 15 |   }
 16 | 
 17 |   var commonEnd = 0;
 18 |   while (oldChars[oldChars.length - 1 - commonEnd] === newChars[newChars.length - 1 - commonEnd] &&
 19 |       commonEnd + commonStart < oldChars.length && commonEnd + commonStart < newChars.length) {
 20 |     commonEnd++;
 21 |   }
 22 | 
 23 |   const del = (oldChars.length !== commonStart + commonEnd)
 24 |     ? oldChars.length - commonStart - commonEnd
 25 |     : 0
 26 |   const ins = (newChars.length !== commonStart + commonEnd)
 27 |     ? newChars.slice(commonStart, newChars.length - commonEnd).join('')
 28 |     : ''
 29 | 
 30 |   return {
 31 |     pos: commonStart, del, ins
 32 |   }
 33 | };
 34 | 
 35 | 
 36 | // let colors = ['red', 'green', 'yellow', 'cyan', 'hotpink']
 37 | // let colors = [
 38 | //   '#f94144',
 39 | //   '#f3722c',
 40 | //   '#f8961e',
 41 | //   '#f9844a',
 42 | //   '#f9c74f',
 43 | //   '#90be6d',
 44 | //   '#43aa8b',
 45 | //   '#4d908e',
 46 | //   '#577590',
 47 | //   '#277da1',
 48 | // ]
 49 | 
 50 | let colors = [
 51 |   "#9b5de5",
 52 |   "#f15bb5",
 53 |   "#fee440",
 54 |   "#00bbf9",
 55 |   "#00f5d4",
 56 | ]
 57 | 
 58 | let colorsFaint = colors.map(c => c + '33')
 59 | 
 60 | export type YjsEntry = {
 61 |   len: number,
 62 |   order: number,
 63 |   origin_left: number,
 64 |   origin_right: number
 65 | }
 66 | 
 67 | export type EnhancedEntries = YjsEntry & {
 68 |   order: number,
 69 |   content: string,
 70 |   len: number,
 71 |   color: string,
 72 |   isDeleted: boolean,
 73 |   originLeft: number | string,
 74 |   originRight: number | string,
 75 | }
 76 | 
 77 | export const getEnhancedEntries = (value: string, rle: boolean, doc: Doc) => {
 78 |   let str = [...value]
 79 |   let entries = doc.get_internal_list_entries() as YjsEntry[]
 80 | 
 81 |   if (!rle) {
 82 |     // Split each item
 83 |     entries = entries.flatMap(e => {
 84 |       const result = []
 85 |       let left = e.origin_left
 86 |       for (let i = 0; i < Math.abs(e.len); i++) {
 87 |         result.push({
 88 |           len: 1 * Math.sign(e.len),
 89 |           order: e.order + i,
 90 |           origin_left: left,
 91 |           origin_right: e.origin_right,
 92 |         })
 93 |         left = i
 94 |       }
 95 |       return result
 96 |     })
 97 |   }
 98 | 
 99 |   return entries
100 |     // .filter(e => e.len > 0)
101 |     .map((e, x) => ({
102 |       ...e,
103 |       len: Math.abs(e.len),
104 |       isDeleted: e.len < 0,
105 |       content: e.len > 0 ? str.splice(0, e.len).join('') : '',
106 |       // color: `radial-gradient(${colors[x % colors.length]}, transparent)`,
107 |       // color: `radial-gradient(ellipse at top, ${colors[x % colors.length]}, transparent), radial-gradient(ellipse at bottom, ${colorsFaint[x % colors.length]}, transparent)`,
108 |       // color: `radial-gradient(ellipse at top, red, transparent), radial-gradient(ellipse at bottom, blue, transparent)`,
109 |       // color: `radial-gradient(circle, ${colors[x % colors.length]} 90%, ${colorsFaint[x % colors.length]} 100%)`,
110 |       color: colors[x % colors.length],
111 |       originLeft: e.origin_left == 4294967295 ? 'ROOT' : e.origin_left,
112 |       originRight: e.origin_right == 4294967295 ? 'ROOT' : e.origin_right,
113 |     }))
114 | }


--------------------------------------------------------------------------------
/vis/svelte.config.js:
--------------------------------------------------------------------------------
1 | const autoPreprocess = require('svelte-preprocess');
2 | 
3 | module.exports = {
4 |   preprocess: autoPreprocess(),
5 | };
6 | 


--------------------------------------------------------------------------------
/vis/tsconfig.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "extends": "@tsconfig/svelte/tsconfig.json",
 3 |   "include": ["src", "types"],
 4 |   "compilerOptions": {
 5 |     "module": "esnext",
 6 |     "moduleResolution": "node",
 7 |     "baseUrl": "./",
 8 |     /* paths - import rewriting/resolving */
 9 |     "paths": {
10 |       // If you configured any Snowpack aliases, add them here.
11 |       // Add this line to get types for streaming imports (packageOptions.source="remote"):
12 |       // "*": [".snowpack/types/*"],
13 |       // More info: https://www.snowpack.dev/guides/streaming-imports
14 |       "diamond-wasm": ["../crates/dt-wasm/pkg"]
15 |     },
16 |     /* noEmit - Snowpack builds (emits) files, not tsc. */
17 |     "noEmit": true,
18 |     /* Additional Options */
19 |     "strict": true,
20 |     "skipLibCheck": true,
21 |     "lib": ["DOM", "esnext"],
22 |     "types": ["svelte", "snowpack-env"],
23 |     "forceConsistentCasingInFileNames": true,
24 |     "resolveJsonModule": true,
25 |     "useDefineForClassFields": true,
26 |     "allowSyntheticDefaultImports": true,
27 |     "importsNotUsedAsValues": "error"
28 |   }
29 | }
30 | 


--------------------------------------------------------------------------------
/vis/types/static.d.ts:
--------------------------------------------------------------------------------
 1 | /* Use this file to declare any custom file extensions for importing */
 2 | /* Use this folder to also add/extend a package d.ts file, if needed. */
 3 | 
 4 | /* CSS MODULES */
 5 | declare module '*.module.css' {
 6 |   const classes: {[key: string]: string};
 7 |   export default classes;
 8 | }
 9 | declare module '*.module.scss' {
10 |   const classes: {[key: string]: string};
11 |   export default classes;
12 | }
13 | declare module '*.module.sass' {
14 |   const classes: {[key: string]: string};
15 |   export default classes;
16 | }
17 | declare module '*.module.less' {
18 |   const classes: {[key: string]: string};
19 |   export default classes;
20 | }
21 | declare module '*.module.styl' {
22 |   const classes: {[key: string]: string};
23 |   export default classes;
24 | }
25 | 
26 | /* CSS */
27 | declare module '*.css';
28 | declare module '*.scss';
29 | declare module '*.sass';
30 | declare module '*.less';
31 | declare module '*.styl';
32 | 
33 | /* IMAGES */
34 | declare module '*.svg' {
35 |   const ref: string;
36 |   export default ref;
37 | }
38 | declare module '*.bmp' {
39 |   const ref: string;
40 |   export default ref;
41 | }
42 | declare module '*.gif' {
43 |   const ref: string;
44 |   export default ref;
45 | }
46 | declare module '*.jpg' {
47 |   const ref: string;
48 |   export default ref;
49 | }
50 | declare module '*.jpeg' {
51 |   const ref: string;
52 |   export default ref;
53 | }
54 | declare module '*.png' {
55 |   const ref: string;
56 |   export default ref;
57 | }
58 | 
59 | /* CUSTOM: ADD YOUR OWN HERE */
60 | 


--------------------------------------------------------------------------------
/wiki/client/index.css:
--------------------------------------------------------------------------------
 1 | * {
 2 |   box-sizing: border-box;
 3 | }
 4 | 
 5 | body {
 6 |   margin: 0;
 7 |   background-color: #9d35db;
 8 | }
 9 | 
10 | textarea {
11 |   position: fixed;
12 |   height: calc(100vh - 4em);
13 |   width: calc(100vw - 4em);
14 |   margin: 2em;
15 |   resize: none;
16 |   padding: 1.2em;
17 |   border: 5px solid #0f0d6b;
18 | 
19 |   font-family: monospace;
20 |   font-size: 16px;
21 | 
22 |   color: #87001d;
23 |   background-color: #f9fff9;
24 | }
25 | 
26 | textarea:focus {
27 |   outline: none;
28 | }
29 | 
30 | #statusContainer {
31 |   position: absolute;
32 |   left: 0;
33 |   right: 0;
34 |   top: 0;
35 |   font-family: sans-serif;
36 |   letter-spacing: 1px;
37 |   font-weight: 700;
38 |   display: flex;
39 |   justify-content: center;
40 |   /* transform: translateY(-50%); */
41 | }
42 | 
43 | #statusContainer > * {
44 |   background-color: rgb(218, 33, 95);
45 |   padding: 3px;
46 | 
47 |   transition: 0.2s linear;
48 |   transition-property: transform background-color;
49 | }
50 | 
51 | #statusContainer > *.connected {
52 |   background-color: rgb(48, 218, 33);
53 |   transform: translateY(-100%);
54 | }
55 | #statusContainer > *.connected::before {
56 |   content: 'Yay'
57 | }
58 | 
59 | #statusContainer > *.connecting {
60 |   background-color: rgb(218, 33, 95);
61 | }
62 | #statusContainer > *.connecting::before {
63 |   content: 'Connecting...'
64 | }
65 | 
66 | #statusContainer > *.waiting {
67 |   background-color: rgb(218, 33, 95);
68 | }
69 | #statusContainer > *.waiting::before {
70 |   content: 'Disconnected!! Waiting to reconnect...'
71 | }
72 | 
73 | #info {
74 |   position: absolute;
75 |   right: 0;
76 |   bottom: 0;
77 |   background-color: rgba(128, 128, 128, 0.5);
78 |   padding: 1em;
79 |   white-space: pre;
80 |   font-family: monospace;
81 |   font-size: 16px;
82 | }
83 | 
84 | #showInfo {
85 |   position: absolute;
86 |   right: 0;
87 |   bottom: 0;
88 |   opacity: 30%;
89 | }


--------------------------------------------------------------------------------
/wiki/client/index.tsx:
--------------------------------------------------------------------------------
 1 | /* @refresh reload */
 2 | import { render, Show } from 'solid-js/web';
 3 | import {createSignal, onMount} from 'solid-js'
 4 | import './index.css';
 5 | 
 6 | import { Status, subscribeDT } from './dt_doc';
 7 | // import * as foo from '@braid-protocol/client'
 8 | 
 9 | // console.log(foo)
10 | 
11 | // import App from './App';
12 | 
13 | // render(() => , document.getElementById('root') as HTMLElement);
14 | 
15 | // const docName = 'foo2'
16 | const docName = `wiki${location.pathname}`
17 | console.log(`Editing ${docName}`)
18 | const apiUrl = `/api/data/${docName}`
19 | 
20 | const connectionMsg: {[K in Status]: string} = {
21 |   connected: 'Connected',
22 |   connecting: 'Connecting...',
23 |   waiting: 'Disconnected!! Waiting to reconnect...'
24 | }
25 | 
26 | const Editor = (props: Record) => {
27 |   let textarea: HTMLTextAreaElement
28 | 
29 |   const [status, setStatus] = createSignal('connecting')
30 |   const [info, setInfo] = createSignal('Loading...')
31 |   const [showInfo, setShowInfo] = createSignal(false)
32 |   const toggleShow = () => setShowInfo(!showInfo())
33 |   // const toggleShow = () => {
34 |   //   console.log('xxx')
35 |   //   setShowInfo(true)
36 |   // }
37 | 
38 |   onMount(() => {
39 |     subscribeDT(apiUrl, textarea, {
40 |       setStatus(status) {
41 |         console.log('STATUS', status)
42 |         setStatus(status)
43 |       },
44 |       setInfo
45 |     })
46 |   })
47 | 
48 |   return (<>
49 |       

50 |         

51 |       

52 | 
53 |       
54 | 
55 |       
56 |         
{info()}

57 |       
58 |       
59 |     )
60 | }
61 | 
62 | 
63 | render(
64 |   () => ,
65 |   document.getElementById('root') as HTMLElement
66 | )
67 | 


--------------------------------------------------------------------------------
/wiki/client/mdstyle.css:
--------------------------------------------------------------------------------
  1 | 
  2 | * {
  3 |   box-sizing: border-box;
  4 | }
  5 | 
  6 | body {
  7 |   margin: 0 auto;
  8 |   /*background-color: #ffd1fa;*/
  9 |   max-width: 40em;
 10 | }
 11 | 
 12 | #content {
 13 | 
 14 | }
 15 | 
 16 | /*! Typebase.less v0.1.0 | MIT License */
 17 | /* Setup */
 18 | html {
 19 |   /* Change default typefaces here */
 20 |   font-family: serif;
 21 |   /*font-size: 137.5%;*/
 22 |   font-size: 115.5%;
 23 |   color: #1e1e1e;
 24 |   -webkit-font-smoothing: antialiased;
 25 | }
 26 | /* Copy & Lists */
 27 | p {
 28 |   line-height: 1.5rem;
 29 |   margin-top: 1.5rem;
 30 |   margin-bottom: 0;
 31 | }
 32 | ul,
 33 | ol {
 34 |   margin-top: 1.5rem;
 35 |   margin-bottom: 1.5rem;
 36 | }
 37 | ul li,
 38 | ol li {
 39 |   line-height: 1.5rem;
 40 | }
 41 | ul ul,
 42 | ol ul,
 43 | ul ol,
 44 | ol ol {
 45 |   margin-top: 0;
 46 |   margin-bottom: 0;
 47 | }
 48 | blockquote {
 49 |   line-height: 1.5rem;
 50 |   margin-top: 1.5rem;
 51 |   margin-bottom: 1.5rem;
 52 | }
 53 | /* Headings */
 54 | h1,
 55 | h2,
 56 | h3,
 57 | h4,
 58 | h5,
 59 | h6 {
 60 |   /* Change heading typefaces here */
 61 |   font-family: sans-serif;
 62 |   margin-top: 1.5rem;
 63 |   margin-bottom: 0;
 64 |   line-height: 1.5rem;
 65 | }
 66 | h1 {
 67 |   font-size: 4.242rem;
 68 |   line-height: 4.5rem;
 69 |   margin-top: 3rem;
 70 | }
 71 | h2 {
 72 |   font-size: 2.828rem;
 73 |   line-height: 3rem;
 74 |   margin-top: 3rem;
 75 | }
 76 | h3 {
 77 |   font-size: 1.414rem;
 78 | }
 79 | h4 {
 80 |   font-size: 0.707rem;
 81 | }
 82 | h5 {
 83 |   font-size: 0.4713333333333333rem;
 84 | }
 85 | h6 {
 86 |   font-size: 0.3535rem;
 87 | }
 88 | /* Tables */
 89 | table {
 90 |   margin-top: 1.5rem;
 91 |   border-spacing: 0px;
 92 |   border-collapse: collapse;
 93 | }
 94 | table td,
 95 | table th {
 96 |   padding: 0;
 97 |   line-height: 33px;
 98 | }
 99 | /* Code blocks */
100 | code {
101 |   vertical-align: bottom;
102 | }
103 | /* Leading paragraph text */
104 | .lead {
105 |   font-size: 1.414rem;
106 | }
107 | /* Hug the block above you */
108 | .hug {
109 |   margin-top: 0;
110 | }
111 | 


--------------------------------------------------------------------------------
/wiki/common/ratelimit.ts:
--------------------------------------------------------------------------------
 1 | export default function rateLimit(min_delay: number, fn: () => void) {
 2 |   let next_call = 0
 3 |   let timer: NodeJS.Timeout | null = null
 4 | 
 5 |   return () => {
 6 |     let now = Date.now()
 7 | 
 8 |     if (next_call <= now) {
 9 |       // Just call the function.
10 |       next_call = now + min_delay
11 | 
12 |       if (timer != null) {
13 |         clearTimeout(timer)
14 |         timer = null
15 |       }
16 |       fn()
17 |     } else {
18 |       // Queue the function call.
19 |       if (timer == null) {
20 |         timer = setTimeout(() => {
21 |           timer = null
22 |           next_call = Date.now() + min_delay
23 |           fn()
24 |         }, next_call - now)
25 |       } // Otherwise its already queued.
26 |     }
27 |   }
28 | }
29 | 
30 | // let f = rateLimit(300, () => { console.log('called') })
31 | 
32 | // f()
33 | // f()
34 | // console.log('\n\n')
35 | // f()
36 | 
37 | // setTimeout(() => {
38 | //   f()
39 | //   f()
40 | // }, 310)


--------------------------------------------------------------------------------
/wiki/common/utils.ts:
--------------------------------------------------------------------------------
  1 | 
  2 | export const vEq = (a: Uint32Array, b: Uint32Array): boolean => {
  3 |   if (a.length !== b.length) return false
  4 |   for (let i = 0; i < a.length; i++) {
  5 |     if (a[i] !== b[i]) return false
  6 |   }
  7 |   return true
  8 | }
  9 | 
 10 | export const assert = (expr: boolean) => {
 11 |   if (!expr) throw Error('Assertion failure')
 12 | }
 13 | 
 14 | export const wait = (time: number = 1000) => (
 15 |   new Promise((res) => setTimeout(res, time))
 16 | )
 17 | 
 18 | export type DiffResult = {pos: number, del: number, ins: string}
 19 | 
 20 | export const calcDiff = (oldval: string, newval: string): DiffResult => {
 21 |   // Strings are immutable and have reference equality. I think this test is O(1), so its worth doing.
 22 |   if (oldval === newval) return {pos: 0, del: 0, ins: ''}
 23 | 
 24 |   let oldChars = [...oldval]
 25 |   let newChars = [...newval]
 26 | 
 27 |   var commonStart = 0;
 28 |   while (oldChars[commonStart] === newChars[commonStart]) {
 29 |     commonStart++;
 30 |   }
 31 | 
 32 |   var commonEnd = 0;
 33 |   while (oldChars[oldChars.length - 1 - commonEnd] === newChars[newChars.length - 1 - commonEnd] &&
 34 |       commonEnd + commonStart < oldChars.length && commonEnd + commonStart < newChars.length) {
 35 |     commonEnd++;
 36 |   }
 37 | 
 38 |   const del = (oldChars.length !== commonStart + commonEnd)
 39 |     ? oldChars.length - commonStart - commonEnd
 40 |     : 0
 41 |   const ins = (newChars.length !== commonStart + commonEnd)
 42 |     ? newChars.slice(commonStart, newChars.length - commonEnd).join('')
 43 |     : ''
 44 | 
 45 |   return {
 46 |     pos: commonStart, del, ins
 47 |   }
 48 | }
 49 | 
 50 | export type DTOp = {kind: 'Ins' | 'Del', start: number, end: number, fwd?: boolean, content?: string}
 51 | 
 52 | export const transformPosition = (cursor: number, {kind, start, end}: DTOp, is_left: boolean = true): number => {
 53 |   let len = end - start
 54 | 
 55 |   return (cursor < start || (cursor === start && is_left)) ? cursor
 56 |     : (kind === 'Ins') ? cursor + len
 57 |     : cursor < end ? start
 58 |     : cursor - len
 59 | }
 60 | 
 61 | const test0 = (cursor: number, op: DTOp, is_left: boolean, expected: number) => {
 62 |   const actual = transformPosition(cursor, op, is_left)
 63 |   if (actual !== expected) {
 64 |     console.error('TEST FAILED', cursor, op, 'is_left', is_left)
 65 |     console.error('  Expected:', expected, '/ Actual:', actual)
 66 |   }
 67 | }
 68 | const test = (cursor: number, op: DTOp, expectLeft: number, expectRight: number = expectLeft) => {
 69 |   test0(cursor, op, true, expectLeft)
 70 |   test0(cursor, op, false, expectRight)
 71 | }
 72 | test(10, {kind: 'Ins', start: 5, end: 9}, 14)
 73 | test(10, {kind: 'Ins', start: 8, end: 12}, 14)
 74 | test(10, {kind: 'Ins', start: 10, end: 14}, 10, 14) // Different outcome!
 75 | test(10, {kind: 'Ins', start: 11, end: 15}, 10)
 76 | 
 77 | test(10, {kind: 'Del', start: 5, end: 9}, 6)
 78 | test(10, {kind: 'Del', start: 6, end: 10}, 6)
 79 | test(10, {kind: 'Del', start: 6, end: 100}, 6)
 80 | test(10, {kind: 'Del', start: 60, end: 100}, 10)
 81 | test(10, {kind: 'Del', start: 10, end: 100}, 10)
 82 | 
 83 | 
 84 | // // This operates in unicode offsets to make it consistent with the equivalent
 85 | // // methods in other languages / systems.
 86 | // const transformPosition = (cursor: number, op: TextOp) => {
 87 | //   let pos = 0
 88 | 
 89 | //   for (let i = 0; i < op.length && cursor > pos; i++) {
 90 | //     const c = op[i]
 91 | 
 92 | //     // I could actually use the op_iter stuff above - but I think its simpler
 93 | //     // like this.
 94 | //     switch (typeof c) {
 95 | //       case 'number': { // skip
 96 | //         pos += c
 97 | //         break
 98 | //       }
 99 | 
100 | //       case 'string': // insert
101 | //         // Its safe to use c.length here because they're both utf16 offsets.
102 | //         // Ignoring pos because the doc doesn't know about the insert yet.
103 | //         const offset = strPosToUni(c)
104 | //         pos += offset
105 | //         cursor += offset
106 | //         break
107 | 
108 | //       case 'object': // delete
109 | //         cursor -= Math.min(dlen(c.d), cursor - pos)
110 | //         break
111 | //     }
112 | //   }
113 | //   return cursor
114 | // }
115 | 
116 | // const transformSelection = (selection: number | [number, number], op: TextOp): number | [number, number] => (
117 | //   typeof selection === 'number'
118 | //     ? transformPosition(selection, op)
119 | //     : selection.map(s => transformPosition(s, op)) as [number, number]
120 | // )
121 | 


--------------------------------------------------------------------------------
/wiki/index.html:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 |   
 4 |     
 5 |     
 6 |     
 7 |     
 8 |     DT Wiki
 9 |   
10 |   
11 |     
12 |     

13 | 
14 |     
15 |   
16 | 
17 | 


--------------------------------------------------------------------------------
/wiki/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "type": "module",
 3 |   "scripts": {
 4 |     "start": "node dist/server/server.js",
 5 |     "dev": "vite",
 6 |     "build": "vite build && npx tsc && brotli -k dist-client/assets/*.wasm"
 7 |   },
 8 |   "dependencies": {
 9 |     "@braid-protocol/client": "^1.0.0",
10 |     "@braid-protocol/server": "^1.0.0",
11 |     "@types/node": "^20.1.1",
12 |     "@types/polka": "^0.5.4",
13 |     "body-parser": "^1.19.2",
14 |     "diamond-types-node": "^1.0.2",
15 |     "diamond-types-web": "^1.0.2",
16 |     "polka": "^0.5.2",
17 |     "sirv": "^2.0.2",
18 |     "solid-js": "^1.3.9",
19 |     "ts-node": "^10.6.0",
20 |     "typescript": "^5.0.4",
21 |     "unicount": "^1.2.0"
22 |   },
23 |   "devDependencies": {
24 |     "vite": "^4.3.5",
25 |     "vite-plugin-solid": "^2.2.5"
26 |   }
27 | }
28 | 


--------------------------------------------------------------------------------
/wiki/vite.config.ts:
--------------------------------------------------------------------------------
 1 | import { defineConfig } from 'vite';
 2 | import solidPlugin from 'vite-plugin-solid';
 3 | 
 4 | export default defineConfig({
 5 |   server: {
 6 |     // https: true,
 7 |     fs: {
 8 |       allow: ['../..'],
 9 |     },
10 |     proxy: {
11 |       '/api': {
12 |         target: 'http://localhost:4321',
13 |         ws: true,
14 |       }
15 |     }
16 |   },
17 |   plugins: [solidPlugin()],
18 |   build: {
19 |     minify: true,
20 |     sourcemap: true,
21 |     outDir: 'dist-client',
22 |     target: 'esnext',
23 |     // polyfillDynamicImport: false,
24 |   },
25 | });
26 | 


--------------------------------------------------------------------------------