├── .cargo └── config.toml ├── .editorconfig ├── .git-blame-ignore-revs ├── .gitattributes ├── .github ├── CODE_OF_CONDUCT.md ├── FUNDING.yml ├── actions │ ├── setup-node │ │ └── action.yml │ └── setup-rust │ │ └── action.yml ├── auto_assign.yml └── workflows │ ├── benchmark.yml │ ├── pr-auto-assign.yml │ ├── y-octo-node.yml │ └── y-octo.yml ├── .gitignore ├── .husky └── pre-commit ├── .nvmrc ├── .prettierignore ├── .taplo.toml ├── .yarn └── releases │ └── yarn-4.5.3.cjs ├── .yarnrc.yml ├── Cargo.toml ├── LICENSE ├── README.md ├── assets └── affine.svg ├── package.json ├── rust-toolchain ├── rustfmt.toml ├── tsconfig.json ├── y-octo-node ├── .gitignore ├── Cargo.toml ├── build.rs ├── index.d.ts ├── index.js ├── package.json ├── scripts │ └── run-test.mts ├── src │ ├── array.rs │ ├── doc.rs │ ├── lib.rs │ ├── map.rs │ ├── text.rs │ └── utils.rs ├── tests │ ├── array.spec.mts │ ├── doc.spec.mts │ ├── map.spec.mts │ └── text.spec.mts └── tsconfig.json ├── y-octo-utils ├── Cargo.toml ├── benches │ ├── apply_benchmarks.rs │ ├── array_ops_benchmarks.rs │ ├── codec_benchmarks.rs │ ├── map_ops_benchmarks.rs │ ├── text_ops_benchmarks.rs │ ├── update_benchmarks.rs │ └── utils │ │ ├── files.rs │ │ └── mod.rs ├── bin │ ├── bench_result_render.rs │ ├── doc_merger.rs │ └── memory_leak_test.rs ├── fuzz │ ├── .gitignore │ ├── Cargo.toml │ └── fuzz_targets │ │ ├── apply_update.rs │ │ ├── codec_doc_any.rs │ │ ├── codec_doc_any_struct.rs │ │ ├── decode_bytes.rs │ │ ├── i32_decode.rs │ │ ├── i32_encode.rs │ │ ├── ins_del_text.rs │ │ ├── sync_message.rs │ │ ├── u64_decode.rs │ │ └── u64_encode.rs ├── src │ ├── codec.rs │ ├── doc.rs │ ├── doc_operation │ │ ├── mod.rs │ │ ├── types.rs │ │ └── yrs_op │ │ │ ├── array.rs │ │ │ ├── map.rs │ │ │ ├── mod.rs │ │ │ ├── text.rs │ │ │ ├── xml_element.rs │ │ │ ├── xml_fragment.rs │ │ │ └── xml_text.rs │ ├── lib.rs │ └── message.rs └── yrs-is-unsafe │ ├── Cargo.toml │ ├── README.md │ ├── bin │ ├── global_lock.rs │ └── mem_usage.rs │ └── src │ └── main.rs ├── y-octo ├── Cargo.toml ├── LICENSE ├── README.md ├── benches │ ├── apply_benchmarks.rs │ ├── array_ops_benchmarks.rs │ ├── codec_benchmarks.rs │ ├── map_ops_benchmarks.rs │ ├── text_ops_benchmarks.rs │ ├── update_benchmarks.rs │ └── utils │ │ ├── files.rs │ │ └── mod.rs └── src │ ├── codec │ ├── buffer.rs │ ├── integer.rs │ ├── mod.rs │ └── string.rs │ ├── doc │ ├── awareness.rs │ ├── codec │ │ ├── any.rs │ │ ├── content.rs │ │ ├── delete_set.rs │ │ ├── id.rs │ │ ├── io │ │ │ ├── codec_v1.rs │ │ │ ├── mod.rs │ │ │ ├── reader.rs │ │ │ └── writer.rs │ │ ├── item.rs │ │ ├── item_flag.rs │ │ ├── mod.rs │ │ ├── refs.rs │ │ ├── update.rs │ │ └── utils │ │ │ ├── items.rs │ │ │ └── mod.rs │ ├── common │ │ ├── mod.rs │ │ ├── range.rs │ │ ├── somr.rs │ │ └── state.rs │ ├── document.rs │ ├── hasher.rs │ ├── history.rs │ ├── mod.rs │ ├── publisher.rs │ ├── store.rs │ ├── types │ │ ├── array.rs │ │ ├── list │ │ │ ├── iterator.rs │ │ │ ├── mod.rs │ │ │ └── search_marker.rs │ │ ├── map.rs │ │ ├── mod.rs │ │ ├── text.rs │ │ ├── value.rs │ │ └── xml.rs │ └── utils.rs │ ├── fixtures │ ├── basic.bin │ ├── database.bin │ ├── edge-case-left-right-same-node.bin │ ├── large.bin │ ├── local_docs.json │ └── with-subdoc.bin │ ├── lib.rs │ ├── protocol │ ├── awareness.rs │ ├── doc.rs │ ├── mod.rs │ ├── scanner.rs │ └── sync.rs │ └── sync.rs └── yarn.lock /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [env] 2 | CARGO_WORKSPACE_DIR = { value = "", relative = true } 3 | 4 | [unstable] 5 | gc = true 6 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.yml] 4 | indent_size = 2 5 | [*.rs] 6 | max_line_length = 120 7 | indent_size = 4 8 | [*.toml] 9 | indent_size = 2 10 | [*] 11 | charset = utf-8 12 | end_of_line = lf 13 | insert_final_newline = true 14 | trim_trailing_whitespace = true 15 | indent_style = space 16 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Run rust fmt 2 | b36d6959d89b4715742ca3938d9161655ca52f62 3 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | /.yarn/** linguist-vendored 2 | /.yarn/releases/* binary 3 | /.yarn/plugins/**/* binary 4 | /.pnp.* binary linguist-generated 5 | -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | - Using welcoming and inclusive language 12 | - Being respectful of differing viewpoints and experiences 13 | - Gracefully accepting constructive criticism 14 | - Focusing on what is best for the community 15 | - Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | - The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | - Trolling, insulting/derogatory comments, and personal or political attacks 21 | - Public or private harassment 22 | - Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | - Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies within all project spaces, and it also applies when an individual is representing the project or its community in public spaces. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project maintainer. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 1.4, available at 44 | 45 | For answers to common questions about this code of conduct, see 46 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [toeverything] 2 | -------------------------------------------------------------------------------- /.github/actions/setup-node/action.yml: -------------------------------------------------------------------------------- 1 | name: "Y-Octo Node.js Setup" 2 | description: "Node.js setup for CI, including cache configuration" 3 | inputs: 4 | extra-flags: 5 | description: "Extra flags to pass to the yarn install." 6 | required: false 7 | default: "--immutable --inline-builds" 8 | package-install: 9 | description: "Run the install step." 10 | required: false 11 | default: "true" 12 | hard-link-nm: 13 | description: "set nmMode to hardlinks-local in .yarnrc.yml" 14 | required: false 15 | default: "true" 16 | 17 | runs: 18 | using: "composite" 19 | steps: 20 | - name: Setup Node.js 21 | uses: actions/setup-node@v4 22 | with: 23 | node-version-file: ".nvmrc" 24 | cache: "yarn" 25 | 26 | - name: Set nmMode 27 | if: ${{ inputs.hard-link-nm == 'true' }} 28 | shell: bash 29 | run: yarn config set nmMode hardlinks-local 30 | 31 | - name: yarn install 32 | if: ${{ inputs.package-install == 'true' }} 33 | continue-on-error: true 34 | shell: bash 35 | run: yarn install ${{ inputs.extra-flags }} 36 | env: 37 | HUSKY: "0" 38 | 39 | - name: yarn install (try again) 40 | if: ${{ steps.install.outcome == 'failure' }} 41 | shell: bash 42 | run: yarn install ${{ inputs.extra-flags }} 43 | env: 44 | HUSKY: "0" 45 | -------------------------------------------------------------------------------- /.github/actions/setup-rust/action.yml: -------------------------------------------------------------------------------- 1 | name: "Rust setup" 2 | description: "Rust setup, including cache configuration" 3 | inputs: 4 | components: 5 | description: "Cargo components" 6 | required: false 7 | targets: 8 | description: "Cargo target" 9 | required: false 10 | toolchain: 11 | description: "Rustup toolchain" 12 | required: false 13 | default: "stable" 14 | 15 | runs: 16 | using: "composite" 17 | steps: 18 | - name: Setup Rust 19 | uses: dtolnay/rust-toolchain@stable 20 | with: 21 | toolchain: ${{ inputs.toolchain }} 22 | targets: ${{ inputs.targets }} 23 | components: ${{ inputs.components }} 24 | - name: Add Targets 25 | if: ${{ inputs.targets }} 26 | run: rustup target add ${{ inputs.targets }} 27 | shell: bash 28 | - uses: Swatinem/rust-cache@v2 29 | -------------------------------------------------------------------------------- /.github/auto_assign.yml: -------------------------------------------------------------------------------- 1 | # This is used for tracking in GitHub project. 2 | # See https://github.com/marketplace/actions/auto-assign-action 3 | 4 | # Set to true to add reviewers to pull requests 5 | addReviewers: false 6 | 7 | # Set to true to add assignees to pull requests 8 | addAssignees: author 9 | 10 | runOnDraft: true 11 | -------------------------------------------------------------------------------- /.github/workflows/benchmark.yml: -------------------------------------------------------------------------------- 1 | name: Benchmark 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | types: [opened, synchronize] 7 | paths-ignore: 8 | - "**/*.md" 9 | 10 | concurrency: 11 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | benchmark: 16 | strategy: 17 | matrix: 18 | os: [ubuntu-latest, windows-latest] # `macos-latest` is too unstable to be useful for benchmark, the variance is always huge. 19 | name: Run benchmark on ${{ matrix.os }} 20 | runs-on: ${{ matrix.os }} 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v4 24 | with: 25 | ref: main 26 | 27 | - name: Setup Rust 28 | uses: ./.github/actions/setup-rust 29 | with: 30 | toolchain: stable 31 | 32 | - name: Run Bench on main Branch 33 | run: | 34 | cargo bench --bench codec_benchmarks --features bench -- --save-baseline main 35 | cargo bench --bench array_ops_benchmarks --features bench -- --save-baseline main 36 | cargo bench --bench map_ops_benchmarks --features bench -- --save-baseline main 37 | cargo bench --bench text_ops_benchmarks --features bench -- --save-baseline main 38 | cargo bench --bench update_benchmarks --features bench -- --save-baseline main 39 | 40 | - name: Checkout main branch 41 | uses: actions/checkout@v4 42 | with: 43 | clean: false 44 | ref: ${{ github.event.pull_request.head.sha }} 45 | 46 | - name: Run Bench on PR Branch 47 | run: | 48 | cargo bench --bench codec_benchmarks --features bench -- --save-baseline pr 49 | cargo bench --bench array_ops_benchmarks --features bench -- --save-baseline pr 50 | cargo bench --bench map_ops_benchmarks --features bench -- --save-baseline pr 51 | cargo bench --bench text_ops_benchmarks --features bench -- --save-baseline pr 52 | cargo bench --bench update_benchmarks --features bench -- --save-baseline pr 53 | 54 | - name: Upload benchmark results 55 | uses: actions/upload-artifact@v4 56 | with: 57 | name: benchmark-results-${{ matrix.os }} 58 | path: ./target/criterion 59 | 60 | benchmark-compare: 61 | runs-on: ubuntu-latest 62 | name: Compare Benchmarks 63 | needs: 64 | - benchmark 65 | 66 | steps: 67 | - name: Checkout 68 | uses: actions/checkout@v4 69 | with: 70 | fetch-depth: 0 71 | 72 | - name: Setup Rust 73 | uses: ./.github/actions/setup-rust 74 | with: 75 | toolchain: stable 76 | 77 | - name: Install critcmp 78 | uses: taiki-e/install-action@v2 79 | with: 80 | tool: critcmp 81 | 82 | - name: Linux | Download PR benchmark results 83 | uses: actions/download-artifact@v4 84 | with: 85 | name: benchmark-results-ubuntu-latest 86 | path: ./target/criterion 87 | 88 | - name: Linux | Compare benchmark results 89 | shell: bash 90 | run: | 91 | critcmp main pr | cargo run -p y-octo-utils --bin bench_result_render --features bench -- Linux >> summary.md 92 | echo "" >> summary.md 93 | 94 | - name: Linux | Cleanup benchmark results 95 | run: rm -rf ./target/criterion 96 | 97 | - name: Windows | Download PR benchmark results 98 | uses: actions/download-artifact@v4 99 | with: 100 | name: benchmark-results-windows-latest 101 | path: ./target/criterion 102 | 103 | - name: Windows | Compare benchmark results 104 | shell: bash 105 | run: | 106 | critcmp main pr | cargo run -p y-octo-utils --bin bench_result_render --features bench -- Windows >> summary.md 107 | cat summary.md > $GITHUB_STEP_SUMMARY 108 | 109 | - name: Find Comment 110 | # Check if the event is not triggered by a fork 111 | if: github.event.pull_request.head.repo.full_name == github.repository 112 | uses: peter-evans/find-comment@v3 113 | id: fc 114 | with: 115 | issue-number: ${{ github.event.pull_request.number }} 116 | comment-author: "github-actions[bot]" 117 | body-includes: Benchmark Results 118 | 119 | - name: Create or update comment 120 | # Check if the event is not triggered by a fork 121 | if: github.event.pull_request.head.repo.full_name == github.repository 122 | uses: peter-evans/create-or-update-comment@v4 123 | with: 124 | issue-number: ${{ github.event.pull_request.number }} 125 | edit-mode: replace 126 | comment-id: ${{ steps.fc.outputs.comment-id }} 127 | body-file: summary.md 128 | -------------------------------------------------------------------------------- /.github/workflows/pr-auto-assign.yml: -------------------------------------------------------------------------------- 1 | name: Pull request auto assign 2 | 3 | # on: pull_request 4 | on: 5 | pull_request_target: 6 | types: [opened, ready_for_review] 7 | 8 | jobs: 9 | add-reviews: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: kentaro-m/auto-assign-action@v2.0.0 13 | -------------------------------------------------------------------------------- /.github/workflows/y-octo-node.yml: -------------------------------------------------------------------------------- 1 | name: Y-Octo Node Binding Build & Test 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [main] 7 | pull_request: 8 | branches: [main] 9 | 10 | env: 11 | DEBUG: napi:* 12 | COVERAGE: true 13 | MACOSX_DEPLOYMENT_TARGET: "10.13" 14 | 15 | jobs: 16 | build-node: 17 | name: Build Node Binding 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | settings: 22 | - target: x86_64-apple-darwin 23 | host: macos-latest 24 | - target: aarch64-apple-darwin 25 | host: macos-latest 26 | - target: x86_64-pc-windows-msvc 27 | host: windows-latest 28 | # - target: aarch64-pc-windows-msvc 29 | # host: windows-latest 30 | - target: x86_64-unknown-linux-gnu 31 | host: ubuntu-latest 32 | # - target: aarch64-unknown-linux-gnu 33 | # host: ubuntu-latest 34 | # - target: x86_64-unknown-linux-musl 35 | # host: ubuntu-latest 36 | # - target: aarch64-unknown-linux-musl 37 | # host: ubuntu-latest 38 | runs-on: ${{ matrix.settings.host }} 39 | env: 40 | RUSTFLAGS: "-C debuginfo=1" 41 | steps: 42 | - uses: actions/checkout@v4 43 | - name: Setup Node.js 44 | uses: ./.github/actions/setup-node 45 | - name: Setup Rust 46 | uses: ./.github/actions/setup-rust 47 | with: 48 | targets: ${{ matrix.settings.target }} 49 | - name: Build node binding 50 | run: yarn build:node --target ${{ matrix.settings.target }} 51 | - name: Upload y-octo.node 52 | uses: actions/upload-artifact@v4 53 | with: 54 | name: y-octo.${{ matrix.settings.target }}.node 55 | path: ./y-octo-node/*.node 56 | if-no-files-found: error 57 | 58 | test-node: 59 | name: Test & Collect Coverage 60 | runs-on: ubuntu-latest 61 | continue-on-error: true 62 | env: 63 | RUSTFLAGS: -D warnings 64 | CARGO_TERM_COLOR: always 65 | steps: 66 | - uses: actions/checkout@v4 67 | 68 | - name: Setup Rust 69 | uses: ./.github/actions/setup-rust 70 | with: 71 | components: llvm-tools-preview 72 | - name: Install latest nextest release 73 | uses: taiki-e/install-action@nextest 74 | - name: Install cargo-llvm-cov 75 | uses: taiki-e/install-action@cargo-llvm-cov 76 | 77 | - name: Collect coverage data 78 | run: cargo llvm-cov nextest --lcov --output-path lcov.info 79 | - name: Upload coverage data to codecov 80 | uses: codecov/codecov-action@v5 81 | with: 82 | name: tests 83 | files: lcov.info 84 | 85 | node-binding-test: 86 | name: Node Binding Test 87 | strategy: 88 | fail-fast: false 89 | matrix: 90 | settings: 91 | - target: aarch64-apple-darwin 92 | host: macos-latest 93 | - target: x86_64-unknown-linux-gnu 94 | host: ubuntu-latest 95 | - target: x86_64-pc-windows-msvc 96 | host: windows-latest 97 | runs-on: ${{ matrix.settings.host }} 98 | needs: build-node 99 | steps: 100 | - uses: actions/checkout@v4 101 | - name: Setup Node.js 102 | uses: ./.github/actions/setup-node 103 | - name: Download y-octo.${{ matrix.settings.target }}.node 104 | uses: actions/download-artifact@v4 105 | with: 106 | name: y-octo.${{ matrix.settings.target }}.node 107 | path: ./y-octo-node 108 | - name: Run node binding tests 109 | run: ls -lah & ls -lah tests 110 | working-directory: y-octo-node 111 | shell: bash 112 | - name: Run node binding tests 113 | run: yarn test:node:coverage 114 | working-directory: y-octo-node 115 | shell: bash 116 | - name: Upload server test coverage results 117 | uses: codecov/codecov-action@v5 118 | with: 119 | token: ${{ secrets.CODECOV_TOKEN }} 120 | files: ./y-octo-node/.coverage/lcov.info 121 | flags: node-binding-test 122 | name: y-octo.${{ matrix.settings.target }}.node 123 | fail_ci_if_error: false 124 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | .turbo 3 | .env 4 | .DS_Store 5 | *.db* 6 | /data/* 7 | **/.next 8 | **/out 9 | **/dist 10 | **/node_modules 11 | **/stats.html 12 | .npmrc 13 | **/.idea 14 | **/.swiftpm 15 | **/*.xcodeproj 16 | **/*.xcworkspace 17 | Cargo.lock 18 | .pnp.* 19 | .yarn/* 20 | !.yarn/patches 21 | !.yarn/plugins 22 | !.yarn/releases 23 | !.yarn/sdks 24 | !.yarn/versions 25 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | yarn lint-staged 2 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 18 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | .yarn 2 | node_modules 3 | target 4 | y-octo/README.md 5 | .cargo 6 | vendor 7 | index.d.ts 8 | index.js 9 | .coverage -------------------------------------------------------------------------------- /.taplo.toml: -------------------------------------------------------------------------------- 1 | exclude = ["node_modules/**/*.toml", "target/**/*.toml"] 2 | 3 | # https://taplo.tamasfe.dev/configuration/formatter-options.html 4 | [formatting] 5 | align_entries = true 6 | indent_tables = true 7 | reorder_keys = true 8 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | 3 | yarnPath: .yarn/releases/yarn-4.5.3.cjs 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | 3 | members = [ 4 | "y-octo", 5 | "y-octo-node", 6 | "y-octo-utils", 7 | "y-octo-utils/yrs-is-unsafe", 8 | ] 9 | resolver = "2" 10 | 11 | [workspace.dependencies] 12 | y-octo = { path = "./y-octo" } 13 | y-octo-utils = { path = "./y-octo-utils" } 14 | y-sync = { git = "https://github.com/toeverything/y-sync", rev = "5626851" } 15 | yrs = { git = "https://github.com/toeverything/y-crdt", rev = "a700f09" } 16 | 17 | [profile.release] 18 | codegen-units = 1 19 | lto = true 20 | opt-level = 3 21 | 22 | [profile.fast-release] 23 | codegen-units = 16 24 | inherits = "release" 25 | lto = false 26 | 27 | [profile.profiling] 28 | debug = true 29 | inherits = "fast-release" 30 | 31 | # [profile.release.package.y-octo-fuzz] 32 | # debug = 1 33 | 34 | [patch.crates-io] 35 | lib0 = { git = "https://github.com/toeverything/y-crdt", rev = "a700f09" } 36 | y-sync = { git = "https://github.com/toeverything/y-sync", rev = "5626851" } 37 | yrs = { git = "https://github.com/toeverything/y-crdt", rev = "a700f09" } 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2023 4 | - DarkSky . 5 | - forehalo . 6 | - x1a0t <405028157@qq.com>. 7 | - Toeverything Pte. Ltd. 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Y-Octo 2 | 3 | [![test](https://github.com/toeverything/y-octo/actions/workflows/y-octo.yml/badge.svg)](https://github.com/toeverything/y-octo/actions/workflows/y-octo.yml) 4 | [![docs]](https://docs.rs/y-octo/latest/y_octo) 5 | [![crates]](https://crates.io/crates/y-octo) 6 | [![codecov]](https://codecov.io/gh/toeverything/y-octo) 7 | 8 | Y-Octo is a high-performance CRDT implementation compatible with [yjs]. 9 | 10 | ## Introduction 11 | 12 | Y-Octo is a tiny, ultra-fast CRDT collaboration library built for all major platforms. Developers can use Y-Octo as the [Single source of truth](https://en.wikipedia.org/wiki/Single_source_of_truth) for their application state, naturally turning the application into a [local-first](https://www.inkandswitch.com/local-first/) collaborative app. 13 | 14 | Y-Octo also has interoperability and binary compatibility with [yjs]. Developers can use [yjs] to develop local-first web applications and collaborate with Y-Octo in native apps alongside web apps. 15 | 16 | ## Who are using 17 | 18 | 19 | 20 | [AFFiNE](https://affine.pro) is using y-octo in production. There are [Electron](https://affine.pro/download) app and [Node.js server](https://github.com/toeverything/AFFiNE/tree/canary/packages/backend/native) using y-octo in production. 21 | 22 | 23 | 24 | [Mysc](https://www.mysc.app/) is using y-octo in the Rust server, and the iOS/Android client via the Swift/Kotlin bindings (Official bindings coming soon). 25 | 26 | ## Features 27 | 28 | - ✅ Collaborative Text 29 | - ✅ Read and write styled Unicode compatible data. 30 | - 🚧 Add, modify and delete text styles. 31 | - 🚧 Embedded JS data types and collaborative types. 32 | - ✅ Collaborative types of thread-safe. 33 | - Collaborative Array 34 | - ✅ Add, modify, and delete basic JS data types. 35 | - ✅ Recursively add, modify, and delete collaborative types. 36 | - ✅ Collaborative types of thread-safe. 37 | - 🚧 Recursive event subscription 38 | - Collaborative Map 39 | - ✅ Add, modify, and delete basic JS data types. 40 | - ✅ Recursively add, modify, and delete collaborative types. 41 | - ✅ Collaborative types of thread-safe. 42 | - 🚧 Recursive event subscription 43 | - 🚧 Collaborative Xml (Fragment / Element) 44 | - ✅ Collaborative Doc Container 45 | - ✅ YATA CRDT state apply/diff compatible with [yjs] 46 | - ✅ State sync of thread-safe. 47 | - ✅ Store all collaborative types and JS data types 48 | - ✅ Update event subscription. 49 | - 🚧 Sub Document. 50 | - ✅ Yjs binary encoding 51 | - ✅ Awareness encoding. 52 | - ✅ Primitive type encoding. 53 | - ✅ Sync Protocol encoding. 54 | - ✅ Yjs update v1 encoding. 55 | - 🚧 Yjs update v2 encoding. 56 | 57 | ## Testing & Linting 58 | 59 | Put everything to the test! We've established various test suites, but we're continually striving to enhance our coverage： 60 | 61 | - Rust Tests 62 | - Unit tests 63 | - [Loom](https://docs.rs/loom/latest/loom/) multi-threading tests 64 | - [Miri](https://github.com/rust-lang/miri) undefined behavior tests 65 | - [Address Sanitizer](https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html) memory error detections 66 | - [Fuzzing](https://github.com/rust-fuzz/cargo-fuzz) fuzzing tests 67 | - Node Tests 68 | - Smoke Tests 69 | - Eslint, Clippy 70 | 71 | ## Related projects 72 | 73 | - [OctoBase]: The open-source embedded database based on Y-Octo. 74 | - [yjs]: Shared data types for building collaborative software in web. 75 | 76 | ## Maintainers 77 | 78 | - [DarkSky](https://github.com/darkskygit) 79 | - [liuyi](https://github.com/forehalo) 80 | - [LongYinan](https://github.com/Brooooooklyn) 81 | 82 | ## Why not [yrs](https://github.com/y-crdt/y-crdt/) 83 | 84 | See [Why we're not using yrs](./y-octo-utils/yrs-is-unsafe/README.md) 85 | 86 | ## License 87 | 88 | Y-Octo are [MIT licensed]. 89 | 90 | [codecov]: https://codecov.io/gh/toeverything/y-octo/graph/badge.svg?token=9AQY5Q1BYH 91 | [crates]: https://img.shields.io/crates/v/y-octo.svg 92 | [docs]: https://img.shields.io/docsrs/y-octo.svg 93 | [test]: https://github.com/toeverything/y-octo/actions/workflows/y-octo.yml/badge.svg 94 | [yjs]: https://github.com/yjs/yjs 95 | [Address Sanitizer]: https://github.com/toeverything/y-octo/actions/workflows/y-octo-asan.yml/badge.svg 96 | [Memory Leak Detect]: https://github.com/toeverything/y-octo/actions/workflows/y-octo-memory-test.yml/badge.svg 97 | [OctoBase]: https://github.com/toeverything/octobase 98 | [BlockSuite]: https://github.com/toeverything/blocksuite 99 | [AFFiNE]: https://github.com/toeverything/affine 100 | [MIT licensed]: ./LICENSE 101 | -------------------------------------------------------------------------------- /assets/affine.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@y-octo/cli", 3 | "version": "0.0.0", 4 | "packageManager": "yarn@4.5.3", 5 | "license": "MIT", 6 | "workspaces": [ 7 | ".", 8 | "y-octo-node" 9 | ], 10 | "engines": { 11 | "node": ">=18.16.1" 12 | }, 13 | "scripts": { 14 | "build:node": "yarn workspace @y-octo/node build", 15 | "test:node": "yarn workspace @y-octo/node test", 16 | "test:node:coverage": "yarn workspace @y-octo/node test:coverage", 17 | "format": "run-p format:toml format:prettier format:rs", 18 | "format:toml": "taplo format", 19 | "format:prettier": "prettier --write .", 20 | "format:rs": "cargo +nightly fmt --all" 21 | }, 22 | "devDependencies": { 23 | "@taplo/cli": "^0.7.0", 24 | "husky": "^9.1.7", 25 | "lint-staged": "^15.2.11", 26 | "npm-run-all": "^4.1.5", 27 | "prettier": "^3.4.2" 28 | }, 29 | "lint-staged": { 30 | "*.@(js|ts|tsx|yml|yaml|json|md)": [ 31 | "prettier --write" 32 | ], 33 | "*.toml": [ 34 | "taplo format" 35 | ], 36 | "*.rs": [ 37 | "cargo +nightly fmt --" 38 | ] 39 | }, 40 | "resolutions": { 41 | "array-buffer-byte-length": "npm:@nolyfill/array-buffer-byte-length@latest", 42 | "arraybuffer.prototype.slice": "npm:@nolyfill/arraybuffer.prototype.slice@latest", 43 | "available-typed-arrays": "npm:@nolyfill/available-typed-arrays@latest", 44 | "define-properties": "npm:@nolyfill/define-properties@latest", 45 | "es-set-tostringtag": "npm:@nolyfill/es-set-tostringtag@latest", 46 | "function-bind": "npm:@nolyfill/function-bind@latest", 47 | "function.prototype.name": "npm:@nolyfill/function.prototype.name@latest", 48 | "get-symbol-description": "npm:@nolyfill/get-symbol-description@latest", 49 | "globalthis": "npm:@nolyfill/globalthis@latest", 50 | "gopd": "npm:@nolyfill/gopd@latest", 51 | "has": "npm:@nolyfill/has@latest", 52 | "has-property-descriptors": "npm:@nolyfill/has-property-descriptors@latest", 53 | "has-proto": "npm:@nolyfill/has-proto@latest", 54 | "has-symbols": "npm:@nolyfill/has-symbols@latest", 55 | "has-tostringtag": "npm:@nolyfill/has-tostringtag@latest", 56 | "internal-slot": "npm:@nolyfill/internal-slot@latest", 57 | "is-array-buffer": "npm:@nolyfill/is-array-buffer@latest", 58 | "is-date-object": "npm:@nolyfill/is-date-object@latest", 59 | "is-regex": "npm:@nolyfill/is-regex@latest", 60 | "is-shared-array-buffer": "npm:@nolyfill/is-shared-array-buffer@latest", 61 | "is-string": "npm:@nolyfill/is-string@latest", 62 | "is-symbol": "npm:@nolyfill/is-symbol@latest", 63 | "is-weakref": "npm:@nolyfill/is-weakref@latest", 64 | "object-keys": "npm:@nolyfill/object-keys@latest", 65 | "object.assign": "npm:@nolyfill/object.assign@latest", 66 | "regexp.prototype.flags": "npm:@nolyfill/regexp.prototype.flags@latest", 67 | "safe-array-concat": "npm:@nolyfill/safe-array-concat@latest", 68 | "safe-regex-test": "npm:@nolyfill/safe-regex-test@latest", 69 | "side-channel": "npm:@nolyfill/side-channel@latest", 70 | "string.prototype.padend": "npm:@nolyfill/string.prototype.padend@latest", 71 | "string.prototype.trim": "npm:@nolyfill/string.prototype.trim@latest", 72 | "string.prototype.trimend": "npm:@nolyfill/string.prototype.trimend@latest", 73 | "string.prototype.trimstart": "npm:@nolyfill/string.prototype.trimstart@latest", 74 | "typed-array-buffer": "npm:@nolyfill/typed-array-buffer@latest", 75 | "typed-array-byte-length": "npm:@nolyfill/typed-array-byte-length@latest", 76 | "typed-array-byte-offset": "npm:@nolyfill/typed-array-byte-offset@latest", 77 | "typed-array-length": "npm:@nolyfill/typed-array-length@latest", 78 | "unbox-primitive": "npm:@nolyfill/unbox-primitive@latest", 79 | "which-boxed-primitive": "npm:@nolyfill/which-boxed-primitive@latest", 80 | "which-typed-array": "npm:@nolyfill/which-typed-array@latest" 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /rust-toolchain: -------------------------------------------------------------------------------- 1 | 1.83.0 2 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # Keep in sync with .editorconfig 2 | format_strings = true 3 | group_imports = "StdExternalCrate" 4 | hard_tabs = false 5 | imports_granularity = "Crate" 6 | max_width = 120 7 | tab_spaces = 4 8 | wrap_comments = true 9 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "verbatimModuleSyntax": true, 4 | // Classification follows https://www.typescriptlang.org/tsconfig 5 | 6 | // Type Checking 7 | "strict": true, 8 | // exactOptionalPropertyTypes: false, 9 | "noFallthroughCasesInSwitch": true, 10 | "noImplicitAny": true, 11 | "noImplicitOverride": true, 12 | "noImplicitReturns": true, 13 | "noImplicitThis": true, 14 | // "noUnusedLocals": true, 15 | // "noUnusedParameters": true, 16 | // noPropertyAccessFromIndexSignature: false, 17 | // noUncheckedIndexedAccess: false, 18 | "useUnknownInCatchVariables": true, 19 | 20 | // Modules 21 | "module": "ESNext", 22 | "moduleResolution": "bundler", 23 | "resolveJsonModule": true, 24 | // Emit 25 | "declaration": true, 26 | "declarationMap": true, 27 | "sourceMap": true, 28 | // skip type emit for @internal types 29 | // "stripInternal": true, 30 | 31 | // JavaScript Support 32 | "allowJs": false, 33 | "checkJs": false, 34 | 35 | // Interop Constraints 36 | "forceConsistentCasingInFileNames": true, 37 | "allowSyntheticDefaultImports": true, 38 | "isolatedModules": true, 39 | 40 | // Language and Environment 41 | "jsx": "preserve", 42 | "jsxImportSource": "@emotion/react", 43 | "lib": ["ESNext", "DOM"], 44 | "target": "ES2022", 45 | "useDefineForClassFields": false, 46 | "experimentalDecorators": true, 47 | "emitDecoratorMetadata": true, 48 | 49 | // Projects 50 | "composite": true, 51 | "incremental": true, 52 | 53 | // Completeness 54 | "skipLibCheck": true, // skip all type checks for .d.ts files 55 | "paths": { 56 | "@y-octo/node/*": ["./y-octo-node/src/*"] 57 | } 58 | }, 59 | "include": [], 60 | "references": [ 61 | { 62 | "path": "./y-octo-node" 63 | } 64 | ], 65 | "files": [], 66 | "exclude": ["node_modules", "target", "lib", "test-results"], 67 | "ts-node": { 68 | "compilerOptions": { 69 | "module": "ESNext", 70 | "moduleResolution": "Node" 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /y-octo-node/.gitignore: -------------------------------------------------------------------------------- 1 | *.node 2 | .coverage -------------------------------------------------------------------------------- /y-octo-node/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = ["DarkSky "] 3 | edition = "2021" 4 | license = "MIT" 5 | name = "y-octo-node" 6 | repository = "https://github.com/toeverything/y-octo" 7 | version = "0.0.1" 8 | 9 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 10 | [lib] 11 | crate-type = ["cdylib"] 12 | 13 | [dependencies] 14 | anyhow = "1" 15 | napi = { version = "2", features = ["anyhow", "napi4"] } 16 | napi-derive = "2" 17 | y-octo = { workspace = true, features = ["large_refs"] } 18 | 19 | [build-dependencies] 20 | napi-build = "2" 21 | -------------------------------------------------------------------------------- /y-octo-node/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | napi_build::setup(); 3 | } 4 | -------------------------------------------------------------------------------- /y-octo-node/index.d.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | 4 | /* auto-generated by NAPI-RS */ 5 | 6 | export declare class YArray { 7 | constructor() 8 | get length(): number 9 | get isEmpty(): boolean 10 | get(index: number): T 11 | insert(index: number, value: YArray | YMap | YText | boolean | number | string | Record | null | undefined): void 12 | remove(index: number, len: number): void 13 | toJson(): JsArray 14 | } 15 | export declare class Doc { 16 | constructor(clientId?: number | undefined | null) 17 | get clientId(): number 18 | get guid(): string 19 | get keys(): Array 20 | getOrCreateArray(key: string): YArray 21 | getOrCreateText(key: string): YText 22 | getOrCreateMap(key: string): YMap 23 | createArray(): YArray 24 | createText(): YText 25 | createMap(): YMap 26 | applyUpdate(update: Buffer): void 27 | encodeStateAsUpdateV1(state?: Buffer | undefined | null): Buffer 28 | gc(): void 29 | onUpdate(callback: (result: Uint8Array) => void): void 30 | } 31 | export declare class YMap { 32 | constructor() 33 | get length(): number 34 | get isEmpty(): boolean 35 | get(key: string): T 36 | set(key: string, value: YArray | YMap | YText | boolean | number | string | Record | null | undefined): void 37 | remove(key: string): void 38 | toJson(): object 39 | } 40 | export declare class YText { 41 | constructor() 42 | get len(): number 43 | get isEmpty(): boolean 44 | insert(index: number, str: string): void 45 | remove(index: number, len: number): void 46 | get length(): number 47 | toString(): string 48 | } 49 | -------------------------------------------------------------------------------- /y-octo-node/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@y-octo/node", 3 | "private": true, 4 | "main": "index.js", 5 | "types": "index.d.ts", 6 | "napi": { 7 | "name": "y-octo", 8 | "triples": { 9 | "additional": [ 10 | "aarch64-apple-darwin", 11 | "aarch64-pc-windows-msvc", 12 | "aarch64-unknown-linux-gnu", 13 | "x86_64-unknown-linux-musl", 14 | "aarch64-unknown-linux-musl" 15 | ] 16 | }, 17 | "ts": { 18 | "constEnum": false 19 | } 20 | }, 21 | "license": "MIT", 22 | "devDependencies": { 23 | "@napi-rs/cli": "^2.18.4", 24 | "@types/node": "^22.10.2", 25 | "@types/prompts": "^2.4.9", 26 | "c8": "^10.1.3", 27 | "prompts": "^2.4.2", 28 | "ts-node": "^10.9.2", 29 | "typescript": "^5.7.2", 30 | "yjs": "^13.6.21" 31 | }, 32 | "engines": { 33 | "node": ">= 10" 34 | }, 35 | "scripts": { 36 | "artifacts": "napi artifacts", 37 | "build": "napi build --platform --release --no-const-enum", 38 | "build:debug": "napi build --platform --no-const-enum", 39 | "universal": "napi universal", 40 | "test": "NODE_NO_WARNINGS=1 yarn exec ts-node-esm ./scripts/run-test.mts all", 41 | "test:watch": "yarn exec ts-node-esm ./scripts/run-test.ts all --watch", 42 | "test:coverage": "NODE_OPTIONS=\"--loader ts-node/esm\" c8 node ./scripts/run-test.mts all", 43 | "version": "napi version" 44 | }, 45 | "version": "0.0.1", 46 | "sharedConfig": { 47 | "nodeArgs": [ 48 | "--loader", 49 | "ts-node/esm", 50 | "--es-module-specifier-resolution=node" 51 | ], 52 | "env": { 53 | "TS_NODE_TRANSPILE_ONLY": "1", 54 | "TS_NODE_PROJECT": "./tsconfig.json", 55 | "NODE_ENV": "development", 56 | "DEBUG": "napi:*" 57 | } 58 | }, 59 | "c8": { 60 | "reporter": [ 61 | "text", 62 | "lcov" 63 | ], 64 | "report-dir": ".coverage", 65 | "exclude": [ 66 | "scripts", 67 | "node_modules", 68 | "**/*.spec.ts" 69 | ] 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /y-octo-node/scripts/run-test.mts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ts-node-esm 2 | import { resolve } from "node:path"; 3 | 4 | import prompts from "prompts"; 5 | import { spawn } from "child_process"; 6 | import { readdir } from "fs/promises"; 7 | import * as process from "process"; 8 | import { fileURLToPath } from "url"; 9 | 10 | import pkg from "../package.json" assert { type: "json" }; 11 | const root = fileURLToPath(new URL("..", import.meta.url)); 12 | const testDir = resolve(root, "tests"); 13 | const files = await readdir(testDir); 14 | 15 | const watchMode = process.argv.includes("--watch"); 16 | 17 | const sharedArgs = [ 18 | ...pkg.sharedConfig.nodeArgs, 19 | "--test", 20 | watchMode ? "--watch" : "", 21 | ]; 22 | 23 | const env = { 24 | ...pkg.sharedConfig.env, 25 | PATH: process.env.PATH, 26 | NODE_ENV: "test", 27 | NODE_NO_WARNINGS: "1", 28 | }; 29 | 30 | if (process.argv[2] === "all") { 31 | const cp = spawn( 32 | "node", 33 | [...sharedArgs, ...files.map((f) => resolve(testDir, f))], 34 | { 35 | cwd: root, 36 | env, 37 | stdio: "inherit", 38 | shell: true, 39 | }, 40 | ); 41 | cp.on("exit", (code) => { 42 | process.exit(code ?? 0); 43 | }); 44 | } else { 45 | const result = await prompts([ 46 | { 47 | type: "select", 48 | name: "file", 49 | message: "Select a file to run", 50 | choices: files.map((file) => ({ 51 | title: file, 52 | value: file, 53 | })), 54 | initial: 1, 55 | }, 56 | ]); 57 | 58 | const target = resolve(testDir, result.file); 59 | 60 | const cp = spawn( 61 | "node", 62 | [ 63 | ...sharedArgs, 64 | "--test-reporter=spec", 65 | "--test-reporter-destination=stdout", 66 | target, 67 | ], 68 | { 69 | cwd: root, 70 | env, 71 | stdio: "inherit", 72 | shell: true, 73 | }, 74 | ); 75 | cp.on("exit", (code) => { 76 | process.exit(code ?? 0); 77 | }); 78 | } 79 | -------------------------------------------------------------------------------- /y-octo-node/src/doc.rs: -------------------------------------------------------------------------------- 1 | use napi::{ 2 | bindgen_prelude::{Buffer as JsBuffer, JsFunction}, 3 | threadsafe_function::{ErrorStrategy, ThreadsafeFunction, ThreadsafeFunctionCallMode}, 4 | }; 5 | use y_octo::{CrdtRead, Doc as YDoc, History, RawDecoder, StateVector}; 6 | 7 | use super::*; 8 | 9 | #[napi] 10 | pub struct Doc { 11 | doc: YDoc, 12 | } 13 | 14 | #[napi] 15 | impl Doc { 16 | #[napi(constructor)] 17 | pub fn new(client_id: Option) -> Self { 18 | Self { 19 | doc: if let Some(client_id) = client_id { 20 | YDoc::with_client(client_id as u64) 21 | } else { 22 | YDoc::default() 23 | }, 24 | } 25 | } 26 | 27 | #[napi(getter)] 28 | pub fn client_id(&self) -> i64 { 29 | self.doc.client() as i64 30 | } 31 | 32 | #[napi(getter)] 33 | pub fn guid(&self) -> &str { 34 | self.doc.guid() 35 | } 36 | 37 | #[napi(getter)] 38 | pub fn keys(&self) -> Vec { 39 | self.doc.keys() 40 | } 41 | 42 | #[napi] 43 | pub fn get_or_create_array(&self, key: String) -> Result { 44 | self.doc 45 | .get_or_create_array(key) 46 | .map(YArray::inner_new) 47 | .map_err(anyhow::Error::from) 48 | } 49 | 50 | #[napi] 51 | pub fn get_or_create_text(&self, key: String) -> Result { 52 | self.doc 53 | .get_or_create_text(key) 54 | .map(YText::inner_new) 55 | .map_err(anyhow::Error::from) 56 | } 57 | 58 | #[napi] 59 | pub fn get_or_create_map(&self, key: String) -> Result { 60 | self.doc 61 | .get_or_create_map(key) 62 | .map(YMap::inner_new) 63 | .map_err(anyhow::Error::from) 64 | } 65 | 66 | #[napi] 67 | pub fn create_array(&self) -> Result { 68 | self.doc 69 | .create_array() 70 | .map(YArray::inner_new) 71 | .map_err(anyhow::Error::from) 72 | } 73 | 74 | #[napi] 75 | pub fn create_text(&self) -> Result { 76 | self.doc 77 | .create_text() 78 | .map(YText::inner_new) 79 | .map_err(anyhow::Error::from) 80 | } 81 | 82 | #[napi] 83 | pub fn create_map(&self) -> Result { 84 | self.doc.create_map().map(YMap::inner_new).map_err(anyhow::Error::from) 85 | } 86 | 87 | #[napi] 88 | pub fn apply_update(&mut self, update: JsBuffer) -> Result<()> { 89 | self.doc.apply_update_from_binary_v1(update)?; 90 | 91 | Ok(()) 92 | } 93 | 94 | #[napi] 95 | pub fn encode_state_as_update_v1(&self, state: Option) -> Result { 96 | let result = match state { 97 | Some(state) => { 98 | let mut decoder = RawDecoder::new(state.as_ref()); 99 | let state = StateVector::read(&mut decoder)?; 100 | self.doc.encode_state_as_update_v1(&state) 101 | } 102 | None => self.doc.encode_update_v1(), 103 | }; 104 | 105 | result.map(|v| v.into()).map_err(anyhow::Error::from) 106 | } 107 | 108 | #[napi] 109 | pub fn gc(&self) -> Result<()> { 110 | self.doc.gc().map_err(anyhow::Error::from) 111 | } 112 | 113 | #[napi(ts_args_type = "callback: (result: Uint8Array) => void")] 114 | pub fn on_update(&mut self, callback: JsFunction) -> Result<()> { 115 | let tsfn: ThreadsafeFunction = 116 | callback.create_threadsafe_function(0, |ctx| Ok(vec![ctx.value]))?; 117 | 118 | let callback = move |update: &[u8], _h: &[History]| { 119 | tsfn.call(JsBuffer::from(update.to_vec()), ThreadsafeFunctionCallMode::Blocking); 120 | }; 121 | self.doc.subscribe(Box::new(callback)); 122 | Ok(()) 123 | } 124 | } 125 | 126 | #[cfg(test)] 127 | mod tests { 128 | use super::*; 129 | 130 | #[test] 131 | fn test_doc_client() { 132 | let client_id = 1; 133 | let doc = Doc::new(Some(client_id)); 134 | assert_eq!(doc.client_id(), 1); 135 | } 136 | 137 | #[test] 138 | fn test_doc_guid() { 139 | let doc = Doc::new(None); 140 | assert_eq!(doc.guid().len(), 21); 141 | } 142 | 143 | #[test] 144 | fn test_create_array() { 145 | let doc = Doc::new(None); 146 | let array = doc.get_or_create_array("array".into()).unwrap(); 147 | assert_eq!(array.length(), 0); 148 | } 149 | 150 | #[test] 151 | fn test_create_text() { 152 | let doc = Doc::new(None); 153 | let text = doc.get_or_create_text("text".into()).unwrap(); 154 | assert_eq!(text.len(), 0); 155 | } 156 | 157 | #[test] 158 | fn test_keys() { 159 | let doc = Doc::new(None); 160 | doc.get_or_create_array("array".into()).unwrap(); 161 | doc.get_or_create_text("text".into()).unwrap(); 162 | doc.get_or_create_map("map".into()).unwrap(); 163 | let mut keys = doc.keys(); 164 | keys.sort(); 165 | assert_eq!(keys, vec!["array", "map", "text"]); 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /y-octo-node/src/lib.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use napi_derive::napi; 3 | 4 | mod array; 5 | mod doc; 6 | mod map; 7 | mod text; 8 | mod utils; 9 | 10 | pub use array::YArray; 11 | pub use doc::Doc; 12 | pub use map::YMap; 13 | pub use text::YText; 14 | use utils::{ 15 | get_any_from_js_object, get_any_from_js_unknown, get_js_unknown_from_any, get_js_unknown_from_value, MixedRefYType, 16 | MixedYType, 17 | }; 18 | -------------------------------------------------------------------------------- /y-octo-node/src/map.rs: -------------------------------------------------------------------------------- 1 | use napi::{Env, JsObject, ValueType}; 2 | use y_octo::{Any, Map, Value}; 3 | 4 | use super::*; 5 | 6 | #[napi] 7 | pub struct YMap { 8 | pub(crate) map: Map, 9 | } 10 | 11 | #[napi] 12 | impl YMap { 13 | #[allow(clippy::new_without_default)] 14 | #[napi(constructor)] 15 | pub fn new() -> Self { 16 | unimplemented!() 17 | } 18 | 19 | pub(crate) fn inner_new(map: Map) -> Self { 20 | Self { map } 21 | } 22 | 23 | #[napi(getter)] 24 | pub fn length(&self) -> i64 { 25 | self.map.len() as i64 26 | } 27 | 28 | #[napi(getter)] 29 | pub fn is_empty(&self) -> bool { 30 | self.map.is_empty() 31 | } 32 | 33 | #[napi(ts_generic_types = "T = unknown", ts_return_type = "T")] 34 | pub fn get(&self, env: Env, key: String) -> Result { 35 | if let Some(value) = self.map.get(&key) { 36 | match value { 37 | Value::Any(any) => get_js_unknown_from_any(env, any).map(MixedYType::D), 38 | Value::Array(array) => Ok(MixedYType::A(YArray::inner_new(array))), 39 | Value::Map(map) => Ok(MixedYType::B(YMap::inner_new(map))), 40 | Value::Text(text) => Ok(MixedYType::C(YText::inner_new(text))), 41 | _ => env.get_null().map(|v| v.into_unknown()).map(MixedYType::D), 42 | } 43 | .map_err(anyhow::Error::from) 44 | } else { 45 | Ok(MixedYType::D(env.get_null()?.into_unknown())) 46 | } 47 | } 48 | 49 | #[napi( 50 | ts_args_type = "key: string, value: YArray | YMap | YText | boolean | number | string | Record | \ 51 | null | undefined" 52 | )] 53 | pub fn set(&mut self, key: String, value: MixedRefYType) -> Result<()> { 54 | match value { 55 | MixedRefYType::A(array) => self.map.insert(key, array.array.clone()).map_err(anyhow::Error::from), 56 | MixedRefYType::B(map) => self.map.insert(key, map.map.clone()).map_err(anyhow::Error::from), 57 | MixedRefYType::C(text) => self.map.insert(key, text.text.clone()).map_err(anyhow::Error::from), 58 | MixedRefYType::D(unknown) => match unknown.get_type() { 59 | Ok(value_type) => match value_type { 60 | ValueType::Undefined | ValueType::Null => { 61 | self.map.insert(key, Any::Null).map_err(anyhow::Error::from) 62 | } 63 | ValueType::Boolean => match unknown.coerce_to_bool().and_then(|v| v.get_value()) { 64 | Ok(boolean) => self.map.insert(key, boolean).map_err(anyhow::Error::from), 65 | Err(e) => Err(anyhow::Error::from(e).context("Failed to coerce value to boolean")), 66 | }, 67 | ValueType::Number => match unknown.coerce_to_number().and_then(|v| v.get_double()) { 68 | Ok(number) => self.map.insert(key, number).map_err(anyhow::Error::from), 69 | Err(e) => Err(anyhow::Error::from(e).context("Failed to coerce value to number")), 70 | }, 71 | ValueType::String => { 72 | match unknown 73 | .coerce_to_string() 74 | .and_then(|v| v.into_utf8()) 75 | .and_then(|s| s.as_str().map(|s| s.to_string())) 76 | { 77 | Ok(string) => self.map.insert(key, string).map_err(anyhow::Error::from), 78 | Err(e) => Err(anyhow::Error::from(e).context("Failed to coerce value to string")), 79 | } 80 | } 81 | ValueType::Object => match unknown.coerce_to_object().and_then(get_any_from_js_object) { 82 | Ok(any) => self.map.insert(key, Value::Any(any)).map_err(anyhow::Error::from), 83 | Err(e) => Err(anyhow::Error::from(e).context("Failed to coerce value to object")), 84 | }, 85 | ValueType::Symbol => Err(anyhow::Error::msg("Symbol values are not supported")), 86 | ValueType::Function => Err(anyhow::Error::msg("Function values are not supported")), 87 | ValueType::External => Err(anyhow::Error::msg("External values are not supported")), 88 | ValueType::Unknown => Err(anyhow::Error::msg("Unknown values are not supported")), 89 | }, 90 | Err(e) => Err(anyhow::Error::from(e)), 91 | }, 92 | } 93 | } 94 | 95 | #[napi] 96 | pub fn remove(&mut self, key: String) { 97 | self.map.remove(&key); 98 | } 99 | 100 | #[napi] 101 | pub fn to_json(&self, env: Env) -> Result { 102 | let mut js_object = env.create_object()?; 103 | for (key, value) in self.map.iter() { 104 | js_object.set(key, get_js_unknown_from_value(env, value))?; 105 | } 106 | Ok(js_object) 107 | } 108 | } 109 | 110 | #[cfg(test)] 111 | mod tests { 112 | 113 | use super::*; 114 | 115 | #[test] 116 | fn test_map_init() { 117 | let doc = Doc::new(None); 118 | let text = doc.get_or_create_map("map".into()).unwrap(); 119 | assert_eq!(text.length(), 0); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /y-octo-node/src/text.rs: -------------------------------------------------------------------------------- 1 | use y_octo::Text; 2 | 3 | use super::*; 4 | 5 | #[napi] 6 | pub struct YText { 7 | pub(crate) text: Text, 8 | } 9 | 10 | #[napi] 11 | impl YText { 12 | #[allow(clippy::new_without_default)] 13 | #[napi(constructor)] 14 | pub fn new() -> Self { 15 | unimplemented!() 16 | } 17 | 18 | pub(crate) fn inner_new(text: Text) -> Self { 19 | Self { text } 20 | } 21 | 22 | #[napi(getter)] 23 | pub fn len(&self) -> i64 { 24 | self.text.len() as i64 25 | } 26 | 27 | #[napi(getter)] 28 | pub fn is_empty(&self) -> bool { 29 | self.text.is_empty() 30 | } 31 | 32 | #[napi] 33 | pub fn insert(&mut self, index: i64, str: String) -> Result<()> { 34 | self.text.insert(index as u64, str).map_err(anyhow::Error::from) 35 | } 36 | 37 | #[napi] 38 | pub fn remove(&mut self, index: i64, len: i64) -> Result<()> { 39 | self.text.remove(index as u64, len as u64).map_err(anyhow::Error::from) 40 | } 41 | 42 | #[napi(getter)] 43 | pub fn length(&self) -> i64 { 44 | self.text.len() as i64 45 | } 46 | 47 | #[allow(clippy::inherent_to_string)] 48 | #[napi] 49 | pub fn to_string(&self) -> String { 50 | self.text.to_string() 51 | } 52 | } 53 | 54 | #[cfg(test)] 55 | mod tests { 56 | use super::*; 57 | 58 | #[test] 59 | fn test_text_init() { 60 | let doc = Doc::new(None); 61 | let text = doc.get_or_create_text("text".into()).unwrap(); 62 | assert_eq!(text.len(), 0); 63 | } 64 | 65 | #[test] 66 | fn test_text_edit() { 67 | let doc = Doc::new(None); 68 | let mut text = doc.get_or_create_text("text".into()).unwrap(); 69 | text.insert(0, "hello".into()).unwrap(); 70 | assert_eq!(text.to_string(), "hello"); 71 | text.insert(5, " world".into()).unwrap(); 72 | assert_eq!(text.to_string(), "hello world"); 73 | text.remove(5, 6).unwrap(); 74 | assert_eq!(text.to_string(), "hello"); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /y-octo-node/src/utils.rs: -------------------------------------------------------------------------------- 1 | use napi::{bindgen_prelude::Either4, Env, Error, JsObject, JsUnknown, Result, Status, ValueType}; 2 | use y_octo::{AHashMap, Any, HashMapExt, Value}; 3 | 4 | use super::*; 5 | 6 | pub type MixedYType = Either4; 7 | pub type MixedRefYType<'a> = Either4<&'a YArray, &'a YMap, &'a YText, JsUnknown>; 8 | 9 | pub fn get_js_unknown_from_any(env: Env, any: Any) -> Result { 10 | match any { 11 | Any::Null | Any::Undefined => env.get_null().map(|v| v.into_unknown()), 12 | Any::True => env.get_boolean(true).map(|v| v.into_unknown()), 13 | Any::False => env.get_boolean(false).map(|v| v.into_unknown()), 14 | Any::Integer(number) => env.create_int32(number).map(|v| v.into_unknown()), 15 | Any::BigInt64(number) => env.create_int64(number).map(|v| v.into_unknown()), 16 | Any::Float32(number) => env.create_double(number.0 as f64).map(|v| v.into_unknown()), 17 | Any::Float64(number) => env.create_double(number.0).map(|v| v.into_unknown()), 18 | Any::String(string) => env.create_string(string.as_str()).map(|v| v.into_unknown()), 19 | Any::Array(array) => { 20 | let mut js_array = env.create_array_with_length(array.len())?; 21 | for (i, value) in array.into_iter().enumerate() { 22 | js_array.set_element(i as u32, get_js_unknown_from_any(env, value)?)?; 23 | } 24 | Ok(js_array.into_unknown()) 25 | } 26 | _ => env.get_null().map(|v| v.into_unknown()), 27 | } 28 | } 29 | 30 | pub fn get_js_unknown_from_value(env: Env, value: Value) -> Result { 31 | match value { 32 | Value::Any(any) => get_js_unknown_from_any(env, any), 33 | Value::Array(array) => env 34 | .create_external(YArray::inner_new(array), None) 35 | .map(|o| o.into_unknown()), 36 | Value::Map(map) => env 37 | .create_external(YMap::inner_new(map), None) 38 | .map(|o| o.into_unknown()), 39 | Value::Text(text) => env 40 | .create_external(YText::inner_new(text), None) 41 | .map(|o| o.into_unknown()), 42 | _ => env.get_null().map(|v| v.into_unknown()), 43 | } 44 | } 45 | 46 | pub fn get_any_from_js_object(object: JsObject) -> Result { 47 | if let Ok(length) = object.get_array_length() { 48 | let mut array = Vec::with_capacity(length as usize); 49 | for i in 0..length { 50 | if let Ok(value) = object.get_element::(i) { 51 | array.push(get_any_from_js_unknown(value)?); 52 | } 53 | } 54 | Ok(Any::Array(array)) 55 | } else { 56 | let mut map = AHashMap::new(); 57 | let keys = object.get_property_names()?; 58 | if let Ok(length) = keys.get_array_length() { 59 | for i in 0..length { 60 | if let Ok((obj, key)) = keys.get_element::(i).and_then(|o| { 61 | o.coerce_to_string() 62 | .and_then(|obj| obj.into_utf8().and_then(|s| s.as_str().map(|s| (obj, s.to_string())))) 63 | }) { 64 | if let Ok(value) = object.get_property::<_, JsUnknown>(obj) { 65 | println!("key: {}", key); 66 | map.insert(key, get_any_from_js_unknown(value)?); 67 | } 68 | } 69 | } 70 | } 71 | Ok(Any::Object(map)) 72 | } 73 | } 74 | 75 | pub fn get_any_from_js_unknown(js_unknown: JsUnknown) -> Result { 76 | match js_unknown.get_type()? { 77 | ValueType::Undefined | ValueType::Null => Ok(Any::Null), 78 | ValueType::Boolean => Ok(js_unknown.coerce_to_bool().and_then(|v| v.get_value())?.into()), 79 | ValueType::Number => Ok(js_unknown 80 | .coerce_to_number() 81 | .and_then(|v| v.get_double()) 82 | .map(|v| v.into())?), 83 | ValueType::String => Ok(js_unknown 84 | .coerce_to_string() 85 | .and_then(|v| v.into_utf8()) 86 | .and_then(|s| s.as_str().map(|s| s.to_string()))? 87 | .into()), 88 | ValueType::Object => { 89 | if let Ok(object) = js_unknown.coerce_to_object() { 90 | get_any_from_js_object(object) 91 | } else { 92 | Err(Error::new(Status::InvalidArg, "Failed to coerce value to object")) 93 | } 94 | } 95 | _ => Err(Error::new(Status::InvalidArg, "Failed to coerce value to any")), 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /y-octo-node/tests/array.spec.mts: -------------------------------------------------------------------------------- 1 | import assert, { equal, deepEqual } from "node:assert"; 2 | import { test } from "node:test"; 3 | 4 | import { Doc, YArray } from "../index"; 5 | 6 | test("array test", { concurrency: false }, async (t) => { 7 | let client_id: number; 8 | let doc: Doc; 9 | t.beforeEach(async () => { 10 | client_id = (Math.random() * 100000) | 0; 11 | doc = new Doc(client_id); 12 | }); 13 | 14 | t.afterEach(async () => { 15 | client_id = -1; 16 | // @ts-ignore - doc must not null in next range 17 | doc = null; 18 | }); 19 | 20 | await t.test("array should be created", () => { 21 | let arr = doc.getOrCreateArray("arr"); 22 | deepEqual(doc.keys, ["arr"]); 23 | equal(arr.length, 0); 24 | }); 25 | 26 | await t.test("array editing", () => { 27 | let arr = doc.getOrCreateArray("arr"); 28 | arr.insert(0, true); 29 | arr.insert(1, false); 30 | arr.insert(2, 1); 31 | arr.insert(3, "hello world"); 32 | equal(arr.length, 4); 33 | equal(arr.get(0), true); 34 | equal(arr.get(1), false); 35 | equal(arr.get(2), 1); 36 | equal(arr.get(3), "hello world"); 37 | equal(arr.length, 4); 38 | arr.remove(1, 1); 39 | equal(arr.length, 3); 40 | equal(arr.get(2), "hello world"); 41 | }); 42 | 43 | await t.test("sub array should can edit", () => { 44 | let map = doc.getOrCreateMap("map"); 45 | let sub = doc.createArray(); 46 | map.set("sub", sub); 47 | 48 | sub.insert(0, true); 49 | sub.insert(1, false); 50 | sub.insert(2, 1); 51 | sub.insert(3, "hello world"); 52 | equal(sub.length, 4); 53 | 54 | let sub2 = map.get("sub"); 55 | assert(sub2); 56 | equal(sub2.get(0), true); 57 | equal(sub2.get(1), false); 58 | equal(sub2.get(2), 1); 59 | equal(sub2.get(3), "hello world"); 60 | equal(sub2.length, 4); 61 | }); 62 | }); 63 | -------------------------------------------------------------------------------- /y-octo-node/tests/doc.spec.mts: -------------------------------------------------------------------------------- 1 | import assert, { equal } from "node:assert"; 2 | import { test } from "node:test"; 3 | 4 | import { Doc } from "../index"; 5 | import * as Y from "yjs"; 6 | 7 | test("doc test", { concurrency: false }, async (t) => { 8 | let client_id: number; 9 | let doc: Doc; 10 | t.beforeEach(async () => { 11 | client_id = (Math.random() * 100000) | 0; 12 | doc = new Doc(client_id); 13 | }); 14 | 15 | t.afterEach(async () => { 16 | client_id = -1; 17 | // @ts-ignore - doc must not null in next range 18 | doc = null; 19 | }); 20 | 21 | await t.test("doc id should be set", () => { 22 | equal(doc.clientId, client_id); 23 | }); 24 | 25 | await t.test("y-octo doc update should be apply", () => { 26 | let array = doc.getOrCreateArray("array"); 27 | let map = doc.getOrCreateMap("map"); 28 | let text = doc.getOrCreateText("text"); 29 | 30 | array.insert(0, true); 31 | array.insert(1, false); 32 | array.insert(2, 1); 33 | array.insert(3, "hello world"); 34 | map.set("a", true); 35 | map.set("b", false); 36 | map.set("c", 1); 37 | map.set("d", "hello world"); 38 | text.insert(0, "a"); 39 | text.insert(1, "b"); 40 | text.insert(2, "c"); 41 | 42 | let doc2 = new Doc(client_id); 43 | doc2.applyUpdate(doc.encodeStateAsUpdateV1()); 44 | 45 | let array2 = doc2.getOrCreateArray("array"); 46 | let map2 = doc2.getOrCreateMap("map"); 47 | let text2 = doc2.getOrCreateText("text"); 48 | 49 | equal(doc2.clientId, client_id); 50 | equal(array2.length, 4); 51 | equal(array2.get(0), true); 52 | equal(array2.get(1), false); 53 | equal(array2.get(2), 1); 54 | equal(array2.get(3), "hello world"); 55 | equal(map2.length, 4); 56 | equal(map2.get("a"), true); 57 | equal(map2.get("b"), false); 58 | equal(map2.get("c"), 1); 59 | equal(map2.get("d"), "hello world"); 60 | equal(text2.toString(), "abc"); 61 | }); 62 | 63 | await t.test("yjs doc update should be apply", () => { 64 | let doc2 = new Y.Doc(); 65 | let array2 = doc2.getArray("array"); 66 | let map2 = doc2.getMap("map"); 67 | let text2 = doc2.getText("text"); 68 | 69 | array2.insert(0, [true]); 70 | array2.insert(1, [false]); 71 | array2.insert(2, [1]); 72 | array2.insert(3, ["hello world"]); 73 | map2.set("a", true); 74 | map2.set("b", false); 75 | map2.set("c", 1); 76 | map2.set("d", "hello world"); 77 | text2.insert(0, "a"); 78 | text2.insert(1, "b"); 79 | text2.insert(2, "c"); 80 | 81 | doc.applyUpdate(Buffer.from(Y.encodeStateAsUpdate(doc2))); 82 | 83 | let array = doc.getOrCreateArray("array"); 84 | let map = doc.getOrCreateMap("map"); 85 | let text = doc.getOrCreateText("text"); 86 | 87 | equal(array.length, 4); 88 | equal(array.get(0), true); 89 | equal(array.get(1), false); 90 | equal(array.get(2), 1); 91 | equal(array.get(3), "hello world"); 92 | equal(map.length, 4); 93 | equal(map.get("a"), true); 94 | equal(map.get("b"), false); 95 | equal(map.get("c"), 1); 96 | equal(map.get("d"), "hello world"); 97 | equal(text.toString(), "abc"); 98 | }); 99 | }); 100 | -------------------------------------------------------------------------------- /y-octo-node/tests/map.spec.mts: -------------------------------------------------------------------------------- 1 | import assert, { equal, deepEqual } from "node:assert"; 2 | import { test } from "node:test"; 3 | 4 | import * as Y from "yjs"; 5 | import { Doc, YArray, YMap, YText } from "../index"; 6 | 7 | test("map test", { concurrency: false }, async (t) => { 8 | let client_id: number; 9 | let doc: Doc; 10 | t.beforeEach(async () => { 11 | client_id = (Math.random() * 100000) | 0; 12 | doc = new Doc(client_id); 13 | }); 14 | 15 | t.afterEach(async () => { 16 | client_id = -1; 17 | // @ts-ignore - doc must not null in next range 18 | doc = null; 19 | }); 20 | 21 | await t.test("map should be created", () => { 22 | let map = doc.getOrCreateMap("map"); 23 | deepEqual(doc.keys, ["map"]); 24 | equal(map.length, 0); 25 | }); 26 | 27 | await t.test("map editing", () => { 28 | let map = doc.getOrCreateMap("map"); 29 | map.set("a", true); 30 | map.set("b", false); 31 | map.set("c", 1); 32 | map.set("d", "hello world"); 33 | equal(map.length, 4); 34 | equal(map.get("a"), true); 35 | equal(map.get("b"), false); 36 | equal(map.get("c"), 1); 37 | equal(map.get("d"), "hello world"); 38 | equal(map.length, 4); 39 | map.remove("b"); 40 | equal(map.length, 3); 41 | equal(map.get("d"), "hello world"); 42 | }); 43 | 44 | await t.test("map should can be nested", () => { 45 | let map = doc.getOrCreateMap("map"); 46 | let sub = doc.createMap(); 47 | map.set("sub", sub); 48 | 49 | sub.set("a", true); 50 | sub.set("b", false); 51 | sub.set("c", 1); 52 | sub.set("d", "hello world"); 53 | equal(sub.length, 4); 54 | 55 | let sub2 = map.get("sub"); 56 | assert(sub2); 57 | equal(sub2.get("a"), true); 58 | equal(sub2.get("b"), false); 59 | equal(sub2.get("c"), 1); 60 | equal(sub2.get("d"), "hello world"); 61 | equal(sub2.length, 4); 62 | }); 63 | 64 | await t.test("y-octo to yjs compatibility test with nested type", () => { 65 | let map = doc.getOrCreateMap("map"); 66 | let sub_array = doc.createArray(); 67 | let sub_map = doc.createMap(); 68 | let sub_text = doc.createText(); 69 | 70 | map.set("array", sub_array); 71 | map.set("map", sub_map); 72 | map.set("text", sub_text); 73 | 74 | sub_array.insert(0, true); 75 | sub_array.insert(1, false); 76 | sub_array.insert(2, 1); 77 | sub_array.insert(3, "hello world"); 78 | sub_map.set("a", true); 79 | sub_map.set("b", false); 80 | sub_map.set("c", 1); 81 | sub_map.set("d", "hello world"); 82 | sub_text.insert(0, "a"); 83 | sub_text.insert(1, "b"); 84 | sub_text.insert(2, "c"); 85 | 86 | let doc2 = new Y.Doc(); 87 | Y.applyUpdate(doc2, doc.encodeStateAsUpdateV1()); 88 | 89 | let map2 = doc2.getMap("map"); 90 | let sub_array2 = map2.get("array") as Y.Array; 91 | let sub_map2 = map2.get("map") as Y.Map; 92 | let sub_text2 = map2.get("text") as Y.Text; 93 | 94 | assert(sub_array2); 95 | equal(sub_array2.length, 4); 96 | equal(sub_array2.get(0), true); 97 | equal(sub_array2.get(1), false); 98 | equal(sub_array2.get(2), 1); 99 | equal(sub_array2.get(3), "hello world"); 100 | assert(sub_map2); 101 | equal(sub_map2.get("a"), true); 102 | equal(sub_map2.get("b"), false); 103 | equal(sub_map2.get("c"), 1); 104 | equal(sub_map2.get("d"), "hello world"); 105 | assert(sub_text2); 106 | equal(sub_text2.toString(), "abc"); 107 | }); 108 | 109 | await t.test("yjs to y-octo compatibility test with nested type", () => { 110 | let doc2 = new Y.Doc(); 111 | let map2 = doc2.getMap("map"); 112 | let sub_array2 = new Y.Array(); 113 | let sub_map2 = new Y.Map(); 114 | let sub_text2 = new Y.Text(); 115 | map2.set("array", sub_array2); 116 | map2.set("map", sub_map2); 117 | map2.set("text", sub_text2); 118 | 119 | sub_array2.insert(0, [true]); 120 | sub_array2.insert(1, [false]); 121 | sub_array2.insert(2, [1]); 122 | sub_array2.insert(3, ["hello world"]); 123 | sub_map2.set("a", true); 124 | sub_map2.set("b", false); 125 | sub_map2.set("c", 1); 126 | sub_map2.set("d", "hello world"); 127 | sub_text2.insert(0, "a"); 128 | sub_text2.insert(1, "b"); 129 | sub_text2.insert(2, "c"); 130 | 131 | doc.applyUpdate(Buffer.from(Y.encodeStateAsUpdate(doc2))); 132 | 133 | let map = doc.getOrCreateMap("map"); 134 | let sub_array = map.get("array"); 135 | let sub_map = map.get("map"); 136 | let sub_text = map.get("text"); 137 | 138 | assert(sub_array); 139 | equal(sub_array.length, 4); 140 | equal(sub_array.get(0), true); 141 | equal(sub_array.get(1), false); 142 | equal(sub_array.get(2), 1); 143 | equal(sub_array.get(3), "hello world"); 144 | assert(sub_map); 145 | equal(sub_map.get("a"), true); 146 | equal(sub_map.get("b"), false); 147 | equal(sub_map.get("c"), 1); 148 | equal(sub_map.get("d"), "hello world"); 149 | assert(sub_text); 150 | equal(sub_text.toString(), "abc"); 151 | }); 152 | }); 153 | -------------------------------------------------------------------------------- /y-octo-node/tests/text.spec.mts: -------------------------------------------------------------------------------- 1 | import assert, { equal, deepEqual } from "node:assert"; 2 | import { test } from "node:test"; 3 | 4 | import { Doc, YText } from "../index"; 5 | 6 | test("text test", { concurrency: false }, async (t) => { 7 | let client_id: number; 8 | let doc: Doc; 9 | t.beforeEach(async () => { 10 | client_id = (Math.random() * 100000) | 0; 11 | doc = new Doc(client_id); 12 | }); 13 | 14 | t.afterEach(async () => { 15 | client_id = -1; 16 | // @ts-ignore - doc must not null in next range 17 | doc = null; 18 | }); 19 | 20 | await t.test("text should be created", () => { 21 | let text = doc.getOrCreateText("text"); 22 | deepEqual(doc.keys, ["text"]); 23 | equal(text.len, 0); 24 | }); 25 | 26 | await t.test("text editing", () => { 27 | let text = doc.getOrCreateText("text"); 28 | text.insert(0, "a"); 29 | text.insert(1, "b"); 30 | text.insert(2, "c"); 31 | equal(text.toString(), "abc"); 32 | text.remove(0, 1); 33 | equal(text.toString(), "bc"); 34 | text.remove(1, 1); 35 | equal(text.toString(), "b"); 36 | text.remove(0, 1); 37 | equal(text.toString(), ""); 38 | }); 39 | 40 | await t.test("sub text should can edit", () => { 41 | let map = doc.getOrCreateMap("map"); 42 | let sub = doc.createText(); 43 | map.set("sub", sub); 44 | 45 | sub.insert(0, "a"); 46 | sub.insert(1, "b"); 47 | sub.insert(2, "c"); 48 | equal(sub.toString(), "abc"); 49 | 50 | let sub2 = map.get("sub"); 51 | assert(sub2); 52 | equal(sub2.toString(), "abc"); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /y-octo-node/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "noEmit": false, 5 | "outDir": "lib", 6 | "composite": true 7 | }, 8 | "include": ["index.d.ts", "tests/**/*.mts"], 9 | "ts-node": { 10 | "esm": true, 11 | "experimentalSpecifierResolution": "node" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /y-octo-utils/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = ["x1a0t <405028157@qq.com>", "DarkSky "] 3 | edition = "2021" 4 | license = "MIT" 5 | name = "y-octo-utils" 6 | version = "0.0.1" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [features] 11 | bench = ["regex"] 12 | default = ["merger"] 13 | fuzz = ["arbitrary", "phf"] 14 | merger = ["clap", "y-octo/large_refs"] 15 | 16 | [dependencies] 17 | arbitrary = { version = "1.3", features = ["derive"], optional = true } 18 | clap = { version = "4.4", features = ["derive"], optional = true } 19 | lib0 = { version = "=0.16.5", features = ["lib0-serde"] } 20 | phf = { version = "0.11", features = ["macros"], optional = true } 21 | rand = "0.8" 22 | rand_chacha = "0.3" 23 | regex = { version = "1.10", optional = true } 24 | y-octo = { workspace = true } 25 | y-sync = { workspace = true } 26 | yrs = { workspace = true } 27 | 28 | [dev-dependencies] 29 | criterion = { version = "0.5", features = ["html_reports"] } 30 | path-ext = "0.1" 31 | proptest = "1.3" 32 | proptest-derive = "0.5" 33 | 34 | [[bin]] 35 | name = "bench_result_render" 36 | path = "bin/bench_result_render.rs" 37 | 38 | [[bin]] 39 | name = "doc_merger" 40 | path = "bin/doc_merger.rs" 41 | 42 | [[bin]] 43 | name = "memory_leak_test" 44 | path = "bin/memory_leak_test.rs" 45 | 46 | [[bench]] 47 | harness = false 48 | name = "array_ops_benchmarks" 49 | 50 | [[bench]] 51 | harness = false 52 | name = "codec_benchmarks" 53 | 54 | [[bench]] 55 | harness = false 56 | name = "map_ops_benchmarks" 57 | 58 | [[bench]] 59 | harness = false 60 | name = "text_ops_benchmarks" 61 | 62 | [[bench]] 63 | harness = false 64 | name = "apply_benchmarks" 65 | 66 | [[bench]] 67 | harness = false 68 | name = "update_benchmarks" 69 | 70 | [lib] 71 | bench = true 72 | -------------------------------------------------------------------------------- /y-octo-utils/benches/apply_benchmarks.rs: -------------------------------------------------------------------------------- 1 | mod utils; 2 | 3 | use std::time::Duration; 4 | 5 | use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; 6 | use path_ext::PathExt; 7 | use utils::Files; 8 | 9 | fn apply(c: &mut Criterion) { 10 | let files = Files::load(); 11 | 12 | let mut group = c.benchmark_group("apply"); 13 | group.measurement_time(Duration::from_secs(15)); 14 | 15 | for file in &files.files { 16 | group.throughput(Throughput::Bytes(file.content.len() as u64)); 17 | group.bench_with_input( 18 | BenchmarkId::new("apply with yrs", file.path.name_str()), 19 | &file.content, 20 | |b, content| { 21 | b.iter(|| { 22 | use yrs::{updates::decoder::Decode, Doc, Transact, Update}; 23 | let update = Update::decode_v1(content).unwrap(); 24 | let doc = Doc::new(); 25 | doc.transact_mut().apply_update(update); 26 | }); 27 | }, 28 | ); 29 | } 30 | 31 | group.finish(); 32 | } 33 | 34 | criterion_group!(benches, apply); 35 | criterion_main!(benches); 36 | -------------------------------------------------------------------------------- /y-octo-utils/benches/array_ops_benchmarks.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use criterion::{criterion_group, criterion_main, Criterion}; 4 | use rand::{Rng, SeedableRng}; 5 | 6 | fn operations(c: &mut Criterion) { 7 | let mut group = c.benchmark_group("ops/array"); 8 | group.measurement_time(Duration::from_secs(15)); 9 | 10 | group.bench_function("yrs/insert", |b| { 11 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; 12 | let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); 13 | 14 | let idxs = (0..99) 15 | .map(|_| rng.gen_range(0..base_text.len() as u32)) 16 | .collect::>(); 17 | b.iter(|| { 18 | use yrs::*; 19 | let doc = Doc::new(); 20 | let array = doc.get_or_insert_array("test"); 21 | 22 | let mut trx = doc.transact_mut(); 23 | for c in base_text.chars() { 24 | array.push_back(&mut trx, c.to_string()).unwrap(); 25 | } 26 | for idx in &idxs { 27 | array.insert(&mut trx, *idx, "test").unwrap(); 28 | } 29 | drop(trx); 30 | }); 31 | }); 32 | 33 | group.bench_function("yrs/insert range", |b| { 34 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; 35 | let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); 36 | 37 | let idxs = (0..99) 38 | .map(|_| rng.gen_range(0..base_text.len() as u32)) 39 | .collect::>(); 40 | b.iter(|| { 41 | use yrs::*; 42 | let doc = Doc::new(); 43 | let array = doc.get_or_insert_array("test"); 44 | 45 | let mut trx = doc.transact_mut(); 46 | for c in base_text.chars() { 47 | array.push_back(&mut trx, c.to_string()).unwrap(); 48 | } 49 | for idx in &idxs { 50 | array.insert_range(&mut trx, *idx, vec!["test1", "test2"]).unwrap(); 51 | } 52 | drop(trx); 53 | }); 54 | }); 55 | 56 | group.bench_function("yrs/remove", |b| { 57 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; 58 | 59 | b.iter(|| { 60 | use yrs::*; 61 | let doc = Doc::new(); 62 | let array = doc.get_or_insert_array("test"); 63 | 64 | let mut trx = doc.transact_mut(); 65 | for c in base_text.chars() { 66 | array.push_back(&mut trx, c.to_string()).unwrap(); 67 | } 68 | for idx in (base_text.len() as u32)..0 { 69 | array.remove(&mut trx, idx).unwrap(); 70 | } 71 | drop(trx); 72 | }); 73 | }); 74 | 75 | group.finish(); 76 | } 77 | 78 | criterion_group!(benches, operations); 79 | criterion_main!(benches); 80 | -------------------------------------------------------------------------------- /y-octo-utils/benches/codec_benchmarks.rs: -------------------------------------------------------------------------------- 1 | use criterion::{criterion_group, criterion_main, Criterion, SamplingMode}; 2 | use lib0::{ 3 | decoding::{Cursor, Read}, 4 | encoding::Write, 5 | }; 6 | 7 | const BENCHMARK_SIZE: u32 = 100000; 8 | 9 | fn codec(c: &mut Criterion) { 10 | let mut codec_group = c.benchmark_group("codec"); 11 | codec_group.sampling_mode(SamplingMode::Flat); 12 | { 13 | codec_group.bench_function("lib0 encode var_int (64 bit)", |b| { 14 | b.iter(|| { 15 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 16 | for i in 0..(BENCHMARK_SIZE as i64) { 17 | encoder.write_var(i); 18 | } 19 | }) 20 | }); 21 | codec_group.bench_function("lib0 decode var_int (64 bit)", |b| { 22 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 23 | for i in 0..(BENCHMARK_SIZE as i64) { 24 | encoder.write_var(i); 25 | } 26 | 27 | b.iter(|| { 28 | let mut decoder = Cursor::from(&encoder); 29 | for i in 0..(BENCHMARK_SIZE as i64) { 30 | let num: i64 = decoder.read_var().unwrap(); 31 | assert_eq!(num, i); 32 | } 33 | }) 34 | }); 35 | } 36 | 37 | { 38 | codec_group.bench_function("lib0 encode var_uint (32 bit)", |b| { 39 | b.iter(|| { 40 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 41 | for i in 0..BENCHMARK_SIZE { 42 | encoder.write_var(i); 43 | } 44 | }) 45 | }); 46 | codec_group.bench_function("lib0 decode var_uint (32 bit)", |b| { 47 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 48 | for i in 0..BENCHMARK_SIZE { 49 | encoder.write_var(i); 50 | } 51 | 52 | b.iter(|| { 53 | let mut decoder = Cursor::from(&encoder); 54 | for i in 0..BENCHMARK_SIZE { 55 | let num: u32 = decoder.read_var().unwrap(); 56 | assert_eq!(num, i); 57 | } 58 | }) 59 | }); 60 | } 61 | 62 | { 63 | codec_group.bench_function("lib0 encode var_uint (64 bit)", |b| { 64 | b.iter(|| { 65 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 66 | for i in 0..(BENCHMARK_SIZE as u64) { 67 | encoder.write_var(i); 68 | } 69 | }) 70 | }); 71 | codec_group.bench_function("lib0 decode var_uint (64 bit)", |b| { 72 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 73 | for i in 0..(BENCHMARK_SIZE as u64) { 74 | encoder.write_var(i); 75 | } 76 | 77 | b.iter(|| { 78 | let mut decoder = Cursor::from(&encoder); 79 | for i in 0..(BENCHMARK_SIZE as u64) { 80 | let num: u64 = decoder.read_var().unwrap(); 81 | assert_eq!(num, i); 82 | } 83 | }) 84 | }); 85 | } 86 | } 87 | 88 | criterion_group!(benches, codec); 89 | criterion_main!(benches); 90 | -------------------------------------------------------------------------------- /y-octo-utils/benches/map_ops_benchmarks.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use criterion::{criterion_group, criterion_main, Criterion}; 4 | 5 | fn operations(c: &mut Criterion) { 6 | let mut group = c.benchmark_group("ops/map"); 7 | group.measurement_time(Duration::from_secs(15)); 8 | 9 | group.bench_function("yrs/insert", |b| { 10 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" 11 | .split(' ') 12 | .collect::>(); 13 | 14 | b.iter(|| { 15 | use yrs::*; 16 | let doc = Doc::new(); 17 | let map = doc.get_or_insert_map("test"); 18 | 19 | let mut trx = doc.transact_mut(); 20 | for (idx, key) in base_text.iter().enumerate() { 21 | map.insert(&mut trx, key.to_string(), idx as f64).unwrap(); 22 | } 23 | 24 | drop(trx); 25 | }); 26 | }); 27 | 28 | group.bench_function("yrs/get", |b| { 29 | use yrs::*; 30 | 31 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" 32 | .split(' ') 33 | .collect::>(); 34 | 35 | let doc = Doc::new(); 36 | let map = doc.get_or_insert_map("test"); 37 | 38 | let mut trx = doc.transact_mut(); 39 | for (idx, key) in base_text.iter().enumerate() { 40 | map.insert(&mut trx, key.to_string(), idx as f64).unwrap(); 41 | } 42 | drop(trx); 43 | 44 | b.iter(|| { 45 | let trx = doc.transact(); 46 | for key in &base_text { 47 | map.get(&trx, key).unwrap(); 48 | } 49 | }); 50 | }); 51 | 52 | group.bench_function("yrs/remove", |b| { 53 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" 54 | .split(' ') 55 | .collect::>(); 56 | 57 | b.iter(|| { 58 | use yrs::*; 59 | let doc = Doc::new(); 60 | let map = doc.get_or_insert_map("test"); 61 | 62 | let mut trx = doc.transact_mut(); 63 | for (idx, key) in base_text.iter().enumerate() { 64 | map.insert(&mut trx, key.to_string(), idx as f64).unwrap(); 65 | } 66 | 67 | for key in &base_text { 68 | map.remove(&mut trx, key).unwrap(); 69 | } 70 | 71 | drop(trx); 72 | }); 73 | }); 74 | 75 | group.finish(); 76 | } 77 | 78 | criterion_group!(benches, operations); 79 | criterion_main!(benches); 80 | -------------------------------------------------------------------------------- /y-octo-utils/benches/text_ops_benchmarks.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use criterion::{criterion_group, criterion_main, Criterion}; 4 | use rand::{Rng, SeedableRng}; 5 | 6 | fn operations(c: &mut Criterion) { 7 | let mut group = c.benchmark_group("ops/text"); 8 | group.measurement_time(Duration::from_secs(15)); 9 | 10 | group.bench_function("yrs/insert", |b| { 11 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; 12 | let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); 13 | 14 | let idxs = (0..99) 15 | .map(|_| rng.gen_range(0..base_text.len() as u32)) 16 | .collect::>(); 17 | b.iter(|| { 18 | use yrs::*; 19 | let doc = Doc::new(); 20 | let text = doc.get_or_insert_text("test"); 21 | let mut trx = doc.transact_mut(); 22 | 23 | text.push(&mut trx, base_text).unwrap(); 24 | for idx in &idxs { 25 | text.insert(&mut trx, *idx, "test").unwrap(); 26 | } 27 | drop(trx); 28 | }); 29 | }); 30 | 31 | group.bench_function("yrs/remove", |b| { 32 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; 33 | 34 | b.iter(|| { 35 | use yrs::*; 36 | let doc = Doc::new(); 37 | let text = doc.get_or_insert_text("test"); 38 | let mut trx = doc.transact_mut(); 39 | 40 | text.push(&mut trx, base_text).unwrap(); 41 | text.push(&mut trx, base_text).unwrap(); 42 | text.push(&mut trx, base_text).unwrap(); 43 | for idx in (base_text.len() as u32)..0 { 44 | text.remove_range(&mut trx, idx, 1).unwrap(); 45 | } 46 | drop(trx); 47 | }); 48 | }); 49 | 50 | group.finish(); 51 | } 52 | 53 | criterion_group!(benches, operations); 54 | criterion_main!(benches); 55 | -------------------------------------------------------------------------------- /y-octo-utils/benches/update_benchmarks.rs: -------------------------------------------------------------------------------- 1 | mod utils; 2 | 3 | use std::time::Duration; 4 | 5 | use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; 6 | use path_ext::PathExt; 7 | use utils::Files; 8 | 9 | fn update(c: &mut Criterion) { 10 | let files = Files::load(); 11 | 12 | let mut group = c.benchmark_group("update"); 13 | group.measurement_time(Duration::from_secs(15)); 14 | 15 | for file in &files.files { 16 | group.throughput(Throughput::Bytes(file.content.len() as u64)); 17 | group.bench_with_input( 18 | BenchmarkId::new("parse with yrs", file.path.name_str()), 19 | &file.content, 20 | |b, content| { 21 | b.iter(|| { 22 | use yrs::{updates::decoder::Decode, Update}; 23 | Update::decode_v1(content).unwrap() 24 | }); 25 | }, 26 | ); 27 | } 28 | 29 | group.finish(); 30 | } 31 | 32 | criterion_group!(benches, update); 33 | criterion_main!(benches); 34 | -------------------------------------------------------------------------------- /y-octo-utils/benches/utils/files.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fs::{read, read_dir}, 3 | path::{Path, PathBuf}, 4 | }; 5 | 6 | use path_ext::PathExt; 7 | 8 | pub struct File { 9 | pub path: PathBuf, 10 | pub content: Vec, 11 | } 12 | 13 | const BASE: &str = "../y-octo/src/fixtures/"; 14 | 15 | impl File { 16 | fn new(path: &Path) -> Self { 17 | let content = read(path).unwrap(); 18 | Self { 19 | path: path.into(), 20 | content, 21 | } 22 | } 23 | } 24 | 25 | pub struct Files { 26 | pub files: Vec, 27 | } 28 | 29 | impl Files { 30 | pub fn load() -> Self { 31 | let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(BASE); 32 | 33 | let files = read_dir(path).unwrap(); 34 | let files = files 35 | .flatten() 36 | .filter(|f| f.path().is_file() && f.path().ext_str() == "bin") 37 | .map(|f| File::new(&f.path())) 38 | .collect::>(); 39 | 40 | Self { files } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /y-octo-utils/benches/utils/mod.rs: -------------------------------------------------------------------------------- 1 | mod files; 2 | 3 | pub use files::Files; 4 | -------------------------------------------------------------------------------- /y-octo-utils/bin/bench_result_render.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::HashMap, 3 | io::{self, BufRead}, 4 | }; 5 | 6 | fn process_duration(duration: &str) -> Option<(f64, f64)> { 7 | let dur_split: Vec = duration.split('±').map(String::from).collect(); 8 | if dur_split.len() != 2 { 9 | return None; 10 | } 11 | let units = dur_split[1] 12 | .chars() 13 | .skip_while(|c| c.is_ascii_digit()) 14 | .collect::(); 15 | let dur_secs = dur_split[0].parse::().ok()?; 16 | let error_secs = dur_split[1] 17 | .chars() 18 | .take_while(|c| c.is_ascii_digit()) 19 | .collect::() 20 | .parse::() 21 | .ok()?; 22 | Some(( 23 | convert_dur_to_seconds(dur_secs, &units), 24 | convert_dur_to_seconds(error_secs, &units), 25 | )) 26 | } 27 | 28 | fn convert_dur_to_seconds(dur: f64, units: &str) -> f64 { 29 | let factors: HashMap<_, _> = [ 30 | ("s", 1.0), 31 | ("ms", 1.0 / 1000.0), 32 | ("µs", 1.0 / 1_000_000.0), 33 | ("ns", 1.0 / 1_000_000_000.0), 34 | ] 35 | .iter() 36 | .cloned() 37 | .collect(); 38 | dur * factors.get(units).unwrap_or(&1.0) 39 | } 40 | 41 | fn is_significant(changes_dur: f64, changes_err: f64, base_dur: f64, base_err: f64) -> bool { 42 | if changes_dur < base_dur { 43 | changes_dur + changes_err < base_dur || base_dur - base_err > changes_dur 44 | } else { 45 | changes_dur - changes_err > base_dur || base_dur + base_err < changes_dur 46 | } 47 | } 48 | 49 | fn convert_to_markdown() -> impl Iterator { 50 | #[cfg(feature = "bench")] 51 | let re = regex::Regex::new(r"\s{2,}").unwrap(); 52 | io::stdin() 53 | .lock() 54 | .lines() 55 | .skip(2) 56 | .flat_map(move |row| { 57 | if let Ok(_row) = row { 58 | let columns = { 59 | #[cfg(feature = "bench")] 60 | { 61 | re.split(&_row).collect::>() 62 | } 63 | #[cfg(not(feature = "bench"))] 64 | Vec::<&str>::new() 65 | }; 66 | let name = columns.first()?; 67 | let base_duration = columns.get(2)?; 68 | let changes_duration = columns.get(5)?; 69 | Some(( 70 | name.to_string(), 71 | base_duration.to_string(), 72 | changes_duration.to_string(), 73 | )) 74 | } else { 75 | None 76 | } 77 | }) 78 | .flat_map(|(name, base_duration, changes_duration)| { 79 | let mut difference = "N/A".to_string(); 80 | let base_undefined = base_duration == "?"; 81 | let changes_undefined = changes_duration == "?"; 82 | 83 | if !base_undefined && !changes_undefined { 84 | let (base_dur_secs, base_err_secs) = process_duration(&base_duration)?; 85 | let (changes_dur_secs, changes_err_secs) = process_duration(&changes_duration)?; 86 | 87 | let diff = -(1.0 - changes_dur_secs / base_dur_secs) * 100.0; 88 | difference = format!("{:+.2}%", diff); 89 | 90 | if is_significant(changes_dur_secs, changes_err_secs, base_dur_secs, base_err_secs) { 91 | difference = format!("**{}**", difference); 92 | } 93 | } 94 | 95 | Some(format!( 96 | "| {} | {} | {} | {} |", 97 | name.replace('|', "\\|"), 98 | if base_undefined { "N/A" } else { &base_duration }, 99 | if changes_undefined { "N/A" } else { &changes_duration }, 100 | difference 101 | )) 102 | }) 103 | } 104 | 105 | fn main() { 106 | let platform = std::env::args().nth(1).expect("Missing platform argument"); 107 | 108 | let headers = vec![ 109 | format!("## Benchmark for {}", platform), 110 | "
".to_string(), 111 | " Click to view benchmark".to_string(), 112 | "".to_string(), 113 | "| Test | Base | PR | % |".to_string(), 114 | "| --- | --- | --- | --- |".to_string(), 115 | ]; 116 | 117 | for line in headers.into_iter().chain(convert_to_markdown()) { 118 | println!("{}", line); 119 | } 120 | println!("
"); 121 | } 122 | -------------------------------------------------------------------------------- /y-octo-utils/bin/doc_merger.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fs::read, 3 | io::{Error, ErrorKind}, 4 | path::PathBuf, 5 | time::Instant, 6 | }; 7 | 8 | use clap::Parser; 9 | use y_octo::Doc; 10 | 11 | /// ybinary merger 12 | #[derive(Parser, Debug)] 13 | #[command(author, version, about, long_about = None)] 14 | struct Args { 15 | /// Path of the ybinary to read 16 | #[arg(short, long)] 17 | path: String, 18 | } 19 | 20 | fn load_path(path: &str) -> Result>, Error> { 21 | let path = PathBuf::from(path); 22 | if path.is_dir() { 23 | let mut updates = Vec::new(); 24 | let mut paths = path 25 | .read_dir()? 26 | .filter_map(|entry| { 27 | let entry = entry.ok()?; 28 | if entry.path().is_file() { 29 | Some(entry.path()) 30 | } else { 31 | None 32 | } 33 | }) 34 | .collect::>(); 35 | paths.sort(); 36 | 37 | for path in paths { 38 | println!("read {:?}", path); 39 | updates.push(read(path)?); 40 | } 41 | Ok(updates) 42 | } else if path.is_file() { 43 | Ok(vec![read(path)?]) 44 | } else { 45 | Err(Error::new(ErrorKind::NotFound, "not a file or directory")) 46 | } 47 | } 48 | 49 | fn main() { 50 | let args = Args::parse(); 51 | jwst_merge(&args.path); 52 | } 53 | 54 | fn jwst_merge(path: &str) { 55 | let updates = load_path(path).unwrap(); 56 | 57 | let mut doc = Doc::default(); 58 | for (i, update) in updates.iter().enumerate() { 59 | println!("apply update{i} {} bytes", update.len()); 60 | doc.apply_update_from_binary_v1(update.clone()).unwrap(); 61 | } 62 | 63 | println!("press enter to continue"); 64 | std::io::stdin().read_line(&mut String::new()).unwrap(); 65 | let ts = Instant::now(); 66 | let history = doc.history().parse_store(Default::default()); 67 | println!("history: {:?}", ts.elapsed()); 68 | for history in history.iter().take(100) { 69 | println!("history: {:?}", history); 70 | } 71 | 72 | doc.gc().unwrap(); 73 | 74 | let binary = { 75 | let binary = doc.encode_update_v1().unwrap(); 76 | 77 | println!("merged {} bytes", binary.len()); 78 | 79 | binary 80 | }; 81 | 82 | { 83 | let mut doc = Doc::default(); 84 | doc.apply_update_from_binary_v1(binary.clone()).unwrap(); 85 | let new_binary = doc.encode_update_v1().unwrap(); 86 | 87 | println!("re-encoded {} bytes", new_binary.len(),); 88 | }; 89 | } 90 | 91 | #[cfg(test)] 92 | mod tests { 93 | use super::*; 94 | 95 | #[test] 96 | #[ignore = "only for debug"] 97 | fn test_gc() { 98 | jwst_merge("/Users/ds/Downloads/out"); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /y-octo-utils/bin/memory_leak_test.rs: -------------------------------------------------------------------------------- 1 | use rand::{Rng, SeedableRng}; 2 | use rand_chacha::ChaCha20Rng; 3 | use y_octo::*; 4 | 5 | fn run_text_test(seed: u64) { 6 | let doc = Doc::with_client(1); 7 | let mut rand = ChaCha20Rng::seed_from_u64(seed); 8 | let mut text = doc.get_or_create_text("test").unwrap(); 9 | text.insert(0, "This is a string with length 32.").unwrap(); 10 | 11 | let iteration = 20; 12 | let mut len = 32; 13 | 14 | for i in 0..iteration { 15 | let mut text = text.clone(); 16 | let ins = i % 2 == 0; 17 | let pos = rand.gen_range(0..if ins { text.len() } else { len / 2 }); 18 | if ins { 19 | let str = format!("hello {i}"); 20 | text.insert(pos, &str).unwrap(); 21 | len += str.len() as u64; 22 | } else { 23 | text.remove(pos, 6).unwrap(); 24 | len -= 6; 25 | } 26 | } 27 | 28 | assert_eq!(text.to_string().len(), len as usize); 29 | assert_eq!(text.len(), len); 30 | } 31 | 32 | fn run_array_test(seed: u64) { 33 | let doc = Doc::with_client(1); 34 | let mut rand = ChaCha20Rng::seed_from_u64(seed); 35 | let mut array = doc.get_or_create_array("test").unwrap(); 36 | array.push(1).unwrap(); 37 | 38 | let iteration = 20; 39 | let mut len = 1; 40 | 41 | for i in 0..iteration { 42 | let mut array = array.clone(); 43 | let ins = i % 2 == 0; 44 | let pos = rand.gen_range(0..if ins { array.len() } else { len / 2 }); 45 | if ins { 46 | array.insert(pos, 1).unwrap(); 47 | len += 1; 48 | } else { 49 | array.remove(pos, 1).unwrap(); 50 | len -= 1; 51 | } 52 | } 53 | 54 | assert_eq!(array.len(), len); 55 | } 56 | 57 | fn run_map_test() { 58 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" 59 | .split(' ') 60 | .collect::>(); 61 | 62 | for _ in 0..10000 { 63 | let doc = Doc::default(); 64 | let mut map = doc.get_or_create_map("test").unwrap(); 65 | for (idx, key) in base_text.iter().enumerate() { 66 | map.insert(key.to_string(), idx).unwrap(); 67 | } 68 | } 69 | } 70 | 71 | fn main() { 72 | let mut rand = ChaCha20Rng::seed_from_u64(rand::thread_rng().gen()); 73 | for _ in 0..10000 { 74 | let seed = rand.gen(); 75 | run_array_test(seed); 76 | run_text_test(seed); 77 | run_map_test(); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | corpus 3 | artifacts 4 | coverage 5 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | edition = "2021" 3 | name = "y-octo-fuzz" 4 | publish = false 5 | version = "0.0.0" 6 | 7 | [package.metadata] 8 | cargo-fuzz = true 9 | 10 | [dependencies] 11 | lib0 = "=0.16.5" 12 | libfuzzer-sys = "0.4" 13 | rand = "0.8" 14 | rand_chacha = "0.3" 15 | yrs = "=0.16.5" 16 | 17 | y-octo-utils = { path = "..", features = ["fuzz"] } 18 | 19 | [dependencies.y-octo] 20 | path = "../../y-octo" 21 | 22 | # Prevent this from interfering with workspaces 23 | [workspace] 24 | members = ["."] 25 | 26 | [profile.release] 27 | debug = 1 28 | 29 | [[bin]] 30 | doc = false 31 | name = "codec_doc_any_struct" 32 | path = "fuzz_targets/codec_doc_any_struct.rs" 33 | test = false 34 | 35 | [[bin]] 36 | doc = false 37 | name = "codec_doc_any" 38 | path = "fuzz_targets/codec_doc_any.rs" 39 | test = false 40 | 41 | [[bin]] 42 | doc = false 43 | name = "decode_bytes" 44 | path = "fuzz_targets/decode_bytes.rs" 45 | test = false 46 | 47 | [[bin]] 48 | doc = false 49 | name = "ins_del_text" 50 | path = "fuzz_targets/ins_del_text.rs" 51 | test = false 52 | 53 | [[bin]] 54 | doc = false 55 | name = "sync_message" 56 | path = "fuzz_targets/sync_message.rs" 57 | test = false 58 | 59 | [[bin]] 60 | doc = false 61 | name = "i32_decode" 62 | path = "fuzz_targets/i32_decode.rs" 63 | test = false 64 | 65 | [[bin]] 66 | doc = false 67 | name = "i32_encode" 68 | path = "fuzz_targets/i32_encode.rs" 69 | test = false 70 | 71 | [[bin]] 72 | doc = false 73 | name = "u64_decode" 74 | path = "fuzz_targets/u64_decode.rs" 75 | test = false 76 | 77 | 78 | [[bin]] 79 | doc = false 80 | name = "u64_encode" 81 | path = "fuzz_targets/u64_encode.rs" 82 | test = false 83 | 84 | [[bin]] 85 | doc = false 86 | name = "apply_update" 87 | path = "fuzz_targets/apply_update.rs" 88 | test = false 89 | 90 | [patch.crates-io] 91 | lib0 = { git = "https://github.com/toeverything/y-crdt", rev = "a700f09" } 92 | y-sync = { git = "https://github.com/toeverything/y-sync", rev = "5626851" } 93 | yrs = { git = "https://github.com/toeverything/y-crdt", rev = "a700f09" } 94 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/fuzz_targets/apply_update.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use std::collections::HashSet; 4 | 5 | use libfuzzer_sys::fuzz_target; 6 | use y_octo_utils::{ 7 | gen_nest_type_from_nest_type, gen_nest_type_from_root, CRDTParam, ManipulateSource, OpType, OpsRegistry, 8 | YrsNestType, 9 | }; 10 | use yrs::Transact; 11 | 12 | fuzz_target!(|crdt_params: Vec| { 13 | let mut doc = yrs::Doc::new(); 14 | let mut cur_crdt_nest_type: Option = None; 15 | let ops_registry = OpsRegistry::new(); 16 | let mut key_set = HashSet::::new(); 17 | for crdt_param in crdt_params { 18 | if key_set.contains(&crdt_param.key) { 19 | continue; 20 | } 21 | 22 | key_set.insert(crdt_param.key.clone()); 23 | match crdt_param.op_type { 24 | OpType::HandleCurrent => { 25 | if cur_crdt_nest_type.is_some() { 26 | ops_registry.operate_yrs_nest_type(&doc, cur_crdt_nest_type.clone().unwrap(), crdt_param); 27 | } 28 | } 29 | OpType::CreateCRDTNestType => { 30 | cur_crdt_nest_type = match cur_crdt_nest_type { 31 | None => gen_nest_type_from_root(&mut doc, &crdt_param), 32 | Some(mut nest_type) => match crdt_param.manipulate_source { 33 | ManipulateSource::CurrentNestType => Some(nest_type), 34 | ManipulateSource::NewNestTypeFromYDocRoot => gen_nest_type_from_root(&mut doc, &crdt_param), 35 | ManipulateSource::NewNestTypeFromCurrent => { 36 | gen_nest_type_from_nest_type(&mut doc, crdt_param.clone(), &mut nest_type) 37 | } 38 | }, 39 | }; 40 | } 41 | }; 42 | } 43 | 44 | let trx = doc.transact_mut(); 45 | let binary_from_yrs = trx.encode_update_v1().unwrap(); 46 | let doc = y_octo::Doc::try_from_binary_v1(&binary_from_yrs).unwrap(); 47 | let binary = doc.encode_update_v1().unwrap(); 48 | assert_eq!(binary, binary_from_yrs); 49 | }); 50 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/fuzz_targets/codec_doc_any.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use libfuzzer_sys::fuzz_target; 4 | use y_octo::{Any, CrdtRead, CrdtWrite, RawDecoder, RawEncoder}; 5 | 6 | fuzz_target!(|data: &[u8]| { 7 | if let Ok(any) = Any::read(&mut RawDecoder::new(data)) { 8 | // ensure decoding and re-encoding results has same result 9 | let mut buffer = RawEncoder::default(); 10 | if let Err(e) = any.write(&mut buffer) { 11 | panic!("Failed to write message: {:?}, {:?}", any, e); 12 | } 13 | if let Ok(any2) = Any::read(&mut RawDecoder::new(&buffer.into_inner())) { 14 | assert_eq!(any, any2); 15 | } 16 | } 17 | }); 18 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/fuzz_targets/codec_doc_any_struct.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use libfuzzer_sys::fuzz_target; 4 | use rand::{distributions::Alphanumeric, Rng}; 5 | use y_octo::{Any, CrdtRead, CrdtWrite, RawDecoder, RawEncoder}; 6 | 7 | fn get_random_string() -> String { 8 | rand::thread_rng() 9 | .sample_iter(&Alphanumeric) 10 | .take(7) 11 | .map(char::from) 12 | .collect() 13 | } 14 | 15 | fuzz_target!(|data: Vec| { 16 | { 17 | let any = Any::Object(data.iter().map(|a| (get_random_string(), a.clone())).collect()); 18 | 19 | let mut buffer = RawEncoder::default(); 20 | if let Err(e) = any.write(&mut buffer) { 21 | panic!("Failed to write message: {:?}, {:?}", any, e); 22 | } 23 | if let Ok(any2) = Any::read(&mut RawDecoder::new(&buffer.into_inner())) { 24 | assert_eq!(any, any2); 25 | } 26 | } 27 | 28 | { 29 | let any = Any::Array(data); 30 | let mut buffer = RawEncoder::default(); 31 | if let Err(e) = any.write(&mut buffer) { 32 | panic!("Failed to write message: {:?}, {:?}", any, e); 33 | } 34 | if let Ok(any2) = Any::read(&mut RawDecoder::new(&buffer.into_inner())) { 35 | assert_eq!(any, any2); 36 | } 37 | } 38 | }); 39 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/fuzz_targets/decode_bytes.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use libfuzzer_sys::fuzz_target; 4 | use y_octo::{read_var_buffer, read_var_i32, read_var_string, read_var_u64}; 5 | 6 | fuzz_target!(|data: Vec| { 7 | let _ = read_var_i32(&data); 8 | let _ = read_var_u64(&data); 9 | let _ = read_var_buffer(&data); 10 | let _ = read_var_string(&data); 11 | }); 12 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/fuzz_targets/i32_decode.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use lib0::encoding::Write; 4 | use libfuzzer_sys::fuzz_target; 5 | use y_octo::{read_var_i32, write_var_i32}; 6 | 7 | fuzz_target!(|data: Vec| { 8 | for i in data { 9 | let mut buf1 = Vec::new(); 10 | write_var_i32(&mut buf1, i).unwrap(); 11 | 12 | let mut buf2 = Vec::new(); 13 | buf2.write_var(i); 14 | 15 | assert_eq!(read_var_i32(&buf1).unwrap().1, i); 16 | assert_eq!(read_var_i32(&buf2).unwrap().1, i); 17 | } 18 | }); 19 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/fuzz_targets/i32_encode.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use libfuzzer_sys::fuzz_target; 4 | use y_octo::write_var_i32; 5 | 6 | fuzz_target!(|data: Vec| { 7 | use lib0::encoding::Write; 8 | 9 | for i in data { 10 | let mut buf1 = Vec::new(); 11 | write_var_i32(&mut buf1, i).unwrap(); 12 | let mut buf2 = Vec::new(); 13 | buf2.write_var(i); 14 | 15 | assert_eq!(buf1, buf2); 16 | } 17 | }); 18 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/fuzz_targets/ins_del_text.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use libfuzzer_sys::fuzz_target; 4 | use rand::{Rng, SeedableRng}; 5 | use rand_chacha::ChaCha20Rng; 6 | use y_octo::*; 7 | 8 | fuzz_target!(|seed: u64| { 9 | // println!("seed: {}", seed); 10 | let doc = Doc::with_client(1); 11 | let mut rand = ChaCha20Rng::seed_from_u64(seed); 12 | let mut text = doc.get_or_create_text("test").unwrap(); 13 | text.insert(0, "This is a string with length 32.").unwrap(); 14 | 15 | let iteration = 20; 16 | let mut len = 32; 17 | 18 | for i in 0..iteration { 19 | let mut text = text.clone(); 20 | let ins = i % 2 == 0; 21 | let pos = rand.gen_range(0..if ins { text.len() } else { len / 2 }); 22 | if ins { 23 | let str = format!("hello {i}"); 24 | text.insert(pos, &str).unwrap(); 25 | len += str.len() as u64; 26 | } else { 27 | text.remove(pos, 6).unwrap(); 28 | len -= 6; 29 | } 30 | } 31 | 32 | assert_eq!(text.to_string().len(), len as usize); 33 | assert_eq!(text.len(), len); 34 | }); 35 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/fuzz_targets/sync_message.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use libfuzzer_sys::fuzz_target; 4 | use y_octo::{read_sync_message, write_sync_message}; 5 | 6 | fuzz_target!(|data: &[u8]| { 7 | let result = read_sync_message(data); 8 | 9 | if let Ok((_, msg)) = result { 10 | // ensure decoding and re-encoding results has same result 11 | let mut buffer = Vec::new(); 12 | if let Err(e) = write_sync_message(&mut buffer, &msg) { 13 | panic!("Failed to write message: {:?}, {:?}", msg, e); 14 | } 15 | let result = read_sync_message(&buffer); 16 | if let Ok((_, msg2)) = result { 17 | assert_eq!(msg, msg2); 18 | } 19 | } 20 | }); 21 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/fuzz_targets/u64_decode.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use lib0::encoding::Write; 4 | use libfuzzer_sys::fuzz_target; 5 | use y_octo::{read_var_u64, write_var_u64}; 6 | 7 | fuzz_target!(|data: Vec| { 8 | for i in data { 9 | let mut buf1 = Vec::new(); 10 | write_var_u64(&mut buf1, i).unwrap(); 11 | 12 | let mut buf2 = Vec::new(); 13 | buf2.write_var(i); 14 | 15 | assert_eq!(read_var_u64(&buf1).unwrap().1, i); 16 | assert_eq!(read_var_u64(&buf2).unwrap().1, i); 17 | } 18 | }); 19 | -------------------------------------------------------------------------------- /y-octo-utils/fuzz/fuzz_targets/u64_encode.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use lib0::encoding::Write; 4 | use libfuzzer_sys::fuzz_target; 5 | use y_octo::write_var_u64; 6 | 7 | fuzz_target!(|data: Vec| { 8 | for i in data { 9 | let mut buf1 = Vec::new(); 10 | buf1.write_var(i); 11 | 12 | let mut buf2 = Vec::new(); 13 | write_var_u64(&mut buf2, i).unwrap(); 14 | 15 | assert_eq!(buf1, buf2); 16 | } 17 | }); 18 | -------------------------------------------------------------------------------- /y-octo-utils/src/codec.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | #[cfg(test)] 4 | mod tests { 5 | 6 | use super::*; 7 | 8 | use lib0::encoding::Write; 9 | 10 | fn test_var_uint_enc_dec(num: u64) { 11 | let mut buf1 = Vec::new(); 12 | write_var_u64(&mut buf1, num).unwrap(); 13 | 14 | let mut buf2 = Vec::new(); 15 | buf2.write_var(num); 16 | 17 | { 18 | let (rest, decoded_num) = read_var_u64(&buf1).unwrap(); 19 | assert_eq!(num, decoded_num); 20 | assert_eq!(rest.len(), 0); 21 | } 22 | 23 | { 24 | let (rest, decoded_num) = read_var_u64(&buf2).unwrap(); 25 | assert_eq!(num, decoded_num); 26 | assert_eq!(rest.len(), 0); 27 | } 28 | } 29 | 30 | fn test_var_int_enc_dec(num: i32) { 31 | { 32 | let mut buf1: Vec = Vec::new(); 33 | write_var_i32(&mut buf1, num).unwrap(); 34 | 35 | let (rest, decoded_num) = read_var_i32(&buf1).unwrap(); 36 | assert_eq!(num, decoded_num); 37 | assert_eq!(rest.len(), 0); 38 | } 39 | 40 | { 41 | let mut buf2 = Vec::new(); 42 | buf2.write_var(num); 43 | 44 | let (rest, decoded_num) = read_var_i32(&buf2).unwrap(); 45 | assert_eq!(num, decoded_num); 46 | assert_eq!(rest.len(), 0); 47 | } 48 | } 49 | 50 | #[test] 51 | fn test_var_uint_codec() { 52 | test_var_uint_enc_dec(0); 53 | test_var_uint_enc_dec(1); 54 | test_var_uint_enc_dec(127); 55 | test_var_uint_enc_dec(0b1000_0000); 56 | test_var_uint_enc_dec(0b1_0000_0000); 57 | test_var_uint_enc_dec(0b1_1111_1111); 58 | test_var_uint_enc_dec(0b10_0000_0000); 59 | test_var_uint_enc_dec(0b11_1111_1111); 60 | test_var_uint_enc_dec(0x7fff_ffff_ffff_ffff); 61 | test_var_uint_enc_dec(u64::max_value()); 62 | } 63 | 64 | #[test] 65 | fn test_var_int() { 66 | test_var_int_enc_dec(0); 67 | test_var_int_enc_dec(1); 68 | test_var_int_enc_dec(-1); 69 | test_var_int_enc_dec(63); 70 | test_var_int_enc_dec(-63); 71 | test_var_int_enc_dec(64); 72 | test_var_int_enc_dec(-64); 73 | test_var_int_enc_dec(i32::MAX); 74 | test_var_int_enc_dec(i32::MIN); 75 | test_var_int_enc_dec(((1 << 20) - 1) * 8); 76 | test_var_int_enc_dec(-((1 << 20) - 1) * 8); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /y-octo-utils/src/doc.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod tests { 3 | use y_octo::Doc; 4 | use yrs::{Map, Transact}; 5 | 6 | #[test] 7 | fn test_basic_yrs_binary_compatibility() { 8 | let yrs_doc = yrs::Doc::new(); 9 | 10 | let map = yrs_doc.get_or_insert_map("abc"); 11 | let mut trx = yrs_doc.transact_mut(); 12 | map.insert(&mut trx, "a", 1).unwrap(); 13 | 14 | let binary_from_yrs = trx.encode_update_v1().unwrap(); 15 | 16 | let doc = Doc::try_from_binary_v1(&binary_from_yrs).unwrap(); 17 | let binary = doc.encode_update_v1().unwrap(); 18 | 19 | assert_eq!(binary_from_yrs, binary); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /y-octo-utils/src/doc_operation/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod types; 2 | pub mod yrs_op; 3 | 4 | pub use types::*; 5 | pub use yrs_op::*; 6 | -------------------------------------------------------------------------------- /y-octo-utils/src/doc_operation/types.rs: -------------------------------------------------------------------------------- 1 | use yrs::{ArrayRef, MapRef, TextRef, XmlElementRef, XmlFragmentRef, XmlTextRef}; 2 | 3 | pub const NEST_DATA_INSERT: &str = "insert"; 4 | pub const NEST_DATA_DELETE: &str = "delete"; 5 | pub const NEST_DATA_CLEAR: &str = "clear"; 6 | 7 | #[derive(Hash, PartialEq, Eq, Clone, Debug, arbitrary::Arbitrary)] 8 | pub enum OpType { 9 | HandleCurrent, 10 | CreateCRDTNestType, 11 | } 12 | 13 | #[derive(Hash, PartialEq, Eq, Clone, Debug, arbitrary::Arbitrary)] 14 | pub enum NestDataOpType { 15 | Insert, 16 | Delete, 17 | Clear, 18 | } 19 | 20 | #[derive(PartialEq, Clone, Debug, arbitrary::Arbitrary)] 21 | pub struct CRDTParam { 22 | pub op_type: OpType, 23 | pub new_nest_type: CRDTNestType, 24 | pub manipulate_source: ManipulateSource, 25 | pub insert_pos: InsertPos, 26 | pub key: String, 27 | pub value: String, 28 | pub nest_data_op_type: NestDataOpType, 29 | } 30 | 31 | #[derive(Debug, Clone, PartialEq, Eq, Hash, arbitrary::Arbitrary)] 32 | pub enum CRDTNestType { 33 | Array, 34 | Map, 35 | Text, 36 | XMLElement, 37 | XMLFragment, 38 | XMLText, 39 | } 40 | 41 | #[derive(Debug, Clone, PartialEq, arbitrary::Arbitrary)] 42 | pub enum ManipulateSource { 43 | NewNestTypeFromYDocRoot, 44 | CurrentNestType, 45 | NewNestTypeFromCurrent, 46 | } 47 | 48 | #[derive(Debug, Clone, PartialEq, arbitrary::Arbitrary)] 49 | pub enum InsertPos { 50 | BEGIN, 51 | MID, 52 | END, 53 | } 54 | 55 | #[derive(Clone)] 56 | pub enum YrsNestType { 57 | ArrayType(ArrayRef), 58 | MapType(MapRef), 59 | TextType(TextRef), 60 | XMLElementType(XmlElementRef), 61 | XMLFragmentType(XmlFragmentRef), 62 | XMLTextType(XmlTextRef), 63 | } 64 | -------------------------------------------------------------------------------- /y-octo-utils/src/doc_operation/yrs_op/array.rs: -------------------------------------------------------------------------------- 1 | use phf::phf_map; 2 | 3 | use super::*; 4 | 5 | fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { 6 | let array = match nest_input { 7 | YrsNestType::ArrayType(array) => array, 8 | _ => unreachable!(), 9 | }; 10 | let mut trx = doc.transact_mut(); 11 | let len = array.len(&trx); 12 | let index = random_pick_num(len, ¶ms.insert_pos); 13 | array.insert(&mut trx, index, params.value).unwrap(); 14 | } 15 | 16 | fn delete_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { 17 | let array = match nest_input { 18 | YrsNestType::ArrayType(array) => array, 19 | _ => unreachable!(), 20 | }; 21 | let mut trx = doc.transact_mut(); 22 | let len = array.len(&trx); 23 | if len >= 1 { 24 | let index = random_pick_num(len - 1, ¶ms.insert_pos); 25 | array.remove(&mut trx, index).unwrap(); 26 | } 27 | } 28 | 29 | fn clear_op(doc: &yrs::Doc, nest_input: &YrsNestType, _params: CRDTParam) { 30 | let array = match nest_input { 31 | YrsNestType::ArrayType(array) => array, 32 | _ => unreachable!(), 33 | }; 34 | let mut trx = doc.transact_mut(); 35 | let len = array.len(&trx); 36 | for _ in 0..len { 37 | array.remove(&mut trx, 0).unwrap(); 38 | } 39 | } 40 | 41 | pub static ARRAY_OPS: TestOps = phf_map! { 42 | "insert" => insert_op, 43 | "delete" => delete_op, 44 | "clear" => clear_op, 45 | }; 46 | 47 | pub fn yrs_create_array_from_nest_type( 48 | doc: &yrs::Doc, 49 | current: &mut YrsNestType, 50 | insert_pos: &InsertPos, 51 | key: String, 52 | ) -> Option { 53 | let cal_index = |len: u32| -> u32 { 54 | match insert_pos { 55 | InsertPos::BEGIN => 0, 56 | InsertPos::MID => len / 2, 57 | InsertPos::END => len, 58 | } 59 | }; 60 | let mut trx = doc.transact_mut(); 61 | let array_prelim = ArrayPrelim::default(); 62 | match current { 63 | YrsNestType::ArrayType(array) => { 64 | let index = cal_index(array.len(&trx)); 65 | Some(array.insert(&mut trx, index, array_prelim).unwrap()) 66 | } 67 | YrsNestType::MapType(map) => Some(map.insert(&mut trx, key, array_prelim).unwrap()), 68 | YrsNestType::TextType(text) => { 69 | let str = text.get_string(&trx); 70 | let len = str.chars().fold(0, |acc, _| acc + 1); 71 | let index = random_pick_num(len, insert_pos) as usize; 72 | let byte_start_offset = str.chars().take(index).fold(0, |acc, ch| acc + ch.len_utf8()); 73 | 74 | Some( 75 | text.insert_embed(&mut trx, byte_start_offset as u32, array_prelim) 76 | .unwrap(), 77 | ) 78 | } 79 | YrsNestType::XMLTextType(xml_text) => { 80 | let str = xml_text.get_string(&trx); 81 | let len = str.chars().fold(0, |acc, _| acc + 1); 82 | let index = random_pick_num(len, insert_pos) as usize; 83 | let byte_start_offset = str.chars().take(index).fold(0, |acc, ch| acc + ch.len_utf8()); 84 | 85 | Some( 86 | xml_text 87 | .insert_embed(&mut trx, byte_start_offset as u32, array_prelim) 88 | .unwrap(), 89 | ) 90 | } 91 | _ => None, 92 | } 93 | } 94 | 95 | #[cfg(test)] 96 | mod tests { 97 | use yrs::Doc; 98 | 99 | use super::*; 100 | 101 | #[test] 102 | fn test_gen_array_ref_ops() { 103 | let doc = Doc::new(); 104 | let array_ref = doc.get_or_insert_array("test_array"); 105 | 106 | let ops_registry = OpsRegistry::new(); 107 | 108 | let mut params = CRDTParam { 109 | op_type: OpType::CreateCRDTNestType, 110 | new_nest_type: CRDTNestType::Array, 111 | manipulate_source: ManipulateSource::NewNestTypeFromYDocRoot, 112 | insert_pos: InsertPos::BEGIN, 113 | key: String::from("test_key"), 114 | value: String::from("test_value"), 115 | nest_data_op_type: NestDataOpType::Insert, 116 | }; 117 | 118 | ops_registry.operate_yrs_nest_type(&doc, YrsNestType::ArrayType(array_ref.clone()), params.clone()); 119 | assert_eq!(array_ref.len(&doc.transact()), 1); 120 | params.nest_data_op_type = NestDataOpType::Delete; 121 | ops_registry.operate_yrs_nest_type(&doc, YrsNestType::ArrayType(array_ref.clone()), params.clone()); 122 | assert_eq!(array_ref.len(&doc.transact()), 0); 123 | 124 | params.nest_data_op_type = NestDataOpType::Clear; 125 | ops_registry.operate_yrs_nest_type(&doc, YrsNestType::ArrayType(array_ref.clone()), params.clone()); 126 | assert_eq!(array_ref.len(&doc.transact()), 0); 127 | } 128 | 129 | #[test] 130 | fn test_yrs_create_array_from_nest_type() { 131 | let doc = Doc::new(); 132 | let array_ref = doc.get_or_insert_array("test_array"); 133 | let key = String::from("test_key"); 134 | 135 | let new_array_ref = yrs_create_array_from_nest_type( 136 | &doc, 137 | &mut YrsNestType::ArrayType(array_ref.clone()), 138 | &InsertPos::BEGIN, 139 | key.clone(), 140 | ); 141 | assert!(new_array_ref.is_some()); 142 | 143 | let map_ref = doc.get_or_insert_map("test_map"); 144 | let new_array_ref = yrs_create_array_from_nest_type( 145 | &doc, 146 | &mut YrsNestType::MapType(map_ref.clone()), 147 | &InsertPos::BEGIN, 148 | key.clone(), 149 | ); 150 | assert!(new_array_ref.is_some()); 151 | 152 | let text_ref = doc.get_or_insert_text("test_text"); 153 | let new_array_ref = yrs_create_array_from_nest_type( 154 | &doc, 155 | &mut YrsNestType::TextType(text_ref.clone()), 156 | &InsertPos::BEGIN, 157 | key.clone(), 158 | ); 159 | assert!(new_array_ref.is_some()); 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /y-octo-utils/src/doc_operation/yrs_op/map.rs: -------------------------------------------------------------------------------- 1 | use phf::phf_map; 2 | 3 | use super::*; 4 | 5 | fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { 6 | let map = match nest_input { 7 | YrsNestType::MapType(map) => map, 8 | _ => unreachable!(), 9 | }; 10 | let mut trx = doc.transact_mut(); 11 | map.insert(&mut trx, params.key, params.value).unwrap(); 12 | } 13 | 14 | fn remove_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { 15 | let map = match nest_input { 16 | YrsNestType::MapType(map) => map, 17 | _ => unreachable!(), 18 | }; 19 | let rand_key = { 20 | let trx = doc.transact_mut(); 21 | let mut iter = map.iter(&trx); 22 | let len = map.len(&trx) as usize; 23 | let skip_step = if len <= 1 { 24 | 0 25 | } else { 26 | random_pick_num((len - 1) as u32, ¶ms.insert_pos) 27 | }; 28 | 29 | iter.nth(skip_step as usize).map(|(key, _value)| key.to_string()) 30 | }; 31 | 32 | if let Some(key) = rand_key { 33 | let mut trx = doc.transact_mut(); 34 | map.remove(&mut trx, &key).unwrap(); 35 | } 36 | } 37 | 38 | fn clear_op(doc: &yrs::Doc, nest_input: &YrsNestType, _params: CRDTParam) { 39 | let map = match nest_input { 40 | YrsNestType::MapType(map) => map, 41 | _ => unreachable!(), 42 | }; 43 | let mut trx = doc.transact_mut(); 44 | map.clear(&mut trx); 45 | } 46 | 47 | pub static MAP_OPS: TestOps = phf_map! { 48 | "insert" => insert_op, 49 | "delete" => remove_op, 50 | "clear" => clear_op, 51 | }; 52 | 53 | pub fn yrs_create_map_from_nest_type( 54 | doc: &yrs::Doc, 55 | current: &mut YrsNestType, 56 | insert_pos: &InsertPos, 57 | key: String, 58 | ) -> Option { 59 | let cal_index = |len: u32| -> u32 { 60 | match insert_pos { 61 | InsertPos::BEGIN => 0, 62 | InsertPos::MID => len / 2, 63 | InsertPos::END => len, 64 | } 65 | }; 66 | let mut trx = doc.transact_mut(); 67 | let map_prelim = MapPrelim::::from(HashMap::new()); 68 | match current { 69 | YrsNestType::ArrayType(array) => { 70 | let index = cal_index(array.len(&trx)); 71 | Some(array.insert(&mut trx, index, map_prelim).unwrap()) 72 | } 73 | YrsNestType::MapType(map) => Some(map.insert(&mut trx, key, map_prelim).unwrap()), 74 | YrsNestType::TextType(text) => { 75 | let str = text.get_string(&trx); 76 | let len = str.chars().fold(0, |acc, _| acc + 1); 77 | let index = random_pick_num(len, insert_pos) as usize; 78 | let byte_start_offset = str.chars().take(index).fold(0, |acc, ch| acc + ch.len_utf8()); 79 | 80 | Some( 81 | text.insert_embed(&mut trx, byte_start_offset as u32, map_prelim) 82 | .unwrap(), 83 | ) 84 | } 85 | YrsNestType::XMLTextType(xml_text) => { 86 | let str = xml_text.get_string(&trx); 87 | let len = str.chars().fold(0, |acc, _| acc + 1); 88 | let index = random_pick_num(len, insert_pos) as usize; 89 | let byte_start_offset = str.chars().take(index).fold(0, |acc, ch| acc + ch.len_utf8()); 90 | 91 | Some( 92 | xml_text 93 | .insert_embed(&mut trx, byte_start_offset as u32, map_prelim) 94 | .unwrap(), 95 | ) 96 | } 97 | _ => None, 98 | } 99 | } 100 | 101 | #[cfg(test)] 102 | mod tests { 103 | use yrs::Doc; 104 | 105 | use super::*; 106 | 107 | #[test] 108 | fn test_gen_map_ref_ops() { 109 | let doc = Doc::new(); 110 | let map_ref = doc.get_or_insert_map("test_map"); 111 | 112 | let ops_registry = OpsRegistry::new(); 113 | 114 | let mut params = CRDTParam { 115 | op_type: OpType::CreateCRDTNestType, 116 | new_nest_type: CRDTNestType::Map, 117 | manipulate_source: ManipulateSource::NewNestTypeFromYDocRoot, 118 | insert_pos: InsertPos::BEGIN, 119 | key: String::from("test_key"), 120 | value: String::from("test_value"), 121 | nest_data_op_type: NestDataOpType::Insert, 122 | }; 123 | 124 | ops_registry.operate_yrs_nest_type(&doc, YrsNestType::MapType(map_ref.clone()), params.clone()); 125 | assert_eq!(map_ref.len(&doc.transact()), 1); 126 | params.nest_data_op_type = NestDataOpType::Delete; 127 | ops_registry.operate_yrs_nest_type(&doc, YrsNestType::MapType(map_ref.clone()), params.clone()); 128 | assert_eq!(map_ref.len(&doc.transact()), 0); 129 | 130 | params.nest_data_op_type = NestDataOpType::Clear; 131 | ops_registry.operate_yrs_nest_type(&doc, YrsNestType::MapType(map_ref.clone()), params.clone()); 132 | assert_eq!(map_ref.len(&doc.transact()), 0); 133 | } 134 | 135 | #[test] 136 | fn test_yrs_create_map_from_nest_type() { 137 | let doc = Doc::new(); 138 | let map_ref = doc.get_or_insert_map("test_map"); 139 | let key = String::from("test_key"); 140 | 141 | let new_map_ref = yrs_create_map_from_nest_type( 142 | &doc, 143 | &mut YrsNestType::MapType(map_ref.clone()), 144 | &InsertPos::BEGIN, 145 | key.clone(), 146 | ); 147 | assert!(new_map_ref.is_some()); 148 | 149 | let map_ref = doc.get_or_insert_map("test_map"); 150 | let new_map_ref = yrs_create_map_from_nest_type( 151 | &doc, 152 | &mut YrsNestType::MapType(map_ref.clone()), 153 | &InsertPos::BEGIN, 154 | key.clone(), 155 | ); 156 | assert!(new_map_ref.is_some()); 157 | 158 | let text_ref = doc.get_or_insert_text("test_text"); 159 | let new_map_ref = yrs_create_map_from_nest_type( 160 | &doc, 161 | &mut YrsNestType::TextType(text_ref.clone()), 162 | &InsertPos::BEGIN, 163 | key.clone(), 164 | ); 165 | assert!(new_map_ref.is_some()); 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /y-octo-utils/src/doc_operation/yrs_op/xml_element.rs: -------------------------------------------------------------------------------- 1 | use phf::phf_map; 2 | 3 | use super::*; 4 | 5 | fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { 6 | let xml_element = match nest_input { 7 | YrsNestType::XMLElementType(xml_element) => xml_element, 8 | _ => unreachable!(), 9 | }; 10 | let mut trx = doc.transact_mut(); 11 | let len = xml_element.len(&trx); 12 | let index = random_pick_num(len, ¶ms.insert_pos); 13 | xml_element 14 | .insert(&mut trx, index, XmlTextPrelim::new(params.value)) 15 | .unwrap(); 16 | } 17 | 18 | fn remove_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { 19 | let xml_element = match nest_input { 20 | YrsNestType::XMLElementType(xml_element) => xml_element, 21 | _ => unreachable!(), 22 | }; 23 | let mut trx = doc.transact_mut(); 24 | let len = xml_element.len(&trx); 25 | if len >= 1 { 26 | let index = random_pick_num(len - 1, ¶ms.insert_pos); 27 | xml_element.remove_range(&mut trx, index, 1).unwrap(); 28 | } 29 | } 30 | 31 | fn clear_op(doc: &yrs::Doc, nest_input: &YrsNestType, _params: CRDTParam) { 32 | let xml_element = match nest_input { 33 | YrsNestType::XMLElementType(xml_element) => xml_element, 34 | _ => unreachable!(), 35 | }; 36 | let mut trx = doc.transact_mut(); 37 | let len = xml_element.len(&trx); 38 | for _ in 0..len { 39 | xml_element.remove_range(&mut trx, 0, 1).unwrap(); 40 | } 41 | } 42 | 43 | pub static XML_ELEMENT_OPS: TestOps = phf_map! { 44 | "insert" => insert_op, 45 | "delete" => remove_op, 46 | "clear" => clear_op, 47 | }; 48 | -------------------------------------------------------------------------------- /y-octo-utils/src/doc_operation/yrs_op/xml_fragment.rs: -------------------------------------------------------------------------------- 1 | use phf::phf_map; 2 | 3 | use super::*; 4 | 5 | fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { 6 | let xml_fragment = match nest_input { 7 | YrsNestType::XMLFragmentType(xml_fragment) => xml_fragment, 8 | _ => unreachable!(), 9 | }; 10 | let mut trx = doc.transact_mut(); 11 | let len = xml_fragment.len(&trx); 12 | let index = random_pick_num(len, ¶ms.insert_pos); 13 | xml_fragment 14 | .insert(&mut trx, index, XmlTextPrelim::new(params.value)) 15 | .unwrap(); 16 | } 17 | 18 | fn remove_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { 19 | let xml_fragment = match nest_input { 20 | YrsNestType::XMLFragmentType(xml_fragment) => xml_fragment, 21 | _ => unreachable!(), 22 | }; 23 | let mut trx = doc.transact_mut(); 24 | let len = xml_fragment.len(&trx); 25 | if len >= 1 { 26 | let index = random_pick_num(len - 1, ¶ms.insert_pos); 27 | xml_fragment.remove_range(&mut trx, index, 1).unwrap(); 28 | } 29 | } 30 | 31 | fn clear_op(doc: &yrs::Doc, nest_input: &YrsNestType, _params: CRDTParam) { 32 | let xml_fragment = match nest_input { 33 | YrsNestType::XMLFragmentType(xml_fragment) => xml_fragment, 34 | _ => unreachable!(), 35 | }; 36 | let mut trx = doc.transact_mut(); 37 | let len = xml_fragment.len(&trx); 38 | for _ in 0..len { 39 | xml_fragment.remove_range(&mut trx, 0, 1).unwrap(); 40 | } 41 | } 42 | 43 | pub static XML_FRAGMENT_OPS: TestOps = phf_map! { 44 | "insert" => insert_op, 45 | "delete" => remove_op, 46 | "clear" => clear_op, 47 | }; 48 | -------------------------------------------------------------------------------- /y-octo-utils/src/doc_operation/yrs_op/xml_text.rs: -------------------------------------------------------------------------------- 1 | use phf::phf_map; 2 | 3 | use super::*; 4 | 5 | fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { 6 | let xml_text = match nest_input { 7 | YrsNestType::XMLTextType(xml_text) => xml_text, 8 | _ => unreachable!(), 9 | }; 10 | let mut trx = doc.transact_mut(); 11 | 12 | let str = xml_text.get_string(&trx); 13 | let len = str.chars().fold(0, |acc, _| acc + 1); 14 | let index = random_pick_num(len, ¶ms.insert_pos) as usize; 15 | let byte_start_offset = str.chars().take(index).fold(0, |acc, ch| acc + ch.len_utf8()); 16 | 17 | xml_text 18 | .insert(&mut trx, byte_start_offset as u32, ¶ms.value) 19 | .unwrap(); 20 | } 21 | 22 | fn remove_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { 23 | let xml_text = match nest_input { 24 | YrsNestType::XMLTextType(xml_text) => xml_text, 25 | _ => unreachable!(), 26 | }; 27 | let mut trx = doc.transact_mut(); 28 | 29 | let str = xml_text.get_string(&trx); 30 | let len = str.chars().fold(0, |acc, _| acc + 1); 31 | if len < 1 { 32 | return; 33 | } 34 | let index = random_pick_num(len - 1, ¶ms.insert_pos) as usize; 35 | let byte_start_offset = str.chars().take(index).fold(0, |acc, ch| acc + ch.len_utf8()); 36 | 37 | let char_byte_len = str.chars().nth(index).unwrap().len_utf8(); 38 | xml_text 39 | .remove_range(&mut trx, byte_start_offset as u32, char_byte_len as u32) 40 | .unwrap(); 41 | } 42 | 43 | fn clear_op(doc: &yrs::Doc, nest_input: &YrsNestType, _params: CRDTParam) { 44 | let xml_text = match nest_input { 45 | YrsNestType::XMLTextType(xml_text) => xml_text, 46 | _ => unreachable!(), 47 | }; 48 | let mut trx = doc.transact_mut(); 49 | 50 | let str = xml_text.get_string(&trx); 51 | let byte_len = str.chars().fold(0, |acc, ch| acc + ch.len_utf8()); 52 | 53 | xml_text.remove_range(&mut trx, 0, byte_len as u32).unwrap(); 54 | } 55 | 56 | pub static XML_TEXT_OPS: TestOps = phf_map! { 57 | "insert" => insert_op, 58 | "delete" => remove_op, 59 | "clear" => clear_op, 60 | }; 61 | -------------------------------------------------------------------------------- /y-octo-utils/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod doc; 2 | mod message; 3 | 4 | #[cfg(feature = "fuzz")] 5 | pub mod doc_operation; 6 | 7 | #[cfg(feature = "fuzz")] 8 | pub use doc_operation::*; 9 | pub use message::{to_sync_message, to_y_message}; 10 | -------------------------------------------------------------------------------- /y-octo-utils/yrs-is-unsafe/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | edition = "2021" 3 | name = "yrs-is-unsafe" 4 | version = "0.1.0" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | y-octo = { workspace = true } 10 | yrs = "=0.16.5" 11 | 12 | [[bin]] 13 | name = "yrs-mem" 14 | path = "./bin/mem_usage.rs" 15 | 16 | [[bin]] 17 | name = "yrs-global-lock" 18 | path = "./bin/global_lock.rs" 19 | -------------------------------------------------------------------------------- /y-octo-utils/yrs-is-unsafe/README.md: -------------------------------------------------------------------------------- 1 | # Why we're not using [yrs](https://crates.io/crates/yrs) 2 | 3 | ## Multi-threading safety 4 | 5 | One of the biggest reason why we are writing our own yjs compatible Rust implementation is [yrs](https://crates.io/crates/yrs) is not safe for multi-threading. 6 | 7 | You can run the following code to see the problem: 8 | 9 | ```bash 10 | cargo run --bin yrs-is-unsafe 11 | ``` 12 | 13 | The source codes is under the [yrs-is-unsafe](y-octo/yrs-is-unsafe/src/main.rs): 14 | 15 |
16 | main.rs 17 | 18 | ```rust 19 | use std::thread::spawn; 20 | 21 | use yrs::Doc; 22 | 23 | fn main() { 24 | let doc = Doc::new(); 25 | 26 | let t1 = { 27 | let doc = doc.clone(); 28 | spawn(move || { 29 | let _ = doc.get_or_insert_map("text"); 30 | }) 31 | }; 32 | 33 | let t2 = { 34 | let doc = doc.clone(); 35 | spawn(move || { 36 | let _ = doc.get_or_insert_map("text"); 37 | }) 38 | }; 39 | 40 | t1.join().unwrap(); 41 | t2.join().unwrap(); 42 | } 43 | ``` 44 | 45 |
46 | 47 | We are facing this problem in our Rust server, and the iOS/Android clients which are running with the multi-threading runtime. 48 | 49 | Adding a global lock is not a good solution in this case. First the performance will be bad, and deadlocks will come after that. In general the deadlocks in application level are more difficult to debug than in the library level. 50 | Second, we assume the Rust compiler can guarantee the multi-threading safety, and with [yrs](https://crates.io/crates/yrs) we must to guarantee the safety by ourselves, and it often leads to bugs. 51 | 52 | ## Memory efficiency 53 | 54 | In the previous versions of [Mysc](https://www.mysc.app/) mobile apps, there are always oom issues happened. 55 | You can run the following command to see the problem: 56 | 57 | ```bash 58 | cargo build --release --bin yrs-mem 59 | /usr/bin/time -l ./target/release/yrs-mem 60 | ``` 61 | 62 | [The source codes](../yrs-is-unsafe/bin/mem_usage.rs): 63 | 64 |
65 | mem_usage.rs 66 | 67 | ```rust 68 | use yrs::{updates::decoder::Decode, Update}; 69 | 70 | fn main() { 71 | if let Ok(_) = Update::decode_v1(&[255, 255, 255, 122]) {}; 72 | } 73 | ``` 74 | 75 |
76 | 77 | On my MacBook pro, the results is like that: 78 | 79 | ```text 80 | .05 real 0.01 user 0.04 sys 81 | 538050560 maximum resident set size 82 | 0 average shared memory size 83 | 0 average unshared data size 84 | 0 average unshared stack size 85 | 32959 page reclaims 86 | 1 page faults 87 | 0 swaps 88 | 0 block input operations 89 | 0 block output operations 90 | 0 messages sent 91 | 0 messages received 92 | 0 signals received 93 | 0 voluntary context switches 94 | 5 involuntary context switches 95 | 298179580 instructions retired 96 | 166031219 cycles elapsed 97 | 538003008 peak memory footprint 98 | ``` 99 | 100 | There is `538050560` bytes memory used, which is about **538MB**. It's too bad for mobile apps or the similar low memory devices. 101 | 102 | ## Panic everywhere 103 | 104 | Unlike most of the Rust libraries, [yrs](https://crates.io/crates/yrs) [panics everywhere](https://github.com/search?q=repo%3Ay-crdt%2Fy-crdt+panic+language%3ARust&type=code&l=Rust), rather than returning the `Result` type. It causes the application crash easily without the guarantee of the compiler's safety checks. We must add `catch_unwind` everywhere in our application to avoid the crash, that is bad. 105 | -------------------------------------------------------------------------------- /y-octo-utils/yrs-is-unsafe/bin/global_lock.rs: -------------------------------------------------------------------------------- 1 | use std::thread::spawn; 2 | 3 | use yrs::{Doc, Transact}; 4 | 5 | fn main() { 6 | let doc = Doc::new(); 7 | 8 | { 9 | let doc = doc.clone(); 10 | spawn(move || { 11 | let _ = doc.transact_mut(); 12 | }); 13 | } 14 | { 15 | let doc = doc.clone(); 16 | spawn(move || { 17 | let _ = doc.transact_mut(); 18 | }); 19 | } 20 | { 21 | let doc = doc.clone(); 22 | spawn(move || { 23 | let _ = doc.transact_mut(); 24 | }); 25 | } 26 | { 27 | let doc = doc.clone(); 28 | spawn(move || { 29 | let _ = doc.transact_mut(); 30 | }); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /y-octo-utils/yrs-is-unsafe/bin/mem_usage.rs: -------------------------------------------------------------------------------- 1 | use yrs::{updates::decoder::Decode, Update}; 2 | 3 | fn main() { 4 | if Update::decode_v1(&[255, 255, 255, 122]).is_ok() {}; 5 | } 6 | -------------------------------------------------------------------------------- /y-octo-utils/yrs-is-unsafe/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::thread::spawn; 2 | 3 | use yrs::Doc; 4 | 5 | fn main() { 6 | let doc = Doc::new(); 7 | 8 | let t1 = { 9 | let doc = doc.clone(); 10 | spawn(move || { 11 | let _ = doc.get_or_insert_map("text"); 12 | }) 13 | }; 14 | 15 | let t2 = { 16 | let doc = doc.clone(); 17 | spawn(move || { 18 | let _ = doc.get_or_insert_map("text"); 19 | }) 20 | }; 21 | 22 | t1.join().unwrap(); 23 | t2.join().unwrap(); 24 | } 25 | -------------------------------------------------------------------------------- /y-octo/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = [ 3 | "DarkSky ", 4 | "forehalo ", 5 | "x1a0t <405028157@qq.com>", 6 | "Brooklyn ", 7 | ] 8 | description = "High-performance and thread-safe CRDT implementation compatible with Yjs" 9 | edition = "2021" 10 | homepage = "https://github.com/toeverything/y-octo" 11 | include = ["src/**/*", "benches/**/*", "bin/**/*", "LICENSE", "README.md"] 12 | keywords = ["collaboration", "crdt", "crdts", "yjs", "yata"] 13 | license = "MIT" 14 | name = "y-octo" 15 | readme = "README.md" 16 | repository = "https://github.com/toeverything/y-octo" 17 | version = "0.0.1" 18 | 19 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 20 | 21 | [dependencies] 22 | ahash = "0.8" 23 | bitvec = "1.0" 24 | byteorder = "1.5" 25 | lasso = { version = "0.7", features = ["multi-threaded"] } 26 | log = "0.4" 27 | nanoid = "0.4" 28 | nom = "7.1" 29 | ordered-float = "4.1" 30 | rand = "0.8" 31 | rand_chacha = "0.3" 32 | rand_distr = "0.4" 33 | serde = { version = "1.0", features = ["derive"] } 34 | serde_json = "1.0" 35 | smol_str = "0.3" 36 | thiserror = "2.0" 37 | 38 | [features] 39 | bench = [] 40 | debug = [] 41 | large_refs = [] 42 | serde_json = [] 43 | 44 | [target.'cfg(fuzzing)'.dependencies] 45 | arbitrary = { version = "1.3", features = ["derive"] } 46 | ordered-float = { version = "4.1", features = ["arbitrary"] } 47 | 48 | [target.'cfg(loom)'.dependencies] 49 | loom = { version = "0.7", features = ["checkpoint"] } 50 | # override the dev-dependencies feature 51 | async-lock = { version = "3.4.0", features = ["loom"] } 52 | 53 | [dev-dependencies] 54 | assert-json-diff = "2.0" 55 | criterion = { version = "0.5", features = ["html_reports"] } 56 | lib0 = { version = "0.16", features = ["lib0-serde"] } 57 | ordered-float = { version = "4.1", features = ["proptest"] } 58 | path-ext = "0.1" 59 | proptest = "1.3" 60 | proptest-derive = "0.5" 61 | yrs = "=0.21.3" 62 | 63 | [lints.rust] 64 | unexpected_cfgs = { level = "warn", check-cfg = [ 65 | 'cfg(debug)', 66 | 'cfg(fuzzing)', 67 | 'cfg(loom)', 68 | ] } 69 | 70 | [[bench]] 71 | harness = false 72 | name = "array_ops_benchmarks" 73 | 74 | [[bench]] 75 | harness = false 76 | name = "codec_benchmarks" 77 | 78 | [[bench]] 79 | harness = false 80 | name = "map_ops_benchmarks" 81 | 82 | [[bench]] 83 | harness = false 84 | name = "text_ops_benchmarks" 85 | 86 | [[bench]] 87 | harness = false 88 | name = "apply_benchmarks" 89 | 90 | [[bench]] 91 | harness = false 92 | name = "update_benchmarks" 93 | 94 | [lib] 95 | bench = true 96 | -------------------------------------------------------------------------------- /y-octo/LICENSE: -------------------------------------------------------------------------------- 1 | ../LICENSE -------------------------------------------------------------------------------- /y-octo/README.md: -------------------------------------------------------------------------------- 1 | ../README.md -------------------------------------------------------------------------------- /y-octo/benches/apply_benchmarks.rs: -------------------------------------------------------------------------------- 1 | mod utils; 2 | 3 | use std::time::Duration; 4 | 5 | use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; 6 | use path_ext::PathExt; 7 | use utils::Files; 8 | 9 | fn apply(c: &mut Criterion) { 10 | let files = Files::load(); 11 | 12 | let mut group = c.benchmark_group("apply"); 13 | group.measurement_time(Duration::from_secs(15)); 14 | 15 | for file in &files.files { 16 | group.throughput(Throughput::Bytes(file.content.len() as u64)); 17 | group.bench_with_input( 18 | BenchmarkId::new("apply with jwst", file.path.name_str()), 19 | &file.content, 20 | |b, content| { 21 | b.iter(|| { 22 | use y_octo::*; 23 | let mut doc = Doc::new(); 24 | doc.apply_update_from_binary_v1(content.clone()).unwrap() 25 | }); 26 | }, 27 | ); 28 | } 29 | 30 | group.finish(); 31 | } 32 | 33 | criterion_group!(benches, apply); 34 | criterion_main!(benches); 35 | -------------------------------------------------------------------------------- /y-octo/benches/array_ops_benchmarks.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use criterion::{criterion_group, criterion_main, Criterion}; 4 | use rand::{Rng, SeedableRng}; 5 | 6 | fn operations(c: &mut Criterion) { 7 | let mut group = c.benchmark_group("ops/array"); 8 | group.measurement_time(Duration::from_secs(15)); 9 | 10 | group.bench_function("jwst/insert", |b| { 11 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; 12 | let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); 13 | 14 | let idxs = (0..99) 15 | .map(|_| rng.gen_range(0..base_text.len() as u64)) 16 | .collect::>(); 17 | b.iter(|| { 18 | use y_octo::*; 19 | let doc = Doc::default(); 20 | let mut array = doc.get_or_create_array("test").unwrap(); 21 | for c in base_text.chars() { 22 | array.push(c.to_string()).unwrap(); 23 | } 24 | for idx in &idxs { 25 | array.insert(*idx, "test").unwrap(); 26 | } 27 | }); 28 | }); 29 | 30 | group.bench_function("jwst/insert range", |b| { 31 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; 32 | let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); 33 | 34 | let idxs = (0..99) 35 | .map(|_| rng.gen_range(0..base_text.len() as u64)) 36 | .collect::>(); 37 | b.iter(|| { 38 | use y_octo::*; 39 | let doc = Doc::default(); 40 | let mut array = doc.get_or_create_array("test").unwrap(); 41 | for c in base_text.chars() { 42 | array.push(c.to_string()).unwrap(); 43 | } 44 | for idx in &idxs { 45 | array.insert(*idx, "test1").unwrap(); 46 | array.insert(idx + 1, "test2").unwrap(); 47 | } 48 | }); 49 | }); 50 | 51 | group.bench_function("jwst/remove", |b| { 52 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; 53 | 54 | b.iter(|| { 55 | use y_octo::*; 56 | let doc = Doc::default(); 57 | let mut array = doc.get_or_create_array("test").unwrap(); 58 | for c in base_text.chars() { 59 | array.push(c.to_string()).unwrap(); 60 | } 61 | for idx in (base_text.len() as u64)..0 { 62 | array.remove(idx, 1).unwrap(); 63 | } 64 | }); 65 | }); 66 | 67 | group.finish(); 68 | } 69 | 70 | criterion_group!(benches, operations); 71 | criterion_main!(benches); 72 | -------------------------------------------------------------------------------- /y-octo/benches/codec_benchmarks.rs: -------------------------------------------------------------------------------- 1 | use criterion::{criterion_group, criterion_main, Criterion, SamplingMode}; 2 | use y_octo::{read_var_i32, read_var_u64, write_var_i32, write_var_u64}; 3 | 4 | const BENCHMARK_SIZE: u32 = 100000; 5 | 6 | fn codec(c: &mut Criterion) { 7 | let mut codec_group = c.benchmark_group("codec"); 8 | codec_group.sampling_mode(SamplingMode::Flat); 9 | 10 | { 11 | codec_group.bench_function("jwst encode var_int (32 bit)", |b| { 12 | b.iter(|| { 13 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 14 | for i in 0..(BENCHMARK_SIZE as i32) { 15 | write_var_i32(&mut encoder, i).unwrap(); 16 | } 17 | }) 18 | }); 19 | codec_group.bench_function("jwst decode var_int (32 bit)", |b| { 20 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 21 | for i in 0..(BENCHMARK_SIZE as i32) { 22 | write_var_i32(&mut encoder, i).unwrap(); 23 | } 24 | 25 | b.iter(|| { 26 | let mut decoder = encoder.as_slice(); 27 | for i in 0..(BENCHMARK_SIZE as i32) { 28 | let (tail, num) = read_var_i32(decoder).unwrap(); 29 | decoder = tail; 30 | assert_eq!(num, i); 31 | } 32 | }) 33 | }); 34 | } 35 | 36 | { 37 | codec_group.bench_function("jwst encode var_uint (32 bit)", |b| { 38 | b.iter(|| { 39 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 40 | for i in 0..BENCHMARK_SIZE { 41 | write_var_u64(&mut encoder, i as u64).unwrap(); 42 | } 43 | }) 44 | }); 45 | codec_group.bench_function("jwst decode var_uint (32 bit)", |b| { 46 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 47 | for i in 0..BENCHMARK_SIZE { 48 | write_var_u64(&mut encoder, i as u64).unwrap(); 49 | } 50 | 51 | b.iter(|| { 52 | let mut decoder = encoder.as_slice(); 53 | for i in 0..BENCHMARK_SIZE { 54 | let (tail, num) = read_var_u64(decoder).unwrap(); 55 | decoder = tail; 56 | assert_eq!(num as u32, i); 57 | } 58 | }) 59 | }); 60 | } 61 | 62 | { 63 | codec_group.bench_function("jwst encode var_uint (64 bit)", |b| { 64 | b.iter(|| { 65 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 66 | for i in 0..(BENCHMARK_SIZE as u64) { 67 | write_var_u64(&mut encoder, i).unwrap(); 68 | } 69 | }) 70 | }); 71 | 72 | codec_group.bench_function("jwst decode var_uint (64 bit)", |b| { 73 | let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); 74 | for i in 0..(BENCHMARK_SIZE as u64) { 75 | write_var_u64(&mut encoder, i).unwrap(); 76 | } 77 | 78 | b.iter(|| { 79 | let mut decoder = encoder.as_slice(); 80 | for i in 0..(BENCHMARK_SIZE as u64) { 81 | let (tail, num) = read_var_u64(decoder).unwrap(); 82 | decoder = tail; 83 | assert_eq!(num, i); 84 | } 85 | }) 86 | }); 87 | } 88 | } 89 | 90 | criterion_group!(benches, codec); 91 | criterion_main!(benches); 92 | -------------------------------------------------------------------------------- /y-octo/benches/map_ops_benchmarks.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use criterion::{criterion_group, criterion_main, Criterion}; 4 | 5 | fn operations(c: &mut Criterion) { 6 | let mut group = c.benchmark_group("ops/map"); 7 | group.measurement_time(Duration::from_secs(15)); 8 | 9 | group.bench_function("jwst/insert", |b| { 10 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" 11 | .split(' ') 12 | .collect::>(); 13 | 14 | b.iter(|| { 15 | use y_octo::*; 16 | let doc = Doc::default(); 17 | let mut map = doc.get_or_create_map("test").unwrap(); 18 | for (idx, key) in base_text.iter().enumerate() { 19 | map.insert(key.to_string(), idx).unwrap(); 20 | } 21 | }); 22 | }); 23 | 24 | group.bench_function("jwst/get", |b| { 25 | use y_octo::*; 26 | 27 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" 28 | .split(' ') 29 | .collect::>(); 30 | let doc = Doc::default(); 31 | let mut map = doc.get_or_create_map("test").unwrap(); 32 | for (idx, key) in base_text.iter().enumerate() { 33 | map.insert(key.to_string(), idx).unwrap(); 34 | } 35 | 36 | b.iter(|| { 37 | for key in &base_text { 38 | map.get(key); 39 | } 40 | }); 41 | }); 42 | 43 | group.bench_function("jwst/remove", |b| { 44 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" 45 | .split(' ') 46 | .collect::>(); 47 | 48 | b.iter(|| { 49 | use y_octo::*; 50 | let doc = Doc::default(); 51 | let mut map = doc.get_or_create_map("test").unwrap(); 52 | for (idx, key) in base_text.iter().enumerate() { 53 | map.insert(key.to_string(), idx).unwrap(); 54 | } 55 | for key in &base_text { 56 | map.remove(key); 57 | } 58 | }); 59 | }); 60 | 61 | group.finish(); 62 | } 63 | 64 | criterion_group!(benches, operations); 65 | criterion_main!(benches); 66 | -------------------------------------------------------------------------------- /y-octo/benches/text_ops_benchmarks.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use criterion::{criterion_group, criterion_main, Criterion}; 4 | use rand::{Rng, SeedableRng}; 5 | 6 | fn operations(c: &mut Criterion) { 7 | let mut group = c.benchmark_group("ops/text"); 8 | group.measurement_time(Duration::from_secs(15)); 9 | 10 | group.bench_function("jwst/insert", |b| { 11 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; 12 | let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); 13 | 14 | let idxs = (0..99) 15 | .map(|_| rng.gen_range(0..base_text.len() as u64)) 16 | .collect::>(); 17 | b.iter(|| { 18 | use y_octo::*; 19 | let doc = Doc::default(); 20 | let mut text = doc.get_or_create_text("test").unwrap(); 21 | 22 | text.insert(0, base_text).unwrap(); 23 | for idx in &idxs { 24 | text.insert(*idx, "test").unwrap(); 25 | } 26 | }); 27 | }); 28 | 29 | group.bench_function("jwst/remove", |b| { 30 | let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; 31 | 32 | b.iter(|| { 33 | use y_octo::*; 34 | let doc = Doc::default(); 35 | let mut text = doc.get_or_create_text("test").unwrap(); 36 | 37 | text.insert(0, base_text).unwrap(); 38 | text.insert(0, base_text).unwrap(); 39 | text.insert(0, base_text).unwrap(); 40 | for idx in (base_text.len() as u64)..0 { 41 | text.remove(idx, 1).unwrap(); 42 | } 43 | }); 44 | }); 45 | 46 | group.finish(); 47 | } 48 | 49 | criterion_group!(benches, operations); 50 | criterion_main!(benches); 51 | -------------------------------------------------------------------------------- /y-octo/benches/update_benchmarks.rs: -------------------------------------------------------------------------------- 1 | mod utils; 2 | 3 | use std::time::Duration; 4 | 5 | use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; 6 | use path_ext::PathExt; 7 | use utils::Files; 8 | 9 | fn update(c: &mut Criterion) { 10 | let files = Files::load(); 11 | 12 | let mut group = c.benchmark_group("update"); 13 | group.measurement_time(Duration::from_secs(15)); 14 | 15 | for file in &files.files { 16 | group.throughput(Throughput::Bytes(file.content.len() as u64)); 17 | group.bench_with_input( 18 | BenchmarkId::new("parse with jwst", file.path.name_str()), 19 | &file.content, 20 | |b, content| { 21 | b.iter(|| { 22 | use y_octo::*; 23 | let mut decoder = RawDecoder::new(content); 24 | Update::read(&mut decoder).unwrap() 25 | }); 26 | }, 27 | ); 28 | } 29 | 30 | group.finish(); 31 | } 32 | 33 | criterion_group!(benches, update); 34 | criterion_main!(benches); 35 | -------------------------------------------------------------------------------- /y-octo/benches/utils/files.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fs::{read, read_dir}, 3 | path::{Path, PathBuf}, 4 | }; 5 | 6 | use path_ext::PathExt; 7 | 8 | pub struct File { 9 | pub path: PathBuf, 10 | pub content: Vec, 11 | } 12 | 13 | const BASE: &str = "src/fixtures/"; 14 | 15 | impl File { 16 | fn new(path: &Path) -> Self { 17 | let content = read(path).unwrap(); 18 | Self { 19 | path: path.into(), 20 | content, 21 | } 22 | } 23 | } 24 | 25 | pub struct Files { 26 | pub files: Vec, 27 | } 28 | 29 | impl Files { 30 | pub fn load() -> Self { 31 | let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(BASE); 32 | 33 | let files = read_dir(path).unwrap(); 34 | let files = files 35 | .flatten() 36 | .filter(|f| f.path().is_file() && f.path().ext_str() == "bin") 37 | .map(|f| File::new(&f.path())) 38 | .collect::>(); 39 | 40 | Self { files } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /y-octo/benches/utils/mod.rs: -------------------------------------------------------------------------------- 1 | mod files; 2 | 3 | pub use files::Files; 4 | -------------------------------------------------------------------------------- /y-octo/src/codec/buffer.rs: -------------------------------------------------------------------------------- 1 | use std::io::{Error, Write}; 2 | 3 | use nom::bytes::complete::take; 4 | 5 | use super::*; 6 | 7 | pub fn read_var_buffer(input: &[u8]) -> IResult<&[u8], &[u8]> { 8 | let (tail, len) = read_var_u64(input)?; 9 | let (tail, val) = take(len as usize)(tail)?; 10 | Ok((tail, val)) 11 | } 12 | 13 | pub fn write_var_buffer(buffer: &mut W, data: &[u8]) -> Result<(), Error> { 14 | write_var_u64(buffer, data.len() as u64)?; 15 | buffer.write_all(data)?; 16 | Ok(()) 17 | } 18 | 19 | #[cfg(test)] 20 | mod tests { 21 | use nom::{ 22 | error::{Error, ErrorKind}, 23 | AsBytes, Err, 24 | }; 25 | 26 | use super::*; 27 | 28 | #[test] 29 | fn test_read_var_buffer() { 30 | // Test case 1: valid input, buffer length = 5 31 | let input = [0x05, 0x01, 0x02, 0x03, 0x04, 0x05]; 32 | let expected_output = [0x01, 0x02, 0x03, 0x04, 0x05]; 33 | let result = read_var_buffer(&input); 34 | assert_eq!(result, Ok((&[][..], &expected_output[..]))); 35 | 36 | // Test case 2: truncated input, missing buffer 37 | let input = [0x05, 0x01, 0x02, 0x03]; 38 | let result = read_var_buffer(&input); 39 | assert_eq!(result, Err(Err::Error(Error::new(&input[1..], ErrorKind::Eof)))); 40 | 41 | // Test case 3: invalid input 42 | let input = [0xFF, 0x01, 0x02, 0x03]; 43 | let result = read_var_buffer(&input); 44 | assert_eq!(result, Err(Err::Error(Error::new(&input[2..], ErrorKind::Eof)))); 45 | 46 | // Test case 4: invalid var int encoding 47 | let input = [0xFF, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01]; 48 | let result = read_var_buffer(&input); 49 | assert_eq!(result, Err(Err::Error(Error::new(&input[7..], ErrorKind::Eof)))); 50 | } 51 | 52 | #[test] 53 | fn test_var_buf_codec() { 54 | test_var_buf_enc_dec(&[]); 55 | test_var_buf_enc_dec(&[0x01, 0x02, 0x03, 0x04, 0x05]); 56 | test_var_buf_enc_dec(b"test_var_buf_enc_dec"); 57 | 58 | #[cfg(not(miri))] 59 | { 60 | use rand::{thread_rng, Rng}; 61 | let mut rng = thread_rng(); 62 | for _ in 0..100 { 63 | test_var_buf_enc_dec(&{ 64 | let mut bytes = vec![0u8; rng.gen_range(0..u16::MAX as usize)]; 65 | rng.fill(&mut bytes[..]); 66 | bytes 67 | }); 68 | } 69 | } 70 | } 71 | 72 | fn test_var_buf_enc_dec(data: &[u8]) { 73 | let mut buf = Vec::::new(); 74 | write_var_buffer(&mut buf, data).unwrap(); 75 | let result = read_var_buffer(buf.as_bytes()); 76 | assert_eq!(result, Ok((&[][..], data))); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /y-octo/src/codec/integer.rs: -------------------------------------------------------------------------------- 1 | use std::io::{Error, Write}; 2 | 3 | use byteorder::WriteBytesExt; 4 | use nom::Needed; 5 | 6 | use super::*; 7 | 8 | pub fn read_var_u64(input: &[u8]) -> IResult<&[u8], u64> { 9 | // parse the first byte 10 | if let Some(next_byte) = input.first() { 11 | let mut shift = 7; 12 | let mut curr_byte = *next_byte; 13 | let mut rest = &input[1..]; 14 | 15 | // same logic in loop, but enable early exit when dealing with small numbers 16 | let mut num = (curr_byte & 0b0111_1111) as u64; 17 | 18 | // if the sign bit is set, we need more bits 19 | while (curr_byte >> 7) & 0b1 != 0 { 20 | if let Some(next_byte) = rest.first() { 21 | curr_byte = *next_byte; 22 | // add the remaining 7 bits to the number 23 | num |= ((curr_byte & 0b0111_1111) as u64).wrapping_shl(shift); 24 | shift += 7; 25 | rest = &rest[1..]; 26 | } else { 27 | return Err(nom::Err::Incomplete(Needed::new(input.len() + 1))); 28 | } 29 | } 30 | 31 | Ok((rest, num)) 32 | } else { 33 | Err(nom::Err::Incomplete(Needed::new(1))) 34 | } 35 | } 36 | 37 | pub fn write_var_u64(buffer: &mut W, mut num: u64) -> Result<(), Error> { 38 | // bit or 0b1000_0000 pre 7 bit if has more bits 39 | while num >= 0b10000000 { 40 | buffer.write_u8(num as u8 & 0b0111_1111 | 0b10000000)?; 41 | num >>= 7; 42 | } 43 | 44 | buffer.write_u8((num & 0b01111111) as u8)?; 45 | 46 | Ok(()) 47 | } 48 | 49 | pub fn read_var_i32(input: &[u8]) -> IResult<&[u8], i32> { 50 | // parse the first byte 51 | if let Some(next_byte) = input.first() { 52 | let mut shift = 6; 53 | let mut curr_byte = *next_byte; 54 | let mut rest: &[u8] = &input[1..]; 55 | 56 | // get the sign bit and the first 6 bits of the number 57 | let sign_bit = (curr_byte >> 6) & 0b1; 58 | let mut num = (curr_byte & 0b0011_1111) as i64; 59 | 60 | // if the sign bit is set, we need more bits 61 | while (curr_byte >> 7) & 0b1 != 0 { 62 | if let Some(next_byte) = rest.first() { 63 | curr_byte = *next_byte; 64 | // add the remaining 7 bits to the number 65 | num |= ((curr_byte & 0b0111_1111) as i64).wrapping_shl(shift); 66 | shift += 7; 67 | rest = &rest[1..]; 68 | } else { 69 | return Err(nom::Err::Incomplete(Needed::new(input.len() + 1))); 70 | } 71 | } 72 | 73 | // negate the number if the sign bit is set 74 | if sign_bit == 1 { 75 | num = -num; 76 | } 77 | 78 | Ok((rest, num as i32)) 79 | } else { 80 | Err(nom::Err::Incomplete(Needed::new(1))) 81 | } 82 | } 83 | 84 | pub fn write_var_i32(buffer: &mut W, num: i32) -> Result<(), Error> { 85 | let mut num = num as i64; 86 | let is_negative = num < 0; 87 | if is_negative { 88 | num = -num; 89 | } 90 | 91 | buffer.write_u8( 92 | // bit or 0b1000_0000 if has more bits 93 | if num > 0b00111111 { 0b10000000 } else { 0 } 94 | // bit or 0b0100_0000 if negative 95 | | if is_negative { 0b0100_0000 } else { 0 } 96 | // store last 6 bits 97 | | num as u8 & 0b0011_1111, 98 | )?; 99 | num >>= 6; 100 | while num > 0 { 101 | buffer.write_u8( 102 | // bit or 0b1000_0000 pre 7 bit if has more bits 103 | if num > 0b01111111 { 0b10000000 } else { 0 } 104 | // store last 7 bits 105 | | num as u8 & 0b0111_1111, 106 | )?; 107 | num >>= 7; 108 | } 109 | 110 | Ok(()) 111 | } 112 | 113 | #[cfg(test)] 114 | mod tests { 115 | 116 | use super::*; 117 | 118 | fn test_var_uint_enc_dec(num: u64) { 119 | let mut buf = Vec::new(); 120 | write_var_u64(&mut buf, num).unwrap(); 121 | 122 | let (rest, decoded_num) = read_var_u64(&buf).unwrap(); 123 | assert_eq!(num, decoded_num); 124 | assert_eq!(rest.len(), 0); 125 | } 126 | 127 | fn test_var_int_enc_dec(num: i32) { 128 | { 129 | let mut buf = Vec::new(); 130 | write_var_i32(&mut buf, num).unwrap(); 131 | 132 | let (rest, decoded_num) = read_var_i32(&buf).unwrap(); 133 | assert_eq!(num, decoded_num); 134 | assert_eq!(rest.len(), 0); 135 | } 136 | } 137 | 138 | #[test] 139 | fn test_var_uint_codec() { 140 | test_var_uint_enc_dec(0); 141 | test_var_uint_enc_dec(1); 142 | test_var_uint_enc_dec(127); 143 | test_var_uint_enc_dec(0b1000_0000); 144 | test_var_uint_enc_dec(0b1_0000_0000); 145 | test_var_uint_enc_dec(0b1_1111_1111); 146 | test_var_uint_enc_dec(0b10_0000_0000); 147 | test_var_uint_enc_dec(0b11_1111_1111); 148 | test_var_uint_enc_dec(0x7fff_ffff_ffff_ffff); 149 | test_var_uint_enc_dec(u64::MAX); 150 | } 151 | 152 | #[test] 153 | fn test_var_int() { 154 | test_var_int_enc_dec(0); 155 | test_var_int_enc_dec(1); 156 | test_var_int_enc_dec(-1); 157 | test_var_int_enc_dec(63); 158 | test_var_int_enc_dec(-63); 159 | test_var_int_enc_dec(64); 160 | test_var_int_enc_dec(-64); 161 | test_var_int_enc_dec(i32::MAX); 162 | test_var_int_enc_dec(i32::MIN); 163 | test_var_int_enc_dec(((1 << 20) - 1) * 8); 164 | test_var_int_enc_dec(-((1 << 20) - 1) * 8); 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /y-octo/src/codec/mod.rs: -------------------------------------------------------------------------------- 1 | mod buffer; 2 | mod integer; 3 | mod string; 4 | 5 | pub use buffer::{read_var_buffer, write_var_buffer}; 6 | pub use integer::{read_var_i32, read_var_u64, write_var_i32, write_var_u64}; 7 | pub use string::{read_var_string, write_var_string}; 8 | 9 | use super::*; 10 | -------------------------------------------------------------------------------- /y-octo/src/codec/string.rs: -------------------------------------------------------------------------------- 1 | use std::io::{Error, Write}; 2 | 3 | use nom::combinator::map_res; 4 | 5 | use super::*; 6 | 7 | pub fn read_var_string(input: &[u8]) -> IResult<&[u8], String> { 8 | map_res(read_var_buffer, |s| String::from_utf8(s.to_vec()))(input) 9 | } 10 | 11 | pub fn write_var_string>(buffer: &mut W, input: S) -> Result<(), Error> { 12 | let bytes = input.as_ref().as_bytes(); 13 | write_var_buffer(buffer, bytes)?; 14 | Ok(()) 15 | } 16 | 17 | #[cfg(test)] 18 | mod tests { 19 | use nom::{ 20 | error::{Error, ErrorKind}, 21 | AsBytes, Err, 22 | }; 23 | 24 | use super::*; 25 | 26 | #[test] 27 | fn test_read_var_string() { 28 | // Test case 1: valid input, string length = 5 29 | let input = [0x05, 0x68, 0x65, 0x6C, 0x6C, 0x6F]; 30 | let expected_output = "hello".to_string(); 31 | let result = read_var_string(&input); 32 | assert_eq!(result, Ok((&[][..], expected_output))); 33 | 34 | // Test case 2: missing string length 35 | let input = [0x68, 0x65, 0x6C, 0x6C, 0x6F]; 36 | let result = read_var_string(&input); 37 | assert_eq!(result, Err(Err::Error(Error::new(&input[1..], ErrorKind::Eof)))); 38 | 39 | // Test case 3: truncated input 40 | let input = [0x05, 0x68, 0x65, 0x6C, 0x6C]; 41 | let result = read_var_string(&input); 42 | assert_eq!(result, Err(Err::Error(Error::new(&input[1..], ErrorKind::Eof)))); 43 | 44 | // Test case 4: invalid input 45 | let input = [0xFF, 0x01, 0x02, 0x03, 0x04]; 46 | let result = read_var_string(&input); 47 | assert_eq!(result, Err(Err::Error(Error::new(&input[2..], ErrorKind::Eof)))); 48 | 49 | // Test case 5: invalid var int encoding 50 | let input = [0xFF, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01]; 51 | let result = read_var_string(&input); 52 | assert_eq!(result, Err(Err::Error(Error::new(&input[7..], ErrorKind::Eof)))); 53 | 54 | // Test case 6: invalid input, invalid UTF-8 encoding 55 | let input = [0x05, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]; 56 | let result = read_var_string(&input); 57 | assert_eq!(result, Err(Err::Error(Error::new(&input[..], ErrorKind::MapRes)))); 58 | } 59 | 60 | #[test] 61 | fn test_var_str_codec() { 62 | test_var_str_enc_dec("".to_string()); 63 | test_var_str_enc_dec(" ".to_string()); 64 | test_var_str_enc_dec("abcde".to_string()); 65 | test_var_str_enc_dec("🃒🃓🃟☗🀥🀫∺∼≂≇⓵➎⓷➏‍".to_string()); 66 | } 67 | 68 | fn test_var_str_enc_dec(input: String) { 69 | let mut buf = Vec::::new(); 70 | write_var_string(&mut buf, input.clone()).unwrap(); 71 | let (rest, decoded_str) = read_var_string(buf.as_bytes()).unwrap(); 72 | assert_eq!(decoded_str, input); 73 | assert_eq!(rest.len(), 0); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /y-octo/src/doc/codec/id.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fmt::Display, 3 | hash::Hash, 4 | ops::{Add, Sub}, 5 | }; 6 | 7 | pub type Client = u64; 8 | pub type Clock = u64; 9 | 10 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default)] 11 | #[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] 12 | #[cfg_attr(test, derive(proptest_derive::Arbitrary))] 13 | pub struct Id { 14 | pub client: Client, 15 | pub clock: Clock, 16 | } 17 | 18 | impl Id { 19 | pub fn new(client: Client, clock: Clock) -> Self { 20 | Self { client, clock } 21 | } 22 | } 23 | 24 | impl From<(Client, Clock)> for Id { 25 | fn from((client, clock): (Client, Clock)) -> Self { 26 | Id::new(client, clock) 27 | } 28 | } 29 | 30 | impl Sub for Id { 31 | type Output = Id; 32 | 33 | fn sub(self, rhs: Clock) -> Self::Output { 34 | (self.client, self.clock - rhs).into() 35 | } 36 | } 37 | 38 | impl Add for Id { 39 | type Output = Id; 40 | 41 | fn add(self, rhs: Clock) -> Self::Output { 42 | (self.client, self.clock + rhs).into() 43 | } 44 | } 45 | 46 | impl Display for Id { 47 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 48 | write!(f, "({}, {})", self.client, self.clock) 49 | } 50 | } 51 | 52 | #[cfg(test)] 53 | mod tests { 54 | use super::*; 55 | 56 | #[test] 57 | fn basic_id_operation() { 58 | let id_with_different_client_1 = Id::new(1, 1); 59 | let id_with_different_client_2 = Id::new(2, 1); 60 | 61 | assert_ne!(id_with_different_client_1, id_with_different_client_2); 62 | assert_eq!(Id::new(1, 1), Id::new(1, 1)); 63 | 64 | let clock = 2; 65 | assert_eq!(Id::new(1, 1) + clock, (1, 3).into()); 66 | assert_eq!(Id::new(1, 3) - clock, (1, 1).into()); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /y-octo/src/doc/codec/io/mod.rs: -------------------------------------------------------------------------------- 1 | mod codec_v1; 2 | mod reader; 3 | mod writer; 4 | 5 | pub use codec_v1::{RawDecoder, RawEncoder}; 6 | pub use reader::{CrdtRead, CrdtReader}; 7 | pub use writer::{CrdtWrite, CrdtWriter}; 8 | 9 | use super::*; 10 | -------------------------------------------------------------------------------- /y-octo/src/doc/codec/io/reader.rs: -------------------------------------------------------------------------------- 1 | use std::io::Error; 2 | 3 | use super::*; 4 | 5 | #[inline] 6 | pub fn map_read_error(e: Error) -> JwstCodecError { 7 | JwstCodecError::IncompleteDocument(e.to_string()) 8 | } 9 | 10 | pub trait CrdtReader { 11 | fn is_empty(&self) -> bool; 12 | fn len(&self) -> u64; 13 | fn read_var_u64(&mut self) -> JwstCodecResult; 14 | fn read_var_i32(&mut self) -> JwstCodecResult; 15 | fn read_var_string(&mut self) -> JwstCodecResult; 16 | fn read_var_buffer(&mut self) -> JwstCodecResult>; 17 | fn read_u8(&mut self) -> JwstCodecResult; 18 | fn read_f32_be(&mut self) -> JwstCodecResult; 19 | fn read_f64_be(&mut self) -> JwstCodecResult; 20 | fn read_i64_be(&mut self) -> JwstCodecResult; 21 | 22 | fn read_info(&mut self) -> JwstCodecResult; 23 | fn read_item_id(&mut self) -> JwstCodecResult; 24 | } 25 | 26 | pub trait CrdtRead { 27 | fn read(reader: &mut R) -> JwstCodecResult 28 | where 29 | Self: Sized; 30 | } 31 | -------------------------------------------------------------------------------- /y-octo/src/doc/codec/io/writer.rs: -------------------------------------------------------------------------------- 1 | use std::io::Error; 2 | 3 | use super::*; 4 | 5 | #[inline] 6 | pub fn map_write_error(e: Error) -> JwstCodecError { 7 | JwstCodecError::InvalidWriteBuffer(e.to_string()) 8 | } 9 | 10 | pub trait CrdtWriter { 11 | fn write_var_u64(&mut self, num: u64) -> JwstCodecResult; 12 | fn write_var_i32(&mut self, num: i32) -> JwstCodecResult; 13 | fn write_var_string>(&mut self, s: S) -> JwstCodecResult; 14 | fn write_var_buffer(&mut self, buf: &[u8]) -> JwstCodecResult; 15 | fn write_u8(&mut self, num: u8) -> JwstCodecResult; 16 | fn write_f32_be(&mut self, num: f32) -> JwstCodecResult; 17 | fn write_f64_be(&mut self, num: f64) -> JwstCodecResult; 18 | fn write_i64_be(&mut self, num: i64) -> JwstCodecResult; 19 | 20 | fn write_info(&mut self, num: u8) -> JwstCodecResult; 21 | fn write_item_id(&mut self, id: &Id) -> JwstCodecResult; 22 | } 23 | 24 | pub trait CrdtWrite { 25 | fn write(&self, writer: &mut W) -> JwstCodecResult 26 | where 27 | Self: Sized; 28 | } 29 | -------------------------------------------------------------------------------- /y-octo/src/doc/codec/item_flag.rs: -------------------------------------------------------------------------------- 1 | use std::sync::atomic::{AtomicU8, Ordering}; 2 | 3 | #[rustfmt::skip] 4 | #[allow(dead_code)] 5 | pub mod item_flags { 6 | pub const ITEM_KEEP : u8 = 0b0000_0001; 7 | pub const ITEM_COUNTABLE : u8 = 0b0000_0010; 8 | pub const ITEM_DELETED : u8 = 0b0000_0100; 9 | pub const ITEM_MARKED : u8 = 0b0000_1000; 10 | pub const ITEM_HAS_PARENT_SUB : u8 = 0b0010_0000; 11 | pub const ITEM_HAS_RIGHT_ID : u8 = 0b0100_0000; 12 | pub const ITEM_HAS_LEFT_ID : u8 = 0b1000_0000; 13 | pub const ITEM_HAS_SIBLING : u8 = 0b1100_0000; 14 | } 15 | 16 | #[derive(Debug)] 17 | pub struct ItemFlag(pub(self) AtomicU8); 18 | 19 | impl Default for ItemFlag { 20 | fn default() -> Self { 21 | Self(AtomicU8::new(0)) 22 | } 23 | } 24 | 25 | impl Clone for ItemFlag { 26 | fn clone(&self) -> Self { 27 | Self(AtomicU8::new(self.0.load(Ordering::Acquire))) 28 | } 29 | } 30 | 31 | impl From for ItemFlag { 32 | fn from(flags: u8) -> Self { 33 | Self(AtomicU8::new(flags)) 34 | } 35 | } 36 | 37 | #[allow(dead_code)] 38 | impl ItemFlag { 39 | #[inline(always)] 40 | pub fn set(&self, flag: u8) { 41 | self.0.fetch_or(flag, Ordering::SeqCst); 42 | } 43 | 44 | #[inline(always)] 45 | pub fn clear(&self, flag: u8) { 46 | self.0.fetch_and(!flag, Ordering::SeqCst); 47 | } 48 | 49 | #[inline(always)] 50 | pub fn check(&self, flag: u8) -> bool { 51 | self.0.load(Ordering::Acquire) & flag == flag 52 | } 53 | 54 | #[inline(always)] 55 | pub fn not(&self, flag: u8) -> bool { 56 | self.0.load(Ordering::Acquire) & flag == 0 57 | } 58 | 59 | #[inline(always)] 60 | pub fn keep(&self) -> bool { 61 | self.check(item_flags::ITEM_KEEP) 62 | } 63 | 64 | #[inline(always)] 65 | pub fn set_keep(&self) { 66 | self.set(item_flags::ITEM_KEEP); 67 | } 68 | 69 | #[inline(always)] 70 | pub fn clear_keep(&self) { 71 | self.clear(item_flags::ITEM_KEEP); 72 | } 73 | 74 | #[inline(always)] 75 | pub fn countable(&self) -> bool { 76 | self.check(item_flags::ITEM_COUNTABLE) 77 | } 78 | 79 | #[inline(always)] 80 | pub fn set_countable(&self) { 81 | self.set(item_flags::ITEM_COUNTABLE); 82 | } 83 | 84 | #[inline(always)] 85 | pub fn clear_countable(&self) { 86 | self.clear(item_flags::ITEM_COUNTABLE); 87 | } 88 | 89 | #[inline(always)] 90 | pub fn deleted(&self) -> bool { 91 | self.check(item_flags::ITEM_DELETED) 92 | } 93 | 94 | #[inline(always)] 95 | pub fn set_deleted(&self) { 96 | self.set(item_flags::ITEM_DELETED); 97 | } 98 | 99 | #[inline(always)] 100 | pub fn clear_deleted(&self) { 101 | self.clear(item_flags::ITEM_DELETED); 102 | } 103 | } 104 | 105 | #[cfg(test)] 106 | mod tests { 107 | use super::*; 108 | 109 | #[test] 110 | fn test_flag_set_and_clear() { 111 | { 112 | let flag = super::ItemFlag::default(); 113 | assert!(!flag.keep()); 114 | flag.set_keep(); 115 | assert!(flag.keep()); 116 | flag.clear_keep(); 117 | assert!(!flag.keep()); 118 | assert_eq!( 119 | flag.0.load(Ordering::SeqCst), 120 | ItemFlag::default().0.load(Ordering::SeqCst) 121 | ); 122 | } 123 | 124 | { 125 | let flag = super::ItemFlag::default(); 126 | assert!(!flag.countable()); 127 | flag.set_countable(); 128 | assert!(flag.countable()); 129 | flag.clear_countable(); 130 | assert!(!flag.countable()); 131 | assert_eq!( 132 | flag.0.load(Ordering::SeqCst), 133 | ItemFlag::default().0.load(Ordering::SeqCst) 134 | ); 135 | } 136 | 137 | { 138 | let flag = super::ItemFlag::default(); 139 | assert!(!flag.deleted()); 140 | flag.set_deleted(); 141 | assert!(flag.deleted()); 142 | flag.clear_deleted(); 143 | assert!(!flag.deleted()); 144 | assert_eq!( 145 | flag.0.load(Ordering::SeqCst), 146 | ItemFlag::default().0.load(Ordering::SeqCst) 147 | ); 148 | } 149 | 150 | { 151 | let flag = super::ItemFlag::default(); 152 | flag.set_keep(); 153 | flag.set_countable(); 154 | flag.set_deleted(); 155 | assert!(flag.keep()); 156 | assert!(flag.countable()); 157 | assert!(flag.deleted()); 158 | flag.clear_keep(); 159 | flag.clear_countable(); 160 | flag.clear_deleted(); 161 | assert!(!flag.keep()); 162 | assert!(!flag.countable()); 163 | assert!(!flag.deleted()); 164 | assert_eq!( 165 | flag.0.load(Ordering::SeqCst), 166 | ItemFlag::default().0.load(Ordering::SeqCst) 167 | ); 168 | } 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /y-octo/src/doc/codec/mod.rs: -------------------------------------------------------------------------------- 1 | mod any; 2 | mod content; 3 | mod delete_set; 4 | mod id; 5 | mod io; 6 | mod item; 7 | mod item_flag; 8 | mod refs; 9 | mod update; 10 | #[cfg(test)] 11 | mod utils; 12 | 13 | pub use any::Any; 14 | pub(crate) use content::Content; 15 | pub use delete_set::DeleteSet; 16 | pub use id::{Client, Clock, Id}; 17 | pub use io::{CrdtRead, CrdtReader, CrdtWrite, CrdtWriter, RawDecoder, RawEncoder}; 18 | pub(crate) use item::{Item, ItemRef, Parent}; 19 | pub(crate) use item_flag::{item_flags, ItemFlag}; 20 | pub(crate) use refs::Node; 21 | pub use update::Update; 22 | #[cfg(test)] 23 | pub(crate) use utils::*; 24 | 25 | use super::*; 26 | -------------------------------------------------------------------------------- /y-octo/src/doc/codec/utils/items.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | pub(crate) struct ItemBuilder { 4 | item: Item, 5 | } 6 | 7 | #[allow(dead_code)] 8 | impl ItemBuilder { 9 | pub fn new() -> ItemBuilder { 10 | Self { item: Item::default() } 11 | } 12 | 13 | pub fn id(mut self, id: Id) -> ItemBuilder { 14 | self.item.id = id; 15 | self 16 | } 17 | 18 | pub fn left(mut self, left: Somr) -> ItemBuilder { 19 | if let Some(l) = left.get() { 20 | self.item.origin_left_id = Some(l.last_id()); 21 | self.item.left = left; 22 | } 23 | self 24 | } 25 | 26 | pub fn right(mut self, right: Somr) -> ItemBuilder { 27 | if let Some(r) = right.get() { 28 | self.item.origin_right_id = Some(r.id); 29 | self.item.right = right; 30 | } 31 | self 32 | } 33 | 34 | pub fn left_id(mut self, left_id: Option) -> ItemBuilder { 35 | self.item.origin_left_id = left_id; 36 | self 37 | } 38 | 39 | pub fn right_id(mut self, right_id: Option) -> ItemBuilder { 40 | self.item.origin_right_id = right_id; 41 | self 42 | } 43 | 44 | pub fn parent(mut self, parent: Option) -> ItemBuilder { 45 | self.item.parent = parent; 46 | self 47 | } 48 | 49 | #[allow(dead_code)] 50 | pub fn parent_sub(mut self, parent_sub: Option) -> ItemBuilder { 51 | self.item.parent_sub = parent_sub; 52 | self 53 | } 54 | 55 | pub fn content(mut self, content: Content) -> ItemBuilder { 56 | self.item.content = content; 57 | self 58 | } 59 | 60 | pub fn flags(mut self, flags: ItemFlag) -> ItemBuilder { 61 | self.item.flags = flags; 62 | self 63 | } 64 | 65 | pub fn build(self) -> Item { 66 | if self.item.content.countable() { 67 | self.item.flags.set(item_flags::ITEM_COUNTABLE); 68 | } 69 | 70 | self.item 71 | } 72 | } 73 | 74 | #[cfg(test)] 75 | mod tests { 76 | use super::*; 77 | 78 | #[test] 79 | fn test_item_builder() { 80 | loom_model!({ 81 | let item = ItemBuilder::new() 82 | .id(Id::new(0, 1)) 83 | .left_id(Some(Id::new(2, 3))) 84 | .right_id(Some(Id::new(4, 5))) 85 | .parent(Some(Parent::String("test".into()))) 86 | .content(Content::Any(vec![Any::String("Hello".into())])) 87 | .build(); 88 | 89 | assert_eq!(item.id, Id::new(0, 1)); 90 | assert_eq!(item.origin_left_id, Some(Id::new(2, 3))); 91 | assert_eq!(item.origin_right_id, Some(Id::new(4, 5))); 92 | assert!(matches!(item.parent, Some(Parent::String(text)) if text == "test")); 93 | assert_eq!(item.parent_sub, None); 94 | assert_eq!(item.content, Content::Any(vec![Any::String("Hello".into())])); 95 | }); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /y-octo/src/doc/codec/utils/mod.rs: -------------------------------------------------------------------------------- 1 | mod items; 2 | 3 | pub(crate) use items::*; 4 | 5 | use super::*; 6 | -------------------------------------------------------------------------------- /y-octo/src/doc/common/mod.rs: -------------------------------------------------------------------------------- 1 | mod range; 2 | mod somr; 3 | mod state; 4 | 5 | pub use range::*; 6 | pub use somr::*; 7 | pub use state::*; 8 | 9 | use super::*; 10 | -------------------------------------------------------------------------------- /y-octo/src/doc/common/state.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Deref, DerefMut}; 2 | 3 | use super::{ 4 | Client, ClientMap, Clock, CrdtRead, CrdtReader, CrdtWrite, CrdtWriter, HashMapExt, Id, JwstCodecResult, 5 | HASHMAP_SAFE_CAPACITY, 6 | }; 7 | 8 | #[derive(Default, Debug, PartialEq, Clone)] 9 | pub struct StateVector(ClientMap); 10 | 11 | impl StateVector { 12 | pub fn set_max(&mut self, client: Client, clock: Clock) { 13 | self.entry(client) 14 | .and_modify(|m_clock| { 15 | if *m_clock < clock { 16 | *m_clock = clock; 17 | } 18 | }) 19 | .or_insert(clock); 20 | } 21 | 22 | pub fn get(&self, client: &Client) -> Clock { 23 | *self.0.get(client).unwrap_or(&0) 24 | } 25 | 26 | pub fn contains(&self, id: &Id) -> bool { 27 | id.clock <= self.get(&id.client) 28 | } 29 | 30 | pub fn set_min(&mut self, client: Client, clock: Clock) { 31 | self.entry(client) 32 | .and_modify(|m_clock| { 33 | if *m_clock > clock { 34 | *m_clock = clock; 35 | } 36 | }) 37 | .or_insert(clock); 38 | } 39 | 40 | pub fn iter(&self) -> impl Iterator { 41 | self.0.iter() 42 | } 43 | 44 | pub fn merge_with(&mut self, other: &Self) { 45 | for (client, clock) in other.iter() { 46 | self.set_min(*client, *clock); 47 | } 48 | } 49 | } 50 | 51 | impl Deref for StateVector { 52 | type Target = ClientMap; 53 | 54 | fn deref(&self) -> &Self::Target { 55 | &self.0 56 | } 57 | } 58 | 59 | impl DerefMut for StateVector { 60 | fn deref_mut(&mut self) -> &mut Self::Target { 61 | &mut self.0 62 | } 63 | } 64 | 65 | impl From<[(Client, Clock); N]> for StateVector { 66 | fn from(value: [(Client, Clock); N]) -> Self { 67 | let mut map = ClientMap::with_capacity(N); 68 | 69 | for (client, clock) in value { 70 | map.insert(client, clock); 71 | } 72 | 73 | Self(map) 74 | } 75 | } 76 | 77 | impl CrdtRead for StateVector { 78 | fn read(decoder: &mut R) -> JwstCodecResult { 79 | let len = decoder.read_var_u64()? as usize; 80 | 81 | // See: [HASHMAP_SAFE_CAPACITY] 82 | let mut map = ClientMap::with_capacity(len.min(HASHMAP_SAFE_CAPACITY)); 83 | for _ in 0..len { 84 | let client = decoder.read_var_u64()?; 85 | let clock = decoder.read_var_u64()?; 86 | map.insert(client, clock); 87 | } 88 | 89 | map.shrink_to_fit(); 90 | Ok(Self(map)) 91 | } 92 | } 93 | 94 | impl CrdtWrite for StateVector { 95 | fn write(&self, encoder: &mut W) -> JwstCodecResult { 96 | encoder.write_var_u64(self.len() as u64)?; 97 | 98 | for (client, clock) in self.iter() { 99 | encoder.write_var_u64(*client)?; 100 | encoder.write_var_u64(*clock)?; 101 | } 102 | 103 | Ok(()) 104 | } 105 | } 106 | 107 | #[cfg(test)] 108 | mod tests { 109 | use super::*; 110 | 111 | #[test] 112 | fn test_state_vector_basic() { 113 | let mut state_vector = StateVector::from([(1, 1), (2, 2), (3, 3)]); 114 | assert_eq!(state_vector.len(), 3); 115 | assert_eq!(state_vector.get(&1), 1); 116 | 117 | state_vector.set_min(1, 0); 118 | assert_eq!(state_vector.get(&1), 0); 119 | 120 | state_vector.set_max(1, 4); 121 | assert_eq!(state_vector.get(&1), 4); 122 | 123 | // set inexistent client 124 | state_vector.set_max(4, 1); 125 | assert_eq!(state_vector.get(&4), 1); 126 | 127 | // same client with larger clock 128 | assert!(!state_vector.contains(&(1, 5).into())); 129 | } 130 | 131 | #[test] 132 | fn test_state_vector_merge() { 133 | let mut state_vector = StateVector::from([(1, 1), (2, 2), (3, 3)]); 134 | let other_state_vector = StateVector::from([(1, 5), (2, 6), (3, 7)]); 135 | state_vector.merge_with(&other_state_vector); 136 | assert_eq!(state_vector, StateVector::from([(3, 3), (1, 1), (2, 2)])); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /y-octo/src/doc/hasher.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::HashMap, 3 | hash::{BuildHasher, Hasher}, 4 | }; 5 | 6 | use super::Client; 7 | 8 | #[derive(Default)] 9 | pub struct ClientHasher(Client); 10 | 11 | impl Hasher for ClientHasher { 12 | fn finish(&self) -> u64 { 13 | self.0 14 | } 15 | 16 | fn write(&mut self, _: &[u8]) {} 17 | 18 | fn write_u64(&mut self, i: u64) { 19 | self.0 = i 20 | } 21 | } 22 | 23 | #[derive(Default, Clone)] 24 | pub struct ClientHasherBuilder; 25 | 26 | impl BuildHasher for ClientHasherBuilder { 27 | type Hasher = ClientHasher; 28 | 29 | fn build_hasher(&self) -> Self::Hasher { 30 | ClientHasher::default() 31 | } 32 | } 33 | 34 | // use ClientID as key 35 | pub type ClientMap = HashMap; 36 | -------------------------------------------------------------------------------- /y-octo/src/doc/mod.rs: -------------------------------------------------------------------------------- 1 | mod awareness; 2 | mod codec; 3 | mod common; 4 | mod document; 5 | mod hasher; 6 | mod history; 7 | mod publisher; 8 | mod store; 9 | mod types; 10 | mod utils; 11 | 12 | pub use ahash::{HashMap, HashMapExt, HashSet, HashSetExt}; 13 | pub use awareness::{Awareness, AwarenessEvent}; 14 | pub use codec::*; 15 | pub use common::*; 16 | pub use document::{Doc, DocOptions}; 17 | pub use hasher::ClientMap; 18 | pub use history::{History, HistoryOptions, StoreHistory}; 19 | use smol_str::SmolStr; 20 | pub(crate) use store::DocStore; 21 | pub use types::*; 22 | pub use utils::*; 23 | 24 | use super::*; 25 | 26 | /// NOTE: 27 | /// - We do not use [HashMap::with_capacity(num_of_clients)] directly here 28 | /// because we don't trust the input data. 29 | /// - For instance, what if the first u64 was somehow set a very big value? 30 | /// - A pre-allocated HashMap with a big capacity may cause OOM. 31 | /// - A kinda safer approach is give it a max capacity of 1024 at first 32 | /// allocation, and then let std makes the growth as need. 33 | pub const HASHMAP_SAFE_CAPACITY: usize = 1 << 10; 34 | -------------------------------------------------------------------------------- /y-octo/src/doc/types/list/iterator.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | pub(crate) struct ListIterator<'a> { 4 | pub(super) _lock: RwLockReadGuard<'a, YType>, 5 | pub(super) cur: Somr, 6 | } 7 | 8 | impl Iterator for ListIterator<'_> { 9 | type Item = Somr; 10 | 11 | fn next(&mut self) -> Option { 12 | while let Some(item) = self.cur.clone().get() { 13 | let cur = std::mem::replace(&mut self.cur, item.right.clone()); 14 | if item.deleted() { 15 | continue; 16 | } 17 | 18 | return Some(cur); 19 | } 20 | 21 | None 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /y-octo/src/doc/types/value.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Display; 2 | 3 | use super::*; 4 | 5 | #[derive(Debug, PartialEq)] 6 | pub enum Value { 7 | Any(Any), 8 | Doc(Doc), 9 | Array(Array), 10 | Map(Map), 11 | Text(Text), 12 | XMLElement(XMLElement), 13 | XMLFragment(XMLFragment), 14 | XMLHook(XMLHook), 15 | XMLText(XMLText), 16 | } 17 | 18 | impl Value { 19 | pub fn to_any(&self) -> Option { 20 | match self { 21 | Value::Any(any) => Some(any.clone()), 22 | _ => None, 23 | } 24 | } 25 | 26 | pub fn to_array(&self) -> Option { 27 | match self { 28 | Value::Array(array) => Some(array.clone()), 29 | _ => None, 30 | } 31 | } 32 | 33 | pub fn to_map(&self) -> Option { 34 | match self { 35 | Value::Map(map) => Some(map.clone()), 36 | _ => None, 37 | } 38 | } 39 | 40 | pub fn to_text(&self) -> Option { 41 | match self { 42 | Value::Text(text) => Some(text.clone()), 43 | _ => None, 44 | } 45 | } 46 | 47 | pub fn from_vec>(el: Vec) -> Self { 48 | Value::Any(Any::Array(el.into_iter().map(|item| item.into()).collect::>())) 49 | } 50 | } 51 | 52 | impl From<&Content> for Value { 53 | fn from(value: &Content) -> Value { 54 | match value { 55 | Content::Any(any) => Value::Any(if any.len() == 1 { 56 | any[0].clone() 57 | } else { 58 | Any::Array(any.clone()) 59 | }), 60 | Content::String(s) => Value::Any(Any::String(s.clone())), 61 | Content::Json(json) => Value::Any(Any::Array( 62 | json.iter() 63 | .map(|item| { 64 | if let Some(s) = item { 65 | Any::String(s.clone()) 66 | } else { 67 | Any::Undefined 68 | } 69 | }) 70 | .collect::>(), 71 | )), 72 | Content::Binary(buf) => Value::Any(Any::Binary(buf.clone())), 73 | Content::Embed(v) => Value::Any(v.clone()), 74 | Content::Type(ty) => match ty.ty().unwrap().kind { 75 | YTypeKind::Array => Value::Array(Array::from_unchecked(ty.clone())), 76 | YTypeKind::Map => Value::Map(Map::from_unchecked(ty.clone())), 77 | YTypeKind::Text => Value::Text(Text::from_unchecked(ty.clone())), 78 | YTypeKind::XMLElement => Value::XMLElement(XMLElement::from_unchecked(ty.clone())), 79 | YTypeKind::XMLFragment => Value::XMLFragment(XMLFragment::from_unchecked(ty.clone())), 80 | YTypeKind::XMLHook => Value::XMLHook(XMLHook::from_unchecked(ty.clone())), 81 | YTypeKind::XMLText => Value::XMLText(XMLText::from_unchecked(ty.clone())), 82 | // actually unreachable 83 | YTypeKind::Unknown => Value::Any(Any::Undefined), 84 | }, 85 | Content::Doc { guid: _, opts } => Value::Doc( 86 | DocOptions::try_from(opts.clone()) 87 | .expect("Failed to parse doc options") 88 | .build(), 89 | ), 90 | Content::Format { .. } => unimplemented!(), 91 | // actually unreachable 92 | Content::Deleted(_) => Value::Any(Any::Undefined), 93 | } 94 | } 95 | } 96 | 97 | impl From for Content { 98 | fn from(value: Value) -> Self { 99 | match value { 100 | Value::Any(any) => Content::from(any), 101 | Value::Doc(doc) => Content::Doc { 102 | guid: doc.guid().to_owned(), 103 | opts: Any::from(doc.options().clone()), 104 | }, 105 | Value::Array(v) => Content::Type(v.0), 106 | Value::Map(v) => Content::Type(v.0), 107 | Value::Text(v) => Content::Type(v.0), 108 | Value::XMLElement(v) => Content::Type(v.0), 109 | Value::XMLFragment(v) => Content::Type(v.0), 110 | Value::XMLHook(v) => Content::Type(v.0), 111 | Value::XMLText(v) => Content::Type(v.0), 112 | } 113 | } 114 | } 115 | 116 | impl> From for Value { 117 | fn from(value: T) -> Self { 118 | Value::Any(value.into()) 119 | } 120 | } 121 | 122 | impl From for Value { 123 | fn from(value: Doc) -> Self { 124 | Value::Doc(value) 125 | } 126 | } 127 | 128 | impl Display for Value { 129 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 130 | match self { 131 | Value::Any(any) => write!(f, "{}", any), 132 | Value::Text(text) => write!(f, "{}", text), 133 | _ => write!(f, ""), 134 | } 135 | } 136 | } 137 | 138 | impl serde::Serialize for Value { 139 | fn serialize(&self, serializer: S) -> Result 140 | where 141 | S: serde::Serializer, 142 | { 143 | match self { 144 | Self::Any(any) => any.serialize(serializer), 145 | Self::Array(array) => array.serialize(serializer), 146 | Self::Map(map) => map.serialize(serializer), 147 | Self::Text(text) => text.serialize(serializer), 148 | // Self::XMLElement(xml_element) => xml_element.serialize(serializer), 149 | // Self::XMLFragment(xml_fragment) => xml_fragment.serialize(serializer), 150 | // Self::XMLHook(xml_hook) => xml_hook.serialize(serializer), 151 | // Self::XMLText(xml_text) => xml_text.serialize(serializer), 152 | // Self::Doc(doc) => doc.serialize(serializer), 153 | _ => serializer.serialize_none(), 154 | } 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /y-octo/src/doc/types/xml.rs: -------------------------------------------------------------------------------- 1 | use super::list::ListType; 2 | use crate::impl_type; 3 | 4 | impl_type!(XMLElement); 5 | impl ListType for XMLElement {} 6 | 7 | impl_type!(XMLFragment); 8 | impl ListType for XMLFragment {} 9 | 10 | impl_type!(XMLText); 11 | impl ListType for XMLText {} 12 | 13 | impl_type!(XMLHook); 14 | impl ListType for XMLHook {} 15 | -------------------------------------------------------------------------------- /y-octo/src/doc/utils.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | pub fn encode_awareness_as_message(awareness: AwarenessStates) -> JwstCodecResult> { 4 | let mut buffer = Vec::new(); 5 | write_sync_message(&mut buffer, &SyncMessage::Awareness(awareness)) 6 | .map_err(|e| JwstCodecError::InvalidWriteBuffer(e.to_string()))?; 7 | 8 | Ok(buffer) 9 | } 10 | 11 | pub fn encode_update_as_message(update: Vec) -> JwstCodecResult> { 12 | let mut buffer = Vec::new(); 13 | write_sync_message(&mut buffer, &SyncMessage::Doc(DocMessage::Update(update))) 14 | .map_err(|e| JwstCodecError::InvalidWriteBuffer(e.to_string()))?; 15 | 16 | Ok(buffer) 17 | } 18 | 19 | pub fn merge_updates_v1, I: IntoIterator>(updates: I) -> JwstCodecResult { 20 | let updates = updates 21 | .into_iter() 22 | .map(Update::decode_v1) 23 | .collect::>>()?; 24 | 25 | Ok(Update::merge(updates)) 26 | } 27 | -------------------------------------------------------------------------------- /y-octo/src/fixtures/basic.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/y-crdt/y-octo/1e705e4c9bb10dec5b7893c225fba2436a6e02f3/y-octo/src/fixtures/basic.bin -------------------------------------------------------------------------------- /y-octo/src/fixtures/database.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/y-crdt/y-octo/1e705e4c9bb10dec5b7893c225fba2436a6e02f3/y-octo/src/fixtures/database.bin -------------------------------------------------------------------------------- /y-octo/src/fixtures/edge-case-left-right-same-node.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/y-crdt/y-octo/1e705e4c9bb10dec5b7893c225fba2436a6e02f3/y-octo/src/fixtures/edge-case-left-right-same-node.bin -------------------------------------------------------------------------------- /y-octo/src/fixtures/large.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/y-crdt/y-octo/1e705e4c9bb10dec5b7893c225fba2436a6e02f3/y-octo/src/fixtures/large.bin -------------------------------------------------------------------------------- /y-octo/src/fixtures/local_docs.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /y-octo/src/fixtures/with-subdoc.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/y-crdt/y-octo/1e705e4c9bb10dec5b7893c225fba2436a6e02f3/y-octo/src/fixtures/with-subdoc.bin -------------------------------------------------------------------------------- /y-octo/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[forbid(unsafe_code)] 2 | mod codec; 3 | mod doc; 4 | mod protocol; 5 | mod sync; 6 | 7 | pub use codec::*; 8 | pub use doc::{ 9 | encode_awareness_as_message, encode_update_as_message, merge_updates_v1, Any, Array, Awareness, AwarenessEvent, 10 | Client, ClientMap, Clock, CrdtRead, CrdtReader, CrdtWrite, CrdtWriter, Doc, DocOptions, HashMap as AHashMap, 11 | HashMapExt, History, HistoryOptions, Id, Map, RawDecoder, RawEncoder, StateVector, StoreHistory, Text, Update, 12 | Value, 13 | }; 14 | pub(crate) use doc::{Content, Item}; 15 | use log::{debug, warn}; 16 | use nom::IResult; 17 | pub use protocol::{ 18 | read_sync_message, write_sync_message, AwarenessState, AwarenessStates, DocMessage, SyncMessage, SyncMessageScanner, 19 | }; 20 | use thiserror::Error; 21 | 22 | #[derive(Debug, Error, PartialEq)] 23 | pub enum JwstCodecError { 24 | #[error("Unexpected Scenario")] 25 | Unexpected, 26 | #[error("Damaged document: corrupt json data")] 27 | DamagedDocumentJson, 28 | #[error("Incomplete document: {0}")] 29 | IncompleteDocument(String), 30 | #[error("Invalid write buffer: {0}")] 31 | InvalidWriteBuffer(String), 32 | #[error("Content does not support splitting in {0}")] 33 | ContentSplitNotSupport(u64), 34 | #[error("GC or Skip does not support splitting")] 35 | ItemSplitNotSupport, 36 | #[error("update is empty")] 37 | UpdateIsEmpty, 38 | #[error("invalid update")] 39 | UpdateInvalid(#[from] nom::Err>), 40 | #[error("update not fully consumed: {0}")] 41 | UpdateNotFullyConsumed(usize), 42 | #[error("invalid struct clock, expect {expect}, actually {actually}")] 43 | StructClockInvalid { expect: u64, actually: u64 }, 44 | #[error("cannot find struct {clock} in {client_id}")] 45 | StructSequenceInvalid { client_id: u64, clock: u64 }, 46 | #[error("struct {0} not exists")] 47 | StructSequenceNotExists(u64), 48 | #[error("Invalid parent")] 49 | InvalidParent, 50 | #[error("Parent not found")] 51 | ParentNotFound, 52 | #[error("Invalid struct type, expect item, actually {0}")] 53 | InvalidStructType(&'static str), 54 | #[error("Can not cast known type to {0}")] 55 | TypeCastError(&'static str), 56 | #[error("Can not found root struct with name: {0}")] 57 | RootStructNotFound(String), 58 | #[error("Index {0} out of bound")] 59 | IndexOutOfBound(u64), 60 | #[error("Document has been released")] 61 | DocReleased, 62 | #[error("Unexpected type, expect {0}")] 63 | UnexpectedType(&'static str), 64 | } 65 | 66 | pub type JwstCodecResult = Result; 67 | -------------------------------------------------------------------------------- /y-octo/src/protocol/awareness.rs: -------------------------------------------------------------------------------- 1 | use nom::multi::count; 2 | 3 | use super::*; 4 | 5 | const NULL_STR: &str = "null"; 6 | 7 | #[derive(Debug, Clone, PartialEq)] 8 | #[cfg_attr(test, derive(proptest_derive::Arbitrary))] 9 | pub struct AwarenessState { 10 | #[cfg_attr(test, proptest(strategy = "0..u32::MAX as u64"))] 11 | pub(crate) clock: u64, 12 | // content is usually a json 13 | pub(crate) content: String, 14 | } 15 | 16 | impl AwarenessState { 17 | pub fn new(clock: u64, content: String) -> Self { 18 | AwarenessState { clock, content } 19 | } 20 | 21 | pub fn clock(&self) -> u64 { 22 | self.clock 23 | } 24 | 25 | pub fn content(&self) -> &str { 26 | &self.content 27 | } 28 | 29 | pub fn is_deleted(&self) -> bool { 30 | self.content == NULL_STR 31 | } 32 | 33 | pub(crate) fn add_clock(&mut self) { 34 | self.clock += 1; 35 | } 36 | 37 | pub(crate) fn set_clock(&mut self, clock: u64) { 38 | self.clock = clock; 39 | } 40 | 41 | pub fn set_content(&mut self, content: String) { 42 | self.add_clock(); 43 | self.content = content; 44 | } 45 | 46 | pub fn delete(&mut self) { 47 | self.set_content(NULL_STR.to_string()); 48 | } 49 | } 50 | 51 | impl Default for AwarenessState { 52 | fn default() -> Self { 53 | AwarenessState { 54 | clock: 0, 55 | content: NULL_STR.to_string(), 56 | } 57 | } 58 | } 59 | 60 | fn read_awareness_state(input: &[u8]) -> IResult<&[u8], (u64, AwarenessState)> { 61 | let (tail, client_id) = read_var_u64(input)?; 62 | let (tail, clock) = read_var_u64(tail)?; 63 | let (tail, content) = read_var_string(tail)?; 64 | 65 | Ok((tail, (client_id, AwarenessState { clock, content }))) 66 | } 67 | 68 | fn write_awareness_state(buffer: &mut W, client_id: u64, state: &AwarenessState) -> Result<(), IoError> { 69 | write_var_u64(buffer, client_id)?; 70 | write_var_u64(buffer, state.clock)?; 71 | write_var_string(buffer, state.content.clone())?; 72 | 73 | Ok(()) 74 | } 75 | 76 | pub type AwarenessStates = HashMap; 77 | 78 | pub fn read_awareness(input: &[u8]) -> IResult<&[u8], AwarenessStates> { 79 | let (tail, len) = read_var_u64(input)?; 80 | let (tail, messages) = count(read_awareness_state, len as usize)(tail)?; 81 | 82 | Ok((tail, messages.into_iter().collect())) 83 | } 84 | 85 | pub fn write_awareness(buffer: &mut W, clients: &AwarenessStates) -> Result<(), IoError> { 86 | write_var_u64(buffer, clients.len() as u64)?; 87 | 88 | for (client_id, state) in clients { 89 | write_awareness_state(buffer, *client_id, state)?; 90 | } 91 | 92 | Ok(()) 93 | } 94 | 95 | // TODO(@darkskygit): impl reader/writer 96 | // awareness state message 97 | #[allow(dead_code)] 98 | #[derive(Debug, PartialEq)] 99 | pub struct AwarenessMessage { 100 | clients: AwarenessStates, 101 | } 102 | 103 | #[cfg(test)] 104 | mod tests { 105 | use super::*; 106 | 107 | #[test] 108 | fn test_awareness() { 109 | let input = [ 110 | 3, // count of state 111 | 1, 5, 1, 1, // first state 112 | 2, 10, 2, 2, 3, // second state 113 | 5, 5, 5, 1, 2, 3, 4, 5, // third state 114 | ]; 115 | 116 | let expected = HashMap::from([ 117 | (1, AwarenessState::new(5, String::from_utf8(vec![1]).unwrap())), 118 | (2, AwarenessState::new(10, String::from_utf8(vec![2, 3]).unwrap())), 119 | ( 120 | 5, 121 | AwarenessState::new(5, String::from_utf8(vec![1, 2, 3, 4, 5]).unwrap()), 122 | ), 123 | ]); 124 | 125 | { 126 | let (tail, result) = read_awareness(&input).unwrap(); 127 | assert!(tail.is_empty()); 128 | assert_eq!(result, expected); 129 | } 130 | 131 | { 132 | let mut buffer = Vec::new(); 133 | // hashmap has not a ordered keys, so buffer not equal each write 134 | // we need re-parse the buffer to check result 135 | write_awareness(&mut buffer, &expected).unwrap(); 136 | let (tail, result) = read_awareness(&buffer).unwrap(); 137 | assert!(tail.is_empty()); 138 | assert_eq!(result, expected); 139 | } 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /y-octo/src/protocol/doc.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | // doc sync message 4 | #[derive(Debug, Clone, PartialEq)] 5 | #[cfg_attr(test, derive(proptest_derive::Arbitrary))] 6 | pub enum DocMessage { 7 | // state vector 8 | // TODO: temporarily skipped in the test, because yrs decoding needs to ensure that the update in step1 is the 9 | // correct state vector binary and any data can be included in our implementation (we will ensure the 10 | // correctness of encoding and decoding in the subsequent decoding process) 11 | #[cfg_attr(test, proptest(skip))] 12 | Step1(Vec), 13 | // update 14 | Step2(Vec), 15 | // update 16 | Update(Vec), 17 | } 18 | 19 | const DOC_MESSAGE_STEP1: u64 = 0; 20 | const DOC_MESSAGE_STEP2: u64 = 1; 21 | const DOC_MESSAGE_UPDATE: u64 = 2; 22 | 23 | pub fn read_doc_message(input: &[u8]) -> IResult<&[u8], DocMessage> { 24 | let (tail, step) = read_var_u64(input)?; 25 | 26 | match step { 27 | DOC_MESSAGE_STEP1 => { 28 | let (tail, sv) = read_var_buffer(tail)?; 29 | // TODO: decode state vector 30 | Ok((tail, DocMessage::Step1(sv.into()))) 31 | } 32 | DOC_MESSAGE_STEP2 => { 33 | let (tail, update) = read_var_buffer(tail)?; 34 | // TODO: decode update 35 | Ok((tail, DocMessage::Step2(update.into()))) 36 | } 37 | DOC_MESSAGE_UPDATE => { 38 | let (tail, update) = read_var_buffer(tail)?; 39 | // TODO: decode update 40 | Ok((tail, DocMessage::Update(update.into()))) 41 | } 42 | _ => Err(nom::Err::Error(Error::new(input, ErrorKind::Tag))), 43 | } 44 | } 45 | 46 | pub fn write_doc_message(buffer: &mut W, msg: &DocMessage) -> Result<(), IoError> { 47 | match msg { 48 | DocMessage::Step1(sv) => { 49 | write_var_u64(buffer, DOC_MESSAGE_STEP1)?; 50 | write_var_buffer(buffer, sv)?; 51 | } 52 | DocMessage::Step2(update) => { 53 | write_var_u64(buffer, DOC_MESSAGE_STEP2)?; 54 | write_var_buffer(buffer, update)?; 55 | } 56 | DocMessage::Update(update) => { 57 | write_var_u64(buffer, DOC_MESSAGE_UPDATE)?; 58 | write_var_buffer(buffer, update)?; 59 | } 60 | } 61 | 62 | Ok(()) 63 | } 64 | 65 | #[cfg(test)] 66 | mod tests { 67 | use super::*; 68 | 69 | #[test] 70 | fn test_doc_message() { 71 | let messages = [ 72 | DocMessage::Step1(vec![0x01, 0x02, 0x03]), 73 | DocMessage::Step2(vec![0x04, 0x05, 0x06]), 74 | DocMessage::Update(vec![0x07, 0x08, 0x09]), 75 | ]; 76 | 77 | for msg in messages { 78 | let mut buffer = Vec::new(); 79 | 80 | write_doc_message(&mut buffer, &msg).unwrap(); 81 | let (tail, decoded) = read_doc_message(&buffer).unwrap(); 82 | 83 | assert_eq!(tail.len(), 0); 84 | assert_eq!(decoded, msg); 85 | } 86 | 87 | // test invalid msg 88 | { 89 | let mut buffer = Vec::new(); 90 | let msg = DocMessage::Step1(vec![0x01, 0x02, 0x03]); 91 | 92 | write_doc_message(&mut buffer, &msg).unwrap(); 93 | buffer[0] = 0xff; // Inject error in message tag 94 | let res = read_doc_message(&buffer); 95 | 96 | match res.as_ref().unwrap_err() { 97 | nom::Err::Error(error) => assert_eq!(error.code, ErrorKind::Tag), 98 | _ => panic!("Expected error ErrorKind::Tag, but got {:?}", res), 99 | } 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /y-octo/src/protocol/mod.rs: -------------------------------------------------------------------------------- 1 | mod awareness; 2 | mod doc; 3 | mod scanner; 4 | mod sync; 5 | 6 | use std::{ 7 | collections::HashMap, 8 | io::{Error as IoError, Write}, 9 | }; 10 | 11 | use awareness::{read_awareness, write_awareness}; 12 | pub use awareness::{AwarenessState, AwarenessStates}; 13 | pub use doc::DocMessage; 14 | use doc::{read_doc_message, write_doc_message}; 15 | use log::debug; 16 | use nom::{ 17 | error::{Error, ErrorKind}, 18 | IResult, 19 | }; 20 | pub use scanner::SyncMessageScanner; 21 | pub use sync::{read_sync_message, write_sync_message, SyncMessage}; 22 | 23 | use super::*; 24 | -------------------------------------------------------------------------------- /y-octo/src/protocol/scanner.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | pub struct SyncMessageScanner<'a> { 4 | buffer: &'a [u8], 5 | } 6 | 7 | impl SyncMessageScanner<'_> { 8 | pub fn new(buffer: &[u8]) -> SyncMessageScanner { 9 | SyncMessageScanner { buffer } 10 | } 11 | } 12 | 13 | impl<'a> Iterator for SyncMessageScanner<'a> { 14 | type Item = Result>>; 15 | 16 | fn next(&mut self) -> Option { 17 | if self.buffer.is_empty() { 18 | return None; 19 | } 20 | 21 | match read_sync_message(self.buffer) { 22 | Ok((tail, message)) => { 23 | self.buffer = tail; 24 | Some(Ok(message)) 25 | } 26 | Err(nom::Err::Incomplete(_)) 27 | | Err(nom::Err::Error(nom::error::Error { 28 | code: nom::error::ErrorKind::Eof, 29 | .. 30 | })) 31 | | Err(nom::Err::Failure(nom::error::Error { 32 | code: nom::error::ErrorKind::Eof, 33 | .. 34 | })) => { 35 | debug!("incomplete sync message"); 36 | None 37 | } 38 | 39 | Err(e) => Some(Err(e)), 40 | } 41 | } 42 | } 43 | 44 | #[cfg(test)] 45 | mod tests { 46 | use proptest::{collection::vec, prelude::*}; 47 | 48 | use super::*; 49 | 50 | proptest! { 51 | #[test] 52 | #[cfg_attr(miri, ignore)] 53 | fn test_sync_message_scanner(messages in vec(any::(), 0..10)) { 54 | let mut buffer = Vec::new(); 55 | 56 | for message in &messages { 57 | write_sync_message(&mut buffer, message).unwrap(); 58 | } 59 | 60 | let result: Result, _> = SyncMessageScanner::new(&buffer).collect(); 61 | assert_eq!(result.unwrap(), messages); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /y-octo/src/protocol/sync.rs: -------------------------------------------------------------------------------- 1 | use byteorder::WriteBytesExt; 2 | 3 | use super::*; 4 | 5 | #[derive(Debug, Clone, PartialEq)] 6 | enum MessageType { 7 | Auth, 8 | Awareness, 9 | AwarenessQuery, 10 | Doc, 11 | } 12 | 13 | fn read_sync_tag(input: &[u8]) -> IResult<&[u8], MessageType> { 14 | let (tail, tag) = read_var_u64(input)?; 15 | let tag = match tag { 16 | 0 => MessageType::Doc, 17 | 1 => MessageType::Awareness, 18 | 2 => MessageType::Auth, 19 | 3 => MessageType::AwarenessQuery, 20 | _ => return Err(nom::Err::Error(Error::new(input, ErrorKind::Tag))), 21 | }; 22 | 23 | Ok((tail, tag)) 24 | } 25 | 26 | fn write_sync_tag(buffer: &mut W, tag: MessageType) -> Result<(), IoError> { 27 | let tag: u64 = match tag { 28 | MessageType::Doc => 0, 29 | MessageType::Awareness => 1, 30 | MessageType::Auth => 2, 31 | MessageType::AwarenessQuery => 3, 32 | }; 33 | 34 | write_var_u64(buffer, tag)?; 35 | 36 | Ok(()) 37 | } 38 | 39 | // sync message 40 | #[derive(Debug, Clone, PartialEq)] 41 | #[cfg_attr(test, derive(proptest_derive::Arbitrary))] 42 | pub enum SyncMessage { 43 | Auth(Option), 44 | Awareness(AwarenessStates), 45 | AwarenessQuery, 46 | Doc(DocMessage), 47 | } 48 | 49 | pub fn read_sync_message(input: &[u8]) -> IResult<&[u8], SyncMessage> { 50 | let (tail, tag) = read_sync_tag(input)?; 51 | 52 | let (tail, message) = match tag { 53 | MessageType::Doc => { 54 | let (tail, doc) = read_doc_message(tail)?; 55 | (tail, SyncMessage::Doc(doc)) 56 | } 57 | MessageType::Awareness => { 58 | let (tail, update) = read_var_buffer(tail)?; 59 | ( 60 | tail, 61 | SyncMessage::Awareness({ 62 | let (awareness_tail, awareness) = read_awareness(update)?; 63 | let tail_len = awareness_tail.len(); 64 | if tail_len > 0 { 65 | debug!("awareness update has trailing bytes: {}", tail_len); 66 | debug_assert!(tail_len > 0, "awareness update has trailing bytes"); 67 | } 68 | awareness 69 | }), 70 | ) 71 | } 72 | MessageType::Auth => { 73 | let (tail, success) = read_var_u64(tail)?; 74 | 75 | if success == 1 { 76 | (tail, SyncMessage::Auth(None)) 77 | } else { 78 | let (tail, reason) = read_var_string(tail)?; 79 | (tail, SyncMessage::Auth(Some(reason))) 80 | } 81 | } 82 | MessageType::AwarenessQuery => (tail, SyncMessage::AwarenessQuery), 83 | }; 84 | 85 | Ok((tail, message)) 86 | } 87 | 88 | pub fn write_sync_message(buffer: &mut W, msg: &SyncMessage) -> Result<(), IoError> { 89 | match msg { 90 | SyncMessage::Auth(reason) => { 91 | const PERMISSION_DENIED: u8 = 0; 92 | const PERMISSION_GRANTED: u8 = 1; 93 | 94 | write_sync_tag(buffer, MessageType::Auth)?; 95 | if let Some(reason) = reason { 96 | buffer.write_u8(PERMISSION_DENIED)?; 97 | write_var_string(buffer, reason)?; 98 | } else { 99 | buffer.write_u8(PERMISSION_GRANTED)?; 100 | } 101 | } 102 | SyncMessage::AwarenessQuery => { 103 | write_sync_tag(buffer, MessageType::AwarenessQuery)?; 104 | } 105 | SyncMessage::Awareness(awareness) => { 106 | write_sync_tag(buffer, MessageType::Awareness)?; 107 | write_var_buffer(buffer, &{ 108 | let mut update = Vec::new(); 109 | write_awareness(&mut update, awareness)?; 110 | update 111 | })?; 112 | } 113 | SyncMessage::Doc(doc) => { 114 | write_sync_tag(buffer, MessageType::Doc)?; 115 | write_doc_message(buffer, doc)?; 116 | } 117 | } 118 | 119 | Ok(()) 120 | } 121 | 122 | #[cfg(test)] 123 | mod tests { 124 | use super::{awareness::AwarenessState, *}; 125 | 126 | #[test] 127 | fn test_sync_tag() { 128 | let messages = [ 129 | MessageType::Auth, 130 | MessageType::Awareness, 131 | MessageType::AwarenessQuery, 132 | MessageType::Doc, 133 | ]; 134 | 135 | for msg in messages { 136 | let mut buffer = Vec::new(); 137 | 138 | write_sync_tag(&mut buffer, msg.clone()).unwrap(); 139 | let (tail, decoded) = read_sync_tag(&buffer).unwrap(); 140 | 141 | assert_eq!(tail.len(), 0); 142 | assert_eq!(decoded, msg); 143 | } 144 | } 145 | 146 | #[test] 147 | fn test_sync_message() { 148 | let messages = [ 149 | SyncMessage::Auth(Some("reason".to_string())), 150 | SyncMessage::Awareness(HashMap::from([(1, AwarenessState::new(1, "test".into()))])), 151 | SyncMessage::AwarenessQuery, 152 | SyncMessage::Doc(DocMessage::Step1(vec![4, 5, 6])), 153 | SyncMessage::Doc(DocMessage::Step2(vec![7, 8, 9])), 154 | SyncMessage::Doc(DocMessage::Update(vec![10, 11, 12])), 155 | ]; 156 | 157 | for msg in messages { 158 | let mut buffer = Vec::new(); 159 | write_sync_message(&mut buffer, &msg).unwrap(); 160 | let (tail, decoded) = read_sync_message(&buffer).unwrap(); 161 | assert_eq!(tail.len(), 0); 162 | assert_eq!(decoded, msg); 163 | } 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /y-octo/src/sync.rs: -------------------------------------------------------------------------------- 1 | #[allow(unused)] 2 | #[cfg(not(loom))] 3 | pub(crate) use std::sync::{ 4 | atomic::{AtomicBool, AtomicU16, AtomicU32, AtomicU8, Ordering}, 5 | Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard, 6 | }; 7 | pub use std::sync::{Arc, Weak}; 8 | #[cfg(all(test, not(loom)))] 9 | pub(crate) use std::{ 10 | sync::{atomic::AtomicUsize, MutexGuard}, 11 | thread, 12 | }; 13 | 14 | #[cfg(loom)] 15 | pub(crate) use loom::{ 16 | sync::{ 17 | atomic::{AtomicBool, AtomicU16, AtomicU8, AtomicUsize, Ordering}, 18 | Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard, 19 | }, 20 | thread, 21 | }; 22 | 23 | #[macro_export(local_inner_macros)] 24 | macro_rules! loom_model { 25 | ($test:block) => { 26 | #[cfg(loom)] 27 | loom::model(move || $test); 28 | 29 | #[cfg(not(loom))] 30 | $test 31 | }; 32 | } 33 | --------------------------------------------------------------------------------