├── .github └── workflows │ ├── audit.yml │ ├── build.yml │ ├── ci.yml │ └── commitlint.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── Makefile.toml ├── README.md ├── collab-database ├── Cargo.toml ├── src │ ├── blocks │ │ ├── block.rs │ │ └── mod.rs │ ├── database.rs │ ├── database_state.rs │ ├── entity.rs │ ├── error.rs │ ├── fields │ │ ├── field.rs │ │ ├── field_id.rs │ │ ├── field_map.rs │ │ ├── field_observer.rs │ │ ├── field_settings.rs │ │ ├── mod.rs │ │ └── type_option │ │ │ ├── checkbox_type_option.rs │ │ │ ├── checklist_type_option.rs │ │ │ ├── date_type_option.rs │ │ │ ├── media_type_option.rs │ │ │ ├── mod.rs │ │ │ ├── number_type_option.rs │ │ │ ├── relation_type_option.rs │ │ │ ├── select_type_option.rs │ │ │ ├── summary_type_option.rs │ │ │ ├── text_type_option.rs │ │ │ ├── timestamp_type_option.rs │ │ │ ├── translate_type_option.rs │ │ │ └── url_type_option.rs │ ├── lib.rs │ ├── macros.rs │ ├── meta │ │ ├── meta_map.rs │ │ └── mod.rs │ ├── rows │ │ ├── cell.rs │ │ ├── comment.rs │ │ ├── mod.rs │ │ ├── row.rs │ │ ├── row_id.rs │ │ ├── row_meta.rs │ │ └── row_observer.rs │ ├── template │ │ ├── builder.rs │ │ ├── check_list_parse.rs │ │ ├── checkbox_parse.rs │ │ ├── csv.rs │ │ ├── date_parse.rs │ │ ├── entity.rs │ │ ├── media_parse.rs │ │ ├── mod.rs │ │ ├── number_parse.rs │ │ ├── option_parse.rs │ │ ├── relation_parse.rs │ │ ├── summary_parse.rs │ │ ├── time_parse.rs │ │ ├── timestamp_parse.rs │ │ ├── translate_parse.rs │ │ └── util.rs │ ├── util.rs │ ├── views │ │ ├── calculation.rs │ │ ├── define.rs │ │ ├── field_order.rs │ │ ├── field_settings.rs │ │ ├── filter.rs │ │ ├── group.rs │ │ ├── layout.rs │ │ ├── layout_settings.rs │ │ ├── mod.rs │ │ ├── row_order.rs │ │ ├── sort.rs │ │ ├── view.rs │ │ ├── view_map.rs │ │ └── view_observer.rs │ └── workspace_database │ │ ├── body.rs │ │ ├── manager.rs │ │ ├── mod.rs │ │ └── relation │ │ ├── db_relation.rs │ │ ├── mod.rs │ │ ├── row_relation.rs │ │ └── row_relation_map.rs └── tests │ ├── asset │ └── selected-services-march-2024-quarter-csv.csv │ ├── database_test │ ├── block_test.rs │ ├── cell_test.rs │ ├── cell_type_option_test.rs │ ├── encode_collab_test.rs │ ├── field_observe_test.rs │ ├── field_setting_test.rs │ ├── field_test.rs │ ├── filter_test.rs │ ├── group_test.rs │ ├── helper.rs │ ├── layout_test.rs │ ├── mod.rs │ ├── restore_test.rs │ ├── row_observe_test.rs │ ├── row_test.rs │ ├── sort_test.rs │ ├── type_option_test.rs │ ├── view_observe_test.rs │ └── view_test.rs │ ├── helper │ ├── mod.rs │ └── util.rs │ ├── history_database │ ├── 020_database.zip │ └── database_020_encode_collab │ ├── main.rs │ ├── template_test │ ├── create_template_test.rs │ ├── import_csv_test.rs │ └── mod.rs │ └── user_test │ ├── async_test │ ├── flush_test.rs │ ├── mod.rs │ ├── row_test.rs │ └── script.rs │ ├── cell_test.rs │ ├── database_test.rs │ ├── helper.rs │ ├── mod.rs │ ├── relation_test.rs │ ├── snapshot_test.rs │ └── type_option_test.rs ├── collab-derive ├── Cargo.toml └── src │ ├── collab.rs │ ├── internal │ ├── ast.rs │ ├── ctxt.rs │ └── mod.rs │ ├── lib.rs │ └── yrs_token.rs ├── collab-document ├── Cargo.toml ├── src │ ├── block_parser │ │ ├── document_parser.rs │ │ ├── mod.rs │ │ ├── parsers │ │ │ ├── bulleted_list.rs │ │ │ ├── callout.rs │ │ │ ├── code_block.rs │ │ │ ├── divider.rs │ │ │ ├── file_block.rs │ │ │ ├── heading.rs │ │ │ ├── image.rs │ │ │ ├── link_preview.rs │ │ │ ├── math_equation.rs │ │ │ ├── mod.rs │ │ │ ├── numbered_list.rs │ │ │ ├── page.rs │ │ │ ├── paragraph.rs │ │ │ ├── quote_list.rs │ │ │ ├── simple_column.rs │ │ │ ├── simple_columns.rs │ │ │ ├── simple_table.rs │ │ │ ├── simple_table_cell.rs │ │ │ ├── simple_table_row.rs │ │ │ ├── subpage.rs │ │ │ ├── todo_list.rs │ │ │ └── toggle_list.rs │ │ ├── registry.rs │ │ ├── text_utils.rs │ │ └── traits.rs │ ├── blocks │ │ ├── attr_keys.rs │ │ ├── block.rs │ │ ├── block_types.rs │ │ ├── children.rs │ │ ├── entities.rs │ │ ├── mod.rs │ │ ├── text.rs │ │ ├── text_entities.rs │ │ └── utils.rs │ ├── document.rs │ ├── document_awareness.rs │ ├── document_data.rs │ ├── error.rs │ ├── importer │ │ ├── define.rs │ │ ├── delta.rs │ │ ├── md_importer.rs │ │ ├── mod.rs │ │ └── util.rs │ └── lib.rs └── tests │ ├── block_parser │ ├── bulleted_list_test.rs │ ├── callout_test.rs │ ├── code_block_test.rs │ ├── divider_test.rs │ ├── document_parser_test.rs │ ├── file_block_test.rs │ ├── heading_test.rs │ ├── image_test.rs │ ├── link_preview_test.rs │ ├── math_equation_test.rs │ ├── mod.rs │ ├── numbered_list_test.rs │ ├── paragraph_test.rs │ ├── parser_test.rs │ ├── quote_list_test.rs │ ├── simple_columns_test.rs │ ├── simple_table_test.rs │ ├── subpage_test.rs │ ├── text_utils_test.rs │ ├── todo_list_test.rs │ └── toggle_list_test.rs │ ├── blocks │ ├── block_test.rs │ ├── block_test_core.rs │ ├── mod.rs │ └── text_test.rs │ ├── conversions │ ├── mod.rs │ └── plain_text_test.rs │ ├── document │ ├── awareness_test.rs │ ├── document_data_test.rs │ ├── document_test.rs │ ├── mod.rs │ ├── redo_undo_test.rs │ └── restore_test.rs │ ├── history_document │ └── 020_document.zip │ ├── importer │ ├── md_importer_customer_test.rs │ ├── md_importer_test.rs │ ├── mod.rs │ └── util.rs │ ├── main.rs │ └── util.rs ├── collab-entity ├── Cargo.toml ├── build.rs ├── proto │ └── collab │ │ ├── common.proto │ │ ├── encoding.proto │ │ ├── params.proto │ │ ├── protocol.proto │ │ ├── pubsub.proto │ │ └── realtime.proto └── src │ ├── collab_object.rs │ ├── define.rs │ ├── lib.rs │ ├── proto │ └── mod.rs │ └── reminder.rs ├── collab-folder ├── Cargo.toml ├── src │ ├── entities.rs │ ├── error.rs │ ├── folder.rs │ ├── folder_diff.rs │ ├── folder_migration.rs │ ├── folder_observe.rs │ ├── hierarchy_builder.rs │ ├── lib.rs │ ├── macros.rs │ ├── relation.rs │ ├── section.rs │ ├── space_info.rs │ ├── view.rs │ └── workspace.rs └── tests │ └── folder_test │ ├── child_views_test.rs │ ├── custom_section.rs │ ├── favorite_test.rs │ ├── history_folder │ ├── folder_data.json │ ├── folder_with_fav_v1.zip │ └── folder_without_fav.zip │ ├── load_disk.rs │ ├── main.rs │ ├── recent_views_test.rs │ ├── serde_test.rs │ ├── space_info_test.rs │ ├── trash_test.rs │ ├── util.rs │ ├── view_test.rs │ └── workspace_test.rs ├── collab-importer ├── Cargo.toml ├── src │ ├── error.rs │ ├── imported_collab.rs │ ├── lib.rs │ ├── notion │ │ ├── file.rs │ │ ├── importer.rs │ │ ├── mod.rs │ │ ├── page.rs │ │ └── walk_dir.rs │ ├── space_view.rs │ ├── util.rs │ └── zip_tool │ │ ├── async_zip.rs │ │ ├── mod.rs │ │ ├── sync_zip.rs │ │ └── util.rs └── tests │ ├── asset │ ├── all_md_files.zip │ ├── blog_post.zip │ ├── blog_post_duplicate_name.zip │ ├── blog_post_no_subpages.zip │ ├── csv_relation.zip │ ├── design.zip │ ├── empty_spaces.zip │ ├── empty_zip.zip │ ├── import_test.zip │ ├── multi_part_zip.zip │ ├── project&task.zip │ ├── project&task_contain_zip_attachment.zip │ ├── project&task_no_subpages.zip │ ├── project.zip │ ├── row_page_with_headings.zip │ ├── two_spaces.zip │ └── two_spaces_with_other_files.zip │ ├── main.rs │ ├── notion_test │ ├── customer_import_test.rs │ ├── import_test.rs │ └── mod.rs │ └── util.rs ├── collab-plugins ├── Cargo.toml ├── src │ ├── cloud_storage │ │ ├── channel.rs │ │ ├── error.rs │ │ ├── mod.rs │ │ ├── msg.rs │ │ ├── postgres │ │ │ ├── mod.rs │ │ │ └── plugin.rs │ │ ├── remote_collab.rs │ │ └── sink.rs │ ├── connect_state.rs │ ├── lib.rs │ └── local_storage │ │ ├── indexeddb │ │ ├── indexeddb_plugin.rs │ │ ├── kv_impl.rs │ │ └── mod.rs │ │ ├── kv │ │ ├── db.rs │ │ ├── doc.rs │ │ ├── error.rs │ │ ├── keys.rs │ │ ├── mod.rs │ │ ├── oid.rs │ │ ├── range.rs │ │ └── snapshot.rs │ │ ├── mod.rs │ │ ├── rocksdb │ │ ├── kv_impl.rs │ │ ├── mod.rs │ │ ├── rocksdb_plugin.rs │ │ ├── snapshot_plugin.rs │ │ └── util.rs │ │ └── storage_config.rs └── tests │ ├── disk │ ├── delete_test.rs │ ├── insert_test.rs │ ├── mod.rs │ ├── range_test.rs │ ├── restore_test.rs │ ├── script.rs │ ├── undo_test.rs │ └── util.rs │ ├── main.rs │ └── web │ ├── edit_collab_test.rs │ ├── indexeddb_test.rs │ ├── mod.rs │ ├── setup_tests.js │ └── test.md ├── collab-user ├── Cargo.toml ├── src │ ├── lib.rs │ ├── reminder.rs │ └── user_awareness.rs └── tests │ ├── main.rs │ ├── reminder_test │ ├── mod.rs │ ├── subscribe_test.rs │ └── test.rs │ └── util.rs ├── collab ├── Cargo.toml ├── src │ ├── any_mut.rs │ ├── core │ │ ├── any_array.rs │ │ ├── any_map.rs │ │ ├── array_wrapper.rs │ │ ├── collab.rs │ │ ├── collab_plugin.rs │ │ ├── collab_search.rs │ │ ├── collab_state.rs │ │ ├── fill.rs │ │ ├── map_wrapper.rs │ │ ├── mod.rs │ │ ├── origin.rs │ │ ├── text_wrapper.rs │ │ ├── transaction.rs │ │ └── value.rs │ ├── entity.rs │ ├── error.rs │ ├── lib.rs │ ├── lock │ │ ├── lock_timeout.rs │ │ └── mod.rs │ └── util.rs └── tests │ ├── edit_test │ ├── awareness_test.rs │ ├── insert_test.rs │ ├── mod.rs │ ├── observer_test.rs │ ├── restore_test.rs │ └── state_vec_test.rs │ ├── main.rs │ └── util │ └── mod.rs ├── docs ├── architecture.md ├── collab_object-CollabPlugins.png ├── collab_object-Create_Document.png ├── collab_object-Edit_Document.png ├── collab_object-Open_Document.png ├── collab_object-Sync_Document.png ├── collab_object.plantuml ├── collab_object.png ├── create_collab_object-CreateReminder.png ├── create_collab_object-UserAwareness.png └── create_collab_object.plantuml ├── resources └── crate_arch.png ├── rust-toolchain.toml └── rustfmt.toml /.github/workflows/audit.yml: -------------------------------------------------------------------------------- 1 | name: Security audit 2 | on: 3 | schedule: 4 | - cron: '0 0 * * *' 5 | push: 6 | paths: 7 | - '**/Cargo.toml' 8 | - '**/Cargo.lock' 9 | jobs: 10 | security_audit: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | - uses: taiki-e/install-action@cargo-deny 15 | - name: Scan for vulnerabilities 16 | run: 17 | cargo deny check advisories -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Collab 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | types: [ opened, synchronize, reopened ] 8 | branches: [ main ] 9 | 10 | env: 11 | CARGO_TERM_COLOR: always 12 | RUST_TOOLCHAIN: "1.85" 13 | 14 | jobs: 15 | fmt: 16 | name: Rustfmt 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v3 20 | - uses: dtolnay/rust-toolchain@stable 21 | with: 22 | toolchain: ${{ env.RUST_TOOLCHAIN }} 23 | components: rustfmt 24 | - name: Enforce formatting 25 | run: cargo fmt --check 26 | clippy: 27 | name: Clippy 28 | runs-on: ubuntu-latest 29 | steps: 30 | - uses: actions/checkout@v3 31 | - uses: dtolnay/rust-toolchain@stable 32 | with: 33 | toolchain: ${{ env.RUST_TOOLCHAIN }} 34 | components: clippy 35 | - name: Install protobuf 36 | run: | 37 | sudo apt-get update 38 | sudo apt-get install protobuf-compiler 39 | - name: Linting 40 | run: cargo clippy --all-targets -- -D warnings 41 | test: 42 | name: Test 43 | runs-on: ubuntu-latest 44 | steps: 45 | - uses: actions/checkout@v3 46 | - name: Install Rust toolchain 47 | id: rust_toolchain 48 | uses: actions-rs/toolchain@v1 49 | with: 50 | toolchain: ${{ env.RUST_TOOLCHAIN }} 51 | override: true 52 | profile: minimal 53 | 54 | - uses: Swatinem/rust-cache@v2 55 | with: 56 | prefix-key: ${{ matrix.os }} 57 | - name: Install protobuf 58 | run: | 59 | sudo apt-get update 60 | sudo apt-get install protobuf-compiler 61 | 62 | - name: Run tests 63 | run: cargo test 64 | 65 | -------------------------------------------------------------------------------- /.github/workflows/commitlint.yml: -------------------------------------------------------------------------------- 1 | name: Lint Commit Messages 2 | on: [pull_request, push] 3 | 4 | jobs: 5 | commitlint: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@v2 9 | with: 10 | fetch-depth: 0 11 | - uses: wagoid/commitlint-github-action@v4 12 | 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | # Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | .idea 12 | 13 | **/temp/** 14 | collab-plugins/.env 15 | collab-plugins/.env.test 16 | **/unit_test** 17 | 18 | # Protobuf generated code 19 | collab-entity/src/proto/*.rs 20 | !collab-entity/src/proto/mod.rs 21 | **/temp/** 22 | 23 | **/.DS_Store -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "collab", 4 | "collab-database", 5 | "collab-user", 6 | "collab-entity", 7 | "collab-document", 8 | "collab-folder", 9 | "collab-plugins", 10 | "collab-importer", 11 | ] 12 | resolver = "2" 13 | 14 | [workspace.dependencies] 15 | collab = { path = "collab" } 16 | collab-database = { path = "collab-database" } 17 | collab-plugins = { path = "collab-plugins" } 18 | collab-user = { path = "collab-user" } 19 | collab-entity = { path = "collab-entity" } 20 | collab-document = { path = "collab-document" } 21 | collab-folder = { path = "collab-folder" } 22 | collab-importer = { path = "collab-importer" } 23 | yrs = { version = "0.23.4", features = ["sync"] } 24 | anyhow = "1.0.94" 25 | thiserror = "1.0.39" 26 | serde = { version = "1.0.157", features = ["derive"] } 27 | serde_json = "1.0.108" 28 | tokio = { version = "1.38", features = ["sync"] } 29 | bytes = "1.5.0" 30 | tracing = "0.1.22" 31 | chrono = { version = "0.4.38", default-features = false, features = ["clock"] } 32 | async-trait = "0.1" 33 | arc-swap = { version = "1.7" } 34 | 35 | [patch.crates-io] 36 | # We're using a specific commit here because rust-rocksdb doesn't publish the latest version that includes the memory alignment fix. 37 | # For more details, see https://github.com/rust-rocksdb/rust-rocksdb/pull/868 38 | rocksdb = { git = "https://github.com/rust-rocksdb/rust-rocksdb", rev = "1710120e4549e04ba3baa6a1ee5a5a801fa45a72" } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # AppFlowy-Collab 3 | 4 | `AppFlowy-Collab` is a project that aims to support the collaborative features of AppFlowy. It consists of several crates that are currently under active development: 5 | 6 | * `collab` 7 | * `collab-database` 8 | * `collab-document` 9 | * `collab-folder` 10 | * `collab-plugins` 11 | * `collab-sync` 12 | 13 | ![architecture.png](resources/crate_arch.png) 14 | 15 | As the project is still a work in progress, it is rapidly evolving to improve its features and functionality. Therefore, 16 | it may still have some bugs and limitations, and its API may change frequently as new features are added and existing 17 | ones are refined. 18 | 19 | ## collab 20 | The `collab` crate is built on top of the [yrs](https://docs.rs/yrs/latest/yrs/) crate, providing a higher level of 21 | abstraction for the collaborative features of AppFlowy. It offers a simple API for creating and managing collaborative 22 | documents. 23 | 24 | ## collab-database 25 | The `collab-database` crate provides a simple API for creating and managing collaborative databases. It is built on top 26 | of the `collab` crate. 27 | 28 | ## collab-document 29 | The `collab-document` crate provides a simple API for creating and managing collaborative documents. It is built on top 30 | of the `collab` crate. 31 | 32 | ## collab-folder 33 | The `collab-folder` crate provides a simple API for creating and managing collaborative folders. It is built on top of 34 | the `collab` crate. 35 | 36 | ## collab-plugins 37 | The `collab-plugins` crate contains a list of plugins that can be used with the `collab` crate. 38 | 39 | ## collab-sync 40 | The `collab-sync` crate supports syncing the collaborative documents to a remote server. -------------------------------------------------------------------------------- /collab-database/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "collab-database" 3 | version = "0.2.0" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | crate-type = ["cdylib", "rlib"] 9 | 10 | [dependencies] 11 | collab = { workspace = true } 12 | collab-entity = { workspace = true } 13 | serde = { workspace = true, features = ["derive", "rc"] } 14 | serde_json.workspace = true 15 | thiserror.workspace = true 16 | anyhow.workspace = true 17 | serde_repr = "0.1" 18 | tokio = { workspace = true, features = ["time", "sync", "rt"] } 19 | tracing.workspace = true 20 | nanoid = "0.4.0" 21 | chrono.workspace = true 22 | lazy_static = "1.4.0" 23 | async-trait.workspace = true 24 | uuid = { version = "1.3.3", features = ["v4", "v5"] } 25 | tokio-stream = { version = "0.1.14", features = ["sync"] } 26 | strum = "0.25" 27 | strum_macros = "0.25" 28 | rayon = "1.10.0" 29 | dashmap = "5" 30 | futures = "0.3.30" 31 | csv = { version = "1.3.0" } 32 | yrs.workspace = true 33 | tokio-util = "0.7" 34 | rusty-money = { version = "0.4.1", features = ["iso"] } 35 | fancy-regex = "0.13.0" 36 | rust_decimal = "1.36.0" 37 | chrono-tz = "0.10.0" 38 | percent-encoding = "2.3.1" 39 | sha2 = "0.10.8" 40 | base64 = "0.22.1" 41 | iana-time-zone = "0.1.61" 42 | 43 | [target.'cfg(target_arch = "wasm32")'.dependencies] 44 | getrandom = { version = "0.2", features = ["js"] } 45 | js-sys = "0.3" 46 | 47 | [dev-dependencies] 48 | collab-plugins = { workspace = true, features = ["verbose_log"] } 49 | collab-database = { path = "../collab-database", features = ["verbose_log"] } 50 | tempfile = "3.8.0" 51 | assert-json-diff = "2.0.2" 52 | lazy_static = "1.4.0" 53 | tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } 54 | rand = "0.8.4" 55 | futures = "0.3.30" 56 | zip = "0.6.6" 57 | tokio = { version = "1.38", features = ["full"] } 58 | 59 | 60 | [features] 61 | verbose_log = [] 62 | import_csv = [] 63 | -------------------------------------------------------------------------------- /collab-database/src/blocks/mod.rs: -------------------------------------------------------------------------------- 1 | pub use block::*; 2 | 3 | mod block; 4 | -------------------------------------------------------------------------------- /collab-database/src/database_state.rs: -------------------------------------------------------------------------------- 1 | use crate::fields::FieldChangeSender; 2 | use tokio::sync::broadcast; 3 | 4 | use crate::rows::RowChangeSender; 5 | use crate::views::ViewChangeSender; 6 | 7 | #[derive(Clone)] 8 | pub struct DatabaseNotify { 9 | pub view_change_tx: ViewChangeSender, 10 | pub row_change_tx: RowChangeSender, 11 | pub field_change_tx: FieldChangeSender, 12 | } 13 | 14 | impl Default for DatabaseNotify { 15 | fn default() -> Self { 16 | let (view_change_tx, _) = broadcast::channel(100); 17 | let (row_change_tx, _) = broadcast::channel(100); 18 | let (field_change_tx, _) = broadcast::channel(100); 19 | Self { 20 | view_change_tx, 21 | row_change_tx, 22 | field_change_tx, 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /collab-database/src/error.rs: -------------------------------------------------------------------------------- 1 | use crate::rows::RowId; 2 | use collab_entity::CollabValidateError; 3 | 4 | #[derive(Debug, thiserror::Error)] 5 | pub enum DatabaseError { 6 | #[error("The database's id is invalid: {0}")] 7 | InvalidDatabaseID(&'static str), 8 | 9 | #[error("The database view's id is invalid: {0}")] 10 | InvalidViewID(&'static str), 11 | 12 | #[error("The database row's id is invalid: {0}")] 13 | InvalidRowID(&'static str), 14 | 15 | #[error("The database is not existing")] 16 | DatabaseNotExist, 17 | 18 | #[error("row: {row_id} not found, reason: {reason}")] 19 | DatabaseRowNotFound { row_id: RowId, reason: String }, 20 | 21 | #[error("The database view is not existing")] 22 | DatabaseViewNotExist, 23 | 24 | #[error(transparent)] 25 | SerdeJson(#[from] serde_json::Error), 26 | 27 | #[error(transparent)] 28 | UuidError(#[from] uuid::Error), 29 | 30 | #[error("No required data:{0}")] 31 | NoRequiredData(String), 32 | 33 | #[error("Record already exist")] 34 | RecordAlreadyExist, 35 | 36 | #[error("Record not found")] 37 | RecordNotFound, 38 | 39 | #[error("Action cancelled")] 40 | ActionCancelled, 41 | 42 | #[error("Invalid CSV:{0}")] 43 | InvalidCSV(String), 44 | 45 | #[error("Import data failed: {0}")] 46 | ImportData(String), 47 | 48 | #[error("Internal failure: {0}")] 49 | Internal(#[from] anyhow::Error), 50 | } 51 | 52 | impl DatabaseError { 53 | pub fn is_no_required_data(&self) -> bool { 54 | matches!(self, DatabaseError::NoRequiredData(_)) 55 | } 56 | } 57 | 58 | impl From for DatabaseError { 59 | fn from(error: CollabValidateError) -> Self { 60 | match error { 61 | CollabValidateError::NoRequiredData(data) => DatabaseError::NoRequiredData(data), 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /collab-database/src/fields/field_id.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{Display, Formatter}; 2 | use std::ops::Deref; 3 | 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] 7 | pub struct FieldId(String); 8 | 9 | impl Display for FieldId { 10 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 11 | f.write_str(&self.0.to_string()) 12 | } 13 | } 14 | 15 | impl FieldId { 16 | pub fn into_inner(self) -> String { 17 | self.0 18 | } 19 | } 20 | 21 | impl Deref for FieldId { 22 | type Target = String; 23 | 24 | fn deref(&self) -> &Self::Target { 25 | &self.0 26 | } 27 | } 28 | 29 | impl From for FieldId { 30 | fn from(data: String) -> Self { 31 | Self(data) 32 | } 33 | } 34 | 35 | impl From for String { 36 | fn from(data: FieldId) -> Self { 37 | data.0 38 | } 39 | } 40 | 41 | impl From for FieldId { 42 | fn from(data: i32) -> Self { 43 | Self(data.to_string()) 44 | } 45 | } 46 | 47 | impl From for FieldId { 48 | fn from(data: i64) -> Self { 49 | Self(data.to_string()) 50 | } 51 | } 52 | 53 | impl From for FieldId { 54 | fn from(data: usize) -> Self { 55 | Self(data.to_string()) 56 | } 57 | } 58 | 59 | impl AsRef for FieldId { 60 | fn as_ref(&self) -> &str { 61 | &self.0 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /collab-database/src/fields/field_observer.rs: -------------------------------------------------------------------------------- 1 | use crate::fields::{Field, field_from_map_ref, field_from_value}; 2 | use collab::preclude::{DeepObservable, EntryChange, Event, MapRef, Subscription}; 3 | use tokio::sync::broadcast; 4 | use tracing::warn; 5 | 6 | pub type FieldChangeSender = broadcast::Sender; 7 | pub type FieldChangeReceiver = broadcast::Receiver; 8 | 9 | #[derive(Clone, Debug)] 10 | pub enum FieldChange { 11 | DidCreateField { field: Field }, 12 | DidUpdateField { field: Field }, 13 | DidDeleteField { field_id: String }, 14 | } 15 | 16 | pub(crate) fn subscribe_field_change( 17 | field_map: &mut MapRef, 18 | change_tx: FieldChangeSender, 19 | ) -> Subscription { 20 | field_map.observe_deep(move |txn, events| { 21 | for deep_event in events.iter() { 22 | match deep_event { 23 | Event::Text(_) => {}, 24 | Event::Array(_) => {}, 25 | Event::Map(event) => { 26 | let keys = event.keys(txn); 27 | for (key, value) in keys.iter() { 28 | let _change_tx = change_tx.clone(); 29 | match value { 30 | EntryChange::Inserted(value) => { 31 | // tracing::trace!("field observer: Inserted: {}:{}", key, value); 32 | if let Some(field) = field_from_value(value.clone(), txn) { 33 | let _ = change_tx.send(FieldChange::DidCreateField { field }); 34 | } 35 | }, 36 | EntryChange::Updated(_, _value) => { 37 | // tracing::trace!("field observer: update: {}:{}", key, value); 38 | if let Some(field) = field_from_map_ref(event.target(), txn) { 39 | let _ = change_tx.send(FieldChange::DidUpdateField { field }); 40 | } 41 | }, 42 | EntryChange::Removed(_value) => { 43 | let field_id = (**key).to_string(); 44 | if !field_id.is_empty() { 45 | let _ = change_tx.send(FieldChange::DidDeleteField { field_id }); 46 | } else { 47 | warn!("field observer: delete: {}", key); 48 | } 49 | }, 50 | } 51 | } 52 | }, 53 | _ => {}, 54 | } 55 | } 56 | }) 57 | } 58 | -------------------------------------------------------------------------------- /collab-database/src/fields/mod.rs: -------------------------------------------------------------------------------- 1 | mod field; 2 | mod field_id; 3 | mod field_map; 4 | mod field_observer; 5 | mod field_settings; 6 | mod type_option; 7 | 8 | pub use field::*; 9 | pub use field_id::*; 10 | pub use field_map::*; 11 | pub use field_observer::*; 12 | pub use field_settings::*; 13 | pub use type_option::*; 14 | -------------------------------------------------------------------------------- /collab-database/src/fields/type_option/relation_type_option.rs: -------------------------------------------------------------------------------- 1 | use super::{TypeOptionData, TypeOptionDataBuilder}; 2 | use crate::fields::{TypeOptionCellReader, TypeOptionCellWriter}; 3 | use crate::rows::Cell; 4 | use crate::template::relation_parse::RelationCellData; 5 | use crate::template::util::ToCellString; 6 | use collab::util::AnyMapExt; 7 | use serde::{Deserialize, Serialize}; 8 | use serde_json::{Value, json}; 9 | use std::str::FromStr; 10 | 11 | #[derive(Debug, Clone, Default, Serialize, Deserialize)] 12 | pub struct RelationTypeOption { 13 | pub database_id: String, 14 | } 15 | 16 | impl From for RelationTypeOption { 17 | fn from(data: TypeOptionData) -> Self { 18 | let database_id: String = data.get_as("database_id").unwrap_or_default(); 19 | Self { database_id } 20 | } 21 | } 22 | 23 | impl From for TypeOptionData { 24 | fn from(data: RelationTypeOption) -> Self { 25 | TypeOptionDataBuilder::from([("database_id".into(), data.database_id.into())]) 26 | } 27 | } 28 | 29 | impl TypeOptionCellReader for RelationTypeOption { 30 | fn json_cell(&self, cell: &Cell) -> Value { 31 | let cell_data = RelationCellData::from(cell); 32 | json!(cell_data) 33 | } 34 | 35 | fn numeric_cell(&self, _cell: &Cell) -> Option { 36 | None 37 | } 38 | 39 | fn convert_raw_cell_data(&self, cell_data: &str) -> String { 40 | let cell_data = RelationCellData::from_str(cell_data).unwrap_or_default(); 41 | cell_data.to_cell_string() 42 | } 43 | } 44 | 45 | impl TypeOptionCellWriter for RelationTypeOption { 46 | fn convert_json_to_cell(&self, json_value: Value) -> Cell { 47 | let cell_data = serde_json::from_value::(json_value).unwrap_or_default(); 48 | Cell::from(cell_data) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /collab-database/src/fields/type_option/summary_type_option.rs: -------------------------------------------------------------------------------- 1 | use super::{TypeOptionData, TypeOptionDataBuilder}; 2 | use crate::fields::{TypeOptionCellReader, TypeOptionCellWriter}; 3 | use crate::rows::Cell; 4 | use crate::template::summary_parse::SummaryCellData; 5 | use collab::util::AnyMapExt; 6 | use serde::{Deserialize, Serialize}; 7 | use serde_json::{Value, json}; 8 | 9 | #[derive(Debug, Clone, Default, Serialize, Deserialize)] 10 | pub struct SummarizationTypeOption { 11 | pub auto_fill: bool, 12 | } 13 | 14 | impl From for SummarizationTypeOption { 15 | fn from(data: TypeOptionData) -> Self { 16 | let auto_fill: bool = data.get_as("auto_fill").unwrap_or_default(); 17 | Self { auto_fill } 18 | } 19 | } 20 | 21 | impl From for TypeOptionData { 22 | fn from(data: SummarizationTypeOption) -> Self { 23 | TypeOptionDataBuilder::from([("auto_fill".into(), data.auto_fill.into())]) 24 | } 25 | } 26 | 27 | impl TypeOptionCellReader for SummarizationTypeOption { 28 | fn json_cell(&self, cell: &Cell) -> Value { 29 | let cell_data = SummaryCellData::from(cell); 30 | json!(cell_data) 31 | } 32 | 33 | fn numeric_cell(&self, _cell: &Cell) -> Option { 34 | None 35 | } 36 | 37 | fn convert_raw_cell_data(&self, cell_data: &str) -> String { 38 | let cell_data = SummaryCellData(cell_data.to_string()); 39 | cell_data.to_string() 40 | } 41 | } 42 | 43 | impl TypeOptionCellWriter for SummarizationTypeOption { 44 | fn convert_json_to_cell(&self, json_value: Value) -> Cell { 45 | let cell_data = serde_json::from_value::(json_value).unwrap_or_default(); 46 | cell_data.into() 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /collab-database/src/fields/type_option/translate_type_option.rs: -------------------------------------------------------------------------------- 1 | use super::{TypeOptionData, TypeOptionDataBuilder}; 2 | use crate::fields::{TypeOptionCellReader, TypeOptionCellWriter}; 3 | use crate::rows::Cell; 4 | use crate::template::translate_parse::TranslateCellData; 5 | use serde::{Deserialize, Serialize}; 6 | use serde_json::{Value, json}; 7 | use yrs::{Any, encoding::serde::from_any}; 8 | 9 | #[derive(Debug, Clone, Serialize, Deserialize)] 10 | pub struct TranslateTypeOption { 11 | #[serde(default)] 12 | pub auto_fill: bool, 13 | /// Use [TranslateTypeOption::language_from_type] to get the language name 14 | #[serde(default, rename = "language")] 15 | pub language_type: i64, 16 | } 17 | 18 | impl TranslateTypeOption { 19 | pub fn language_from_type(language_type: i64) -> &'static str { 20 | match language_type { 21 | 0 => "Traditional Chinese", 22 | 1 => "English", 23 | 2 => "French", 24 | 3 => "German", 25 | 4 => "Hindi", 26 | 5 => "Spanish", 27 | 6 => "Portuguese", 28 | 7 => "Standard Arabic", 29 | 8 => "Simplified Chinese", 30 | _ => "English", 31 | } 32 | } 33 | } 34 | 35 | impl TypeOptionCellReader for TranslateTypeOption { 36 | fn json_cell(&self, cell: &Cell) -> Value { 37 | json!(self.stringify_cell(cell)) 38 | } 39 | 40 | fn numeric_cell(&self, _cell: &Cell) -> Option { 41 | None 42 | } 43 | 44 | fn convert_raw_cell_data(&self, cell_data: &str) -> String { 45 | let cell = serde_json::from_str::(cell_data).unwrap_or_default(); 46 | cell.to_string() 47 | } 48 | } 49 | 50 | impl TypeOptionCellWriter for TranslateTypeOption { 51 | fn convert_json_to_cell(&self, json_value: Value) -> Cell { 52 | let cell = TranslateCellData(json_value.as_str().unwrap_or_default().to_string()); 53 | cell.into() 54 | } 55 | } 56 | 57 | impl Default for TranslateTypeOption { 58 | fn default() -> Self { 59 | Self { 60 | auto_fill: false, 61 | language_type: 1, 62 | } 63 | } 64 | } 65 | 66 | impl From for TranslateTypeOption { 67 | fn from(data: TypeOptionData) -> Self { 68 | from_any(&Any::from(data)).unwrap() 69 | } 70 | } 71 | 72 | impl From for TypeOptionData { 73 | fn from(value: TranslateTypeOption) -> Self { 74 | TypeOptionDataBuilder::from([ 75 | ("auto_fill".into(), value.auto_fill.into()), 76 | ("language".into(), Any::BigInt(value.language_type)), 77 | ]) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /collab-database/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod database; 2 | pub mod fields; 3 | pub mod meta; 4 | pub mod rows; 5 | pub mod views; 6 | pub mod workspace_database; 7 | 8 | #[macro_use] 9 | mod macros; 10 | pub mod blocks; 11 | pub mod database_state; 12 | pub mod entity; 13 | pub mod error; 14 | pub mod template; 15 | pub mod util; 16 | -------------------------------------------------------------------------------- /collab-database/src/meta/meta_map.rs: -------------------------------------------------------------------------------- 1 | use collab::preclude::{Any, Map, MapRef, ReadTxn, TransactionMut}; 2 | use collab_entity::define::DATABASE_INLINE_VIEW; 3 | use std::ops::Deref; 4 | use tracing::error; 5 | 6 | pub struct MetaMap { 7 | container: MapRef, 8 | } 9 | 10 | impl MetaMap { 11 | pub fn new(container: MapRef) -> Self { 12 | Self { container } 13 | } 14 | 15 | /// Set the inline view id 16 | pub(crate) fn set_inline_view_id(&self, txn: &mut TransactionMut, view_id: &str) { 17 | self 18 | .container 19 | .insert(txn, DATABASE_INLINE_VIEW, Any::String(view_id.into())); 20 | } 21 | 22 | /// Get the inline view id 23 | pub(crate) fn get_inline_view_id(&self, txn: &T) -> Option { 24 | let out = self.container.get(txn, DATABASE_INLINE_VIEW); 25 | if out.is_none() { 26 | error!("Can't find inline view id"); 27 | } 28 | 29 | match out?.cast::() { 30 | Ok(id) => Some(id), 31 | Err(err) => { 32 | error!("Failed to cast inline view id: {:?}", err); 33 | None 34 | }, 35 | } 36 | } 37 | } 38 | 39 | impl Deref for MetaMap { 40 | type Target = MapRef; 41 | 42 | fn deref(&self) -> &Self::Target { 43 | &self.container 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /collab-database/src/meta/mod.rs: -------------------------------------------------------------------------------- 1 | mod meta_map; 2 | 3 | pub use meta_map::*; 4 | -------------------------------------------------------------------------------- /collab-database/src/rows/cell.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::ops::Deref; 3 | 4 | use collab::preclude::{Any, FillRef, Map, MapRef, TransactionMut}; 5 | use collab::util::AnyMapExt; 6 | 7 | use crate::database::timestamp; 8 | use crate::rows::{CREATED_AT, LAST_MODIFIED, RowId}; 9 | use crate::template::entity::CELL_DATA; 10 | 11 | pub type Cells = HashMap; 12 | 13 | pub struct CellsUpdate<'a, 'b> { 14 | map_ref: &'a MapRef, 15 | txn: &'a mut TransactionMut<'b>, 16 | } 17 | 18 | impl<'a, 'b> CellsUpdate<'a, 'b> { 19 | pub fn new(txn: &'a mut TransactionMut<'b>, map_ref: &'a MapRef) -> Self { 20 | Self { map_ref, txn } 21 | } 22 | 23 | pub fn insert_cell(self, key: &str, cell: Cell) -> Self { 24 | let cell_map_ref: MapRef = self.map_ref.get_or_init(self.txn, key); 25 | if cell_map_ref.get(self.txn, CREATED_AT).is_none() { 26 | cell_map_ref.insert(self.txn, CREATED_AT, Any::BigInt(timestamp())); 27 | } 28 | 29 | Any::from(cell).fill(self.txn, &cell_map_ref).unwrap(); 30 | cell_map_ref.insert(self.txn, LAST_MODIFIED, Any::BigInt(timestamp())); 31 | self 32 | } 33 | 34 | /// Override the existing cell's key/value contained in the [Cell] 35 | /// It will create the cell if it's not exist 36 | pub fn insert>(self, key: &str, value: T) -> Self { 37 | let cell = value.into(); 38 | self.insert_cell(key, cell) 39 | } 40 | 41 | pub fn clear(self, key: &str) -> Self { 42 | let cell_map_ref: MapRef = self.map_ref.get_or_init(self.txn, key); 43 | cell_map_ref.clear(self.txn); 44 | 45 | self 46 | } 47 | } 48 | 49 | pub type Cell = HashMap; 50 | pub type CellBuilder = HashMap; 51 | pub type CellUpdate = MapRef; 52 | 53 | pub const CELL_FIELD_TYPE: &str = "field_type"; 54 | pub fn get_field_type_from_cell>(cell: &Cell) -> Option { 55 | let field_type: i64 = cell.get_as(CELL_FIELD_TYPE)?; 56 | Some(T::from(field_type)) 57 | } 58 | 59 | /// Create a new [CellBuilder] with the field type. 60 | pub fn new_cell_builder(field_type: impl Into) -> CellBuilder { 61 | HashMap::from([(CELL_FIELD_TYPE.into(), Any::BigInt(field_type.into()))]) 62 | } 63 | 64 | pub struct RowCell { 65 | pub row_id: RowId, 66 | /// The cell might be empty if no value is written before 67 | pub cell: Option, 68 | } 69 | 70 | impl RowCell { 71 | pub fn new(row_id: RowId, cell: Option) -> Self { 72 | Self { row_id, cell } 73 | } 74 | 75 | pub fn text(&self) -> Option { 76 | self 77 | .cell 78 | .as_ref() 79 | .and_then(|cell| cell.get_as::(CELL_DATA)) 80 | } 81 | } 82 | 83 | impl Deref for RowCell { 84 | type Target = Option; 85 | 86 | fn deref(&self) -> &Self::Target { 87 | &self.cell 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /collab-database/src/rows/comment.rs: -------------------------------------------------------------------------------- 1 | use collab::preclude::Any; 2 | use collab::util::deserialize_i64_from_numeric; 3 | use serde::{Deserialize, Serialize}; 4 | 5 | #[derive(Debug, Serialize, Deserialize)] 6 | pub struct RowComment { 7 | uid: i64, 8 | content: String, 9 | #[serde(deserialize_with = "deserialize_i64_from_numeric")] 10 | created_at: i64, 11 | } 12 | 13 | impl TryFrom for RowComment { 14 | type Error = anyhow::Error; 15 | 16 | fn try_from(value: Any) -> Result { 17 | let mut json = String::new(); 18 | value.to_json(&mut json); 19 | let comment = serde_json::from_str(&json)?; 20 | Ok(comment) 21 | } 22 | } 23 | 24 | impl From for Any { 25 | fn from(item: RowComment) -> Self { 26 | let json = serde_json::to_string(&item).unwrap(); 27 | Any::from_json(&json).unwrap() 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /collab-database/src/rows/mod.rs: -------------------------------------------------------------------------------- 1 | pub use cell::*; 2 | pub use comment::*; 3 | pub use row::*; 4 | pub use row_id::*; 5 | pub use row_meta::*; 6 | pub use row_observer::*; 7 | mod cell; 8 | mod comment; 9 | mod row; 10 | mod row_id; 11 | mod row_meta; 12 | mod row_observer; 13 | -------------------------------------------------------------------------------- /collab-database/src/rows/row_id.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{Display, Formatter}; 2 | use std::ops::Deref; 3 | 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] 7 | pub struct RowId(String); 8 | 9 | impl Display for RowId { 10 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 11 | f.write_str(&self.0.to_string()) 12 | } 13 | } 14 | 15 | impl RowId { 16 | pub fn into_inner(self) -> String { 17 | self.0 18 | } 19 | } 20 | 21 | impl Deref for RowId { 22 | type Target = String; 23 | 24 | fn deref(&self) -> &Self::Target { 25 | &self.0 26 | } 27 | } 28 | 29 | impl From for RowId { 30 | fn from(data: String) -> Self { 31 | Self(data) 32 | } 33 | } 34 | 35 | impl From for String { 36 | fn from(data: RowId) -> Self { 37 | data.0 38 | } 39 | } 40 | 41 | impl From for RowId { 42 | fn from(data: uuid::Uuid) -> Self { 43 | Self(data.to_string()) 44 | } 45 | } 46 | 47 | impl From for RowId { 48 | fn from(data: i32) -> Self { 49 | Self(data.to_string()) 50 | } 51 | } 52 | 53 | impl From for RowId { 54 | fn from(data: i64) -> Self { 55 | Self(data.to_string()) 56 | } 57 | } 58 | 59 | impl From for RowId { 60 | fn from(data: usize) -> Self { 61 | Self(data.to_string()) 62 | } 63 | } 64 | 65 | impl From<&str> for RowId { 66 | fn from(data: &str) -> Self { 67 | Self(data.to_string()) 68 | } 69 | } 70 | 71 | impl AsRef for RowId { 72 | fn as_ref(&self) -> &str { 73 | &self.0 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /collab-database/src/template/checkbox_parse.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /collab-database/src/template/entity.rs: -------------------------------------------------------------------------------- 1 | use crate::entity::{CreateDatabaseParams, FieldType}; 2 | use crate::views::{DatabaseLayout, LayoutSettings}; 3 | use collab::preclude::Any; 4 | 5 | use crate::template::util::create_database_params_from_template; 6 | use std::collections::HashMap; 7 | 8 | pub const CELL_DATA: &str = "data"; 9 | pub struct DatabaseTemplate { 10 | pub database_id: String, 11 | pub view_id: String, 12 | pub fields: Vec, 13 | pub rows: Vec, 14 | pub views: Vec, 15 | } 16 | 17 | impl DatabaseTemplate { 18 | pub fn into_params(self) -> CreateDatabaseParams { 19 | create_database_params_from_template(self) 20 | } 21 | } 22 | 23 | pub struct DatabaseViewTemplate { 24 | pub name: String, 25 | pub layout: DatabaseLayout, 26 | pub layout_settings: LayoutSettings, 27 | pub filters: Vec>, 28 | pub group_settings: Vec>, 29 | pub sorts: Vec>, 30 | } 31 | 32 | pub struct FieldTemplate { 33 | pub field_id: String, 34 | pub name: String, 35 | pub field_type: FieldType, 36 | pub is_primary: bool, 37 | pub type_options: HashMap>, 38 | } 39 | 40 | pub type CellTemplate = HashMap; 41 | pub type CellTemplateData = HashMap; 42 | 43 | #[derive(Debug, Clone)] 44 | pub struct RowTemplate { 45 | pub row_id: String, 46 | pub height: i32, 47 | pub visibility: bool, 48 | pub cells: CellTemplate, 49 | } 50 | -------------------------------------------------------------------------------- /collab-database/src/template/media_parse.rs: -------------------------------------------------------------------------------- 1 | use crate::fields::media_type_option::{MediaCellData, MediaFile, MediaFileType, MediaUploadType}; 2 | use crate::template::builder::FileUrlBuilder; 3 | use crate::template::csv::CSVResource; 4 | use futures::stream::{FuturesOrdered, StreamExt}; 5 | 6 | use std::path::PathBuf; 7 | 8 | use tokio::fs::metadata; 9 | 10 | pub(crate) async fn replace_cells_with_files( 11 | cells: Vec, 12 | database_id: &str, 13 | csv_resource: &Option, 14 | file_url_builder: &Option>, 15 | ) -> Vec> { 16 | match csv_resource { 17 | None => vec![], 18 | Some(csv_resource) => { 19 | let mut futures = FuturesOrdered::new(); 20 | for cell in cells { 21 | futures.push_back(async move { 22 | if cell.is_empty() { 23 | None 24 | } else { 25 | let files = futures::stream::iter(cell.split(',')) 26 | .filter_map(|file| { 27 | let path = csv_resource 28 | .files 29 | .iter() 30 | .find(|resource| resource.ends_with(file)) 31 | .map(PathBuf::from); 32 | 33 | async move { 34 | let path = path?; 35 | if metadata(&path).await.is_ok() { 36 | let file_name = path 37 | .file_name() 38 | .unwrap_or_default() 39 | .to_string_lossy() 40 | .to_string(); 41 | let url = file_url_builder.as_ref()?.build(database_id, &path).await?; 42 | let media_type = MediaFileType::from_file(&path); 43 | 44 | Some(MediaFile::new( 45 | file_name, 46 | url, 47 | MediaUploadType::Cloud, 48 | media_type, 49 | )) 50 | } else { 51 | None 52 | } 53 | } 54 | }) 55 | .collect::>() 56 | .await; 57 | Some(MediaCellData { files }) 58 | } 59 | }); 60 | } 61 | 62 | futures.collect().await 63 | }, 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /collab-database/src/template/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod builder; 2 | pub mod check_list_parse; 3 | pub mod checkbox_parse; 4 | pub mod csv; 5 | pub mod date_parse; 6 | pub mod entity; 7 | pub mod media_parse; 8 | pub mod number_parse; 9 | pub mod option_parse; 10 | pub mod relation_parse; 11 | pub mod summary_parse; 12 | pub mod time_parse; 13 | pub mod timestamp_parse; 14 | pub mod translate_parse; 15 | pub mod util; 16 | -------------------------------------------------------------------------------- /collab-database/src/template/number_parse.rs: -------------------------------------------------------------------------------- 1 | use crate::entity::FieldType; 2 | use crate::rows::{Cell, new_cell_builder}; 3 | use crate::template::entity::CELL_DATA; 4 | use crate::template::util::{ToCellString, TypeOptionCellData}; 5 | use collab::util::AnyMapExt; 6 | use serde::{Deserialize, Serialize}; 7 | 8 | #[derive(Clone, Debug, Default, Deserialize, Serialize)] 9 | pub struct NumberCellData(pub String); 10 | 11 | impl TypeOptionCellData for NumberCellData { 12 | fn is_cell_empty(&self) -> bool { 13 | self.0.is_empty() 14 | } 15 | } 16 | 17 | impl AsRef for NumberCellData { 18 | fn as_ref(&self) -> &str { 19 | &self.0 20 | } 21 | } 22 | 23 | impl From<&Cell> for NumberCellData { 24 | fn from(cell: &Cell) -> Self { 25 | let s = cell.get_as::(CELL_DATA).unwrap_or_default(); 26 | Self(s) 27 | } 28 | } 29 | 30 | impl From for Cell { 31 | fn from(data: NumberCellData) -> Self { 32 | let mut cell = new_cell_builder(FieldType::Number); 33 | cell.insert(CELL_DATA.into(), data.0.into()); 34 | cell 35 | } 36 | } 37 | 38 | impl std::convert::From for NumberCellData { 39 | fn from(s: String) -> Self { 40 | Self(s) 41 | } 42 | } 43 | 44 | impl ToCellString for NumberCellData { 45 | fn to_cell_string(&self) -> String { 46 | self.0.clone() 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /collab-database/src/template/option_parse.rs: -------------------------------------------------------------------------------- 1 | use crate::database::gen_option_id; 2 | use crate::fields::select_type_option::{SelectOption, SelectOptionColor}; 3 | use std::collections::HashSet; 4 | 5 | pub(crate) const SELECT_OPTION_SEPARATOR: &str = ","; 6 | pub(crate) fn replace_cells_with_options_id( 7 | cells: Vec, 8 | options: &[SelectOption], 9 | separator: &str, 10 | ) -> Vec { 11 | cells 12 | .into_iter() 13 | .map(|cell| { 14 | cell 15 | .split(separator) 16 | .map(|part| { 17 | options 18 | .iter() 19 | .find(|option| option.name == part.trim()) 20 | .map_or(part.to_string(), |option| option.id.clone()) 21 | }) 22 | .collect::>() 23 | .join(separator) 24 | }) 25 | .collect() 26 | } 27 | 28 | pub fn build_options_from_cells(cells: &[String]) -> Vec { 29 | let mut option_names = HashSet::new(); 30 | for cell in cells { 31 | cell.split(SELECT_OPTION_SEPARATOR).for_each(|cell| { 32 | let trim_cell = cell.trim(); 33 | if !trim_cell.is_empty() { 34 | option_names.insert(trim_cell.to_string()); 35 | } 36 | }); 37 | } 38 | 39 | let mut options = vec![]; 40 | for (index, name) in option_names.into_iter().enumerate() { 41 | // pick a color by mod 8 42 | let color = SelectOptionColor::from(index % 8); 43 | let option = SelectOption { 44 | id: gen_option_id(), 45 | name, 46 | color, 47 | }; 48 | options.push(option); 49 | } 50 | 51 | options 52 | } 53 | -------------------------------------------------------------------------------- /collab-database/src/template/relation_parse.rs: -------------------------------------------------------------------------------- 1 | use crate::entity::FieldType; 2 | use std::str::FromStr; 3 | 4 | use crate::error::DatabaseError; 5 | use crate::rows::{Cell, RowId, new_cell_builder}; 6 | use crate::template::entity::CELL_DATA; 7 | use crate::template::util::{ToCellString, TypeOptionCellData}; 8 | use serde::{Deserialize, Serialize}; 9 | use std::sync::Arc; 10 | use yrs::Any; 11 | 12 | #[derive(Debug, Clone, Default, Serialize, Deserialize)] 13 | pub struct RelationCellData { 14 | pub row_ids: Vec, 15 | } 16 | 17 | impl FromStr for RelationCellData { 18 | type Err = DatabaseError; 19 | 20 | fn from_str(s: &str) -> Result { 21 | if s.is_empty() { 22 | return Ok(RelationCellData { row_ids: vec![] }); 23 | } 24 | 25 | let ids = s 26 | .split(", ") 27 | .map(|id| id.to_string().into()) 28 | .collect::>(); 29 | 30 | Ok(RelationCellData { row_ids: ids }) 31 | } 32 | } 33 | 34 | impl TypeOptionCellData for RelationCellData { 35 | fn is_cell_empty(&self) -> bool { 36 | self.row_ids.is_empty() 37 | } 38 | } 39 | 40 | impl From<&Cell> for RelationCellData { 41 | fn from(value: &Cell) -> Self { 42 | let row_ids = match value.get(CELL_DATA) { 43 | Some(Any::Array(array)) => array 44 | .iter() 45 | .flat_map(|item| { 46 | if let Any::String(string) = item { 47 | Some(RowId::from(string.clone().to_string())) 48 | } else { 49 | None 50 | } 51 | }) 52 | .collect(), 53 | _ => vec![], 54 | }; 55 | Self { row_ids } 56 | } 57 | } 58 | 59 | impl From for Cell { 60 | fn from(value: RelationCellData) -> Self { 61 | let data = Any::Array(Arc::from( 62 | value 63 | .row_ids 64 | .into_iter() 65 | .map(|id| Any::String(Arc::from(id.to_string()))) 66 | .collect::>(), 67 | )); 68 | let mut cell = new_cell_builder(FieldType::Relation); 69 | cell.insert(CELL_DATA.into(), data); 70 | cell 71 | } 72 | } 73 | 74 | impl ToCellString for RelationCellData { 75 | fn to_cell_string(&self) -> String { 76 | self 77 | .row_ids 78 | .iter() 79 | .map(|id| id.to_string()) 80 | .collect::>() 81 | .join(", ") 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /collab-database/src/template/summary_parse.rs: -------------------------------------------------------------------------------- 1 | use crate::entity::FieldType; 2 | use crate::rows::{Cell, new_cell_builder}; 3 | use crate::template::entity::CELL_DATA; 4 | use crate::template::util::{ToCellString, TypeOptionCellData}; 5 | use collab::util::AnyMapExt; 6 | use serde::{Deserialize, Serialize}; 7 | 8 | #[derive(Default, Debug, Clone, Serialize, Deserialize)] 9 | pub struct SummaryCellData(pub String); 10 | 11 | impl TypeOptionCellData for SummaryCellData { 12 | fn is_cell_empty(&self) -> bool { 13 | self.0.is_empty() 14 | } 15 | } 16 | 17 | impl std::ops::Deref for SummaryCellData { 18 | type Target = String; 19 | 20 | fn deref(&self) -> &Self::Target { 21 | &self.0 22 | } 23 | } 24 | 25 | impl From<&Cell> for SummaryCellData { 26 | fn from(cell: &Cell) -> Self { 27 | Self(cell.get_as::(CELL_DATA).unwrap_or_default()) 28 | } 29 | } 30 | 31 | impl From for Cell { 32 | fn from(data: SummaryCellData) -> Self { 33 | let mut cell = new_cell_builder(FieldType::Summary); 34 | cell.insert(CELL_DATA.into(), data.0.into()); 35 | cell 36 | } 37 | } 38 | 39 | impl ToCellString for SummaryCellData { 40 | fn to_cell_string(&self) -> String { 41 | self.0.clone() 42 | } 43 | } 44 | 45 | impl AsRef for SummaryCellData { 46 | fn as_ref(&self) -> &str { 47 | &self.0 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /collab-database/src/template/time_parse.rs: -------------------------------------------------------------------------------- 1 | use crate::entity::FieldType; 2 | use crate::rows::{Cell, new_cell_builder}; 3 | use crate::template::entity::CELL_DATA; 4 | use crate::template::util::{ToCellString, TypeOptionCellData}; 5 | use collab::util::AnyMapExt; 6 | use serde::{Deserialize, Serialize}; 7 | 8 | #[derive(Clone, Debug, Default, Serialize, Deserialize)] 9 | pub struct TimeCellData(pub Option); 10 | 11 | impl TypeOptionCellData for TimeCellData { 12 | fn is_cell_empty(&self) -> bool { 13 | self.0.is_none() 14 | } 15 | } 16 | 17 | impl From<&Cell> for TimeCellData { 18 | fn from(cell: &Cell) -> Self { 19 | Self( 20 | cell 21 | .get_as::(CELL_DATA) 22 | .and_then(|data| data.parse::().ok()), 23 | ) 24 | } 25 | } 26 | 27 | impl std::convert::From<&str> for TimeCellData { 28 | fn from(s: &str) -> Self { 29 | Self(s.trim().to_string().parse::().ok()) 30 | } 31 | } 32 | 33 | impl ToCellString for TimeCellData { 34 | fn to_cell_string(&self) -> String { 35 | if let Some(time) = self.0 { 36 | time.to_string() 37 | } else { 38 | "".to_string() 39 | } 40 | } 41 | } 42 | 43 | impl From<&TimeCellData> for Cell { 44 | fn from(data: &TimeCellData) -> Self { 45 | let mut cell = new_cell_builder(FieldType::Time); 46 | cell.insert(CELL_DATA.into(), data.to_cell_string().into()); 47 | cell 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /collab-database/src/template/timestamp_parse.rs: -------------------------------------------------------------------------------- 1 | use crate::entity::FieldType; 2 | use crate::rows::{Cell, new_cell_builder}; 3 | use crate::template::entity::CELL_DATA; 4 | 5 | use crate::template::util::{ToCellString, TypeOptionCellData}; 6 | use collab::util::AnyMapExt; 7 | use serde::{Deserialize, Serialize}; 8 | 9 | #[derive(Clone, Debug, Default, Serialize, Deserialize)] 10 | pub struct TimestampCellData { 11 | pub timestamp: Option, 12 | } 13 | 14 | impl TypeOptionCellData for TimestampCellData { 15 | fn is_cell_empty(&self) -> bool { 16 | self.timestamp.is_none() 17 | } 18 | } 19 | 20 | impl TimestampCellData { 21 | pub fn new>>(timestamp: T) -> Self { 22 | Self { 23 | timestamp: timestamp.into(), 24 | } 25 | } 26 | 27 | pub fn to_cell>(&self, field_type: T) -> Cell { 28 | let data: TimestampCellDataWrapper = (field_type.into(), self.clone()).into(); 29 | data.into() 30 | } 31 | } 32 | 33 | impl ToCellString for TimestampCellData { 34 | fn to_cell_string(&self) -> String { 35 | serde_json::to_string(self).unwrap() 36 | } 37 | } 38 | 39 | impl From<&Cell> for TimestampCellData { 40 | fn from(cell: &Cell) -> Self { 41 | let timestamp = cell 42 | .get_as::(CELL_DATA) 43 | .and_then(|data| data.parse::().ok()); 44 | Self { timestamp } 45 | } 46 | } 47 | 48 | /// Wrapper for DateCellData that also contains the field type. 49 | /// Handy struct to use when you need to convert a DateCellData to a Cell. 50 | struct TimestampCellDataWrapper { 51 | data: TimestampCellData, 52 | field_type: FieldType, 53 | } 54 | 55 | impl From<(FieldType, TimestampCellData)> for TimestampCellDataWrapper { 56 | fn from((field_type, data): (FieldType, TimestampCellData)) -> Self { 57 | Self { data, field_type } 58 | } 59 | } 60 | 61 | impl From for Cell { 62 | fn from(wrapper: TimestampCellDataWrapper) -> Self { 63 | let (field_type, data) = (wrapper.field_type, wrapper.data); 64 | let timestamp_string = data.timestamp.unwrap_or_default().to_string(); 65 | 66 | let mut cell = new_cell_builder(field_type); 67 | cell.insert(CELL_DATA.into(), timestamp_string.into()); 68 | cell 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /collab-database/src/template/translate_parse.rs: -------------------------------------------------------------------------------- 1 | use crate::entity::FieldType; 2 | use crate::rows::{Cell, new_cell_builder}; 3 | use crate::template::entity::CELL_DATA; 4 | use crate::template::util::{ToCellString, TypeOptionCellData}; 5 | use collab::util::AnyMapExt; 6 | use serde::{Deserialize, Serialize}; 7 | 8 | #[derive(Default, Debug, Clone, Serialize, Deserialize)] 9 | pub struct TranslateCellData(pub String); 10 | 11 | impl TypeOptionCellData for TranslateCellData { 12 | fn is_cell_empty(&self) -> bool { 13 | self.0.is_empty() 14 | } 15 | } 16 | 17 | impl std::ops::Deref for TranslateCellData { 18 | type Target = String; 19 | 20 | fn deref(&self) -> &Self::Target { 21 | &self.0 22 | } 23 | } 24 | 25 | impl From<&Cell> for TranslateCellData { 26 | fn from(cell: &Cell) -> Self { 27 | Self(cell.get_as(CELL_DATA).unwrap_or_default()) 28 | } 29 | } 30 | 31 | impl From for Cell { 32 | fn from(data: TranslateCellData) -> Self { 33 | let mut cell = new_cell_builder(FieldType::Translate); 34 | cell.insert(CELL_DATA.into(), data.0.into()); 35 | cell 36 | } 37 | } 38 | 39 | impl ToCellString for TranslateCellData { 40 | fn to_cell_string(&self) -> String { 41 | self.0.clone() 42 | } 43 | } 44 | 45 | impl AsRef for TranslateCellData { 46 | fn as_ref(&self) -> &str { 47 | &self.0 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /collab-database/src/util.rs: -------------------------------------------------------------------------------- 1 | use crate::error::DatabaseError; 2 | use collab::entity::EncodedCollab; 3 | use collab::preclude::Collab; 4 | use collab_entity::CollabType; 5 | pub(crate) fn encoded_collab( 6 | collab: &Collab, 7 | collab_type: &CollabType, 8 | ) -> Result { 9 | let encoded_collab = 10 | collab.encode_collab_v1(|collab| collab_type.validate_require_data(collab))?; 11 | Ok(encoded_collab) 12 | } 13 | -------------------------------------------------------------------------------- /collab-database/src/views/calculation.rs: -------------------------------------------------------------------------------- 1 | use collab::preclude::Any; 2 | use std::collections::HashMap; 3 | 4 | pub type CalculationArray = Vec; 5 | pub type CalculationMap = HashMap; 6 | pub type CalculationMapBuilder = HashMap; 7 | -------------------------------------------------------------------------------- /collab-database/src/views/define.rs: -------------------------------------------------------------------------------- 1 | pub const VIEW_ID: &str = "id"; 2 | pub const VIEW_NAME: &str = "name"; 3 | pub const VIEW_DATABASE_ID: &str = "database_id"; 4 | pub const DATABASE_VIEW_LAYOUT: &str = "layout"; 5 | pub const VIEW_LAYOUT_SETTINGS: &str = "layout_settings"; 6 | pub const DATABASE_VIEW_FILTERS: &str = "filters"; 7 | pub const DATABASE_VIEW_GROUPS: &str = "groups"; 8 | pub const DATABASE_VIEW_SORTS: &str = "sorts"; 9 | pub const DATABASE_VIEW_FIELD_SETTINGS: &str = "field_settings"; 10 | pub const DATABASE_VIEW_ROW_ORDERS: &str = "row_orders"; 11 | pub const DATABASE_VIEW_FIELD_ORDERS: &str = "field_orders"; 12 | pub const VIEW_CREATE_AT: &str = "created_at"; 13 | pub const VIEW_MODIFY_AT: &str = "modified_at"; 14 | pub const IS_INLINE: &str = "is_inline"; 15 | pub const VIEW_CALCULATIONS: &str = "calculations"; 16 | -------------------------------------------------------------------------------- /collab-database/src/views/field_settings.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::HashMap, 3 | ops::{Deref, DerefMut}, 4 | }; 5 | 6 | use collab::preclude::{ 7 | Any, FillRef, Map, MapExt, MapRef, ReadTxn, ToJson, TransactionMut, YrsValue, 8 | }; 9 | use collab::util::AnyExt; 10 | use serde::{Deserialize, Serialize}; 11 | 12 | pub type FieldSettingsMap = HashMap; 13 | pub type FieldSettingsMapBuilder = HashMap; 14 | 15 | #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] 16 | pub struct FieldSettingsByFieldIdMap(HashMap); 17 | 18 | impl FieldSettingsByFieldIdMap { 19 | pub fn new() -> Self { 20 | Self::default() 21 | } 22 | 23 | pub fn into_inner(self) -> HashMap { 24 | self.0 25 | } 26 | 27 | pub fn fill_map_ref(self, txn: &mut TransactionMut, map_ref: &MapRef) { 28 | self 29 | .into_inner() 30 | .into_iter() 31 | .for_each(|(field_id, settings)| { 32 | let field_settings_map_ref: MapRef = map_ref.get_or_init_map(txn, field_id); 33 | Any::from(settings) 34 | .fill(txn, &field_settings_map_ref) 35 | .unwrap(); 36 | }); 37 | } 38 | 39 | /// Returns a [FieldSettingsMap] from FieldSettingsByIdMap based on the field ID 40 | pub fn get_settings_with_field_id(&self, field_id: &str) -> Option<&FieldSettingsMap> { 41 | self.get(field_id) 42 | } 43 | } 44 | 45 | impl From<(&'_ T, &MapRef)> for FieldSettingsByFieldIdMap { 46 | fn from(params: (&'_ T, &MapRef)) -> Self { 47 | let mut this = Self::new(); 48 | params.1.iter(params.0).for_each(|(k, v)| { 49 | if let YrsValue::YMap(map_ref) = v { 50 | this.insert(k.to_string(), map_ref.to_json(params.0).into_map().unwrap()); 51 | } 52 | }); 53 | this 54 | } 55 | } 56 | 57 | impl From> for FieldSettingsByFieldIdMap { 58 | fn from(data: HashMap) -> Self { 59 | Self(data) 60 | } 61 | } 62 | 63 | impl Deref for FieldSettingsByFieldIdMap { 64 | type Target = HashMap; 65 | 66 | fn deref(&self) -> &Self::Target { 67 | &self.0 68 | } 69 | } 70 | 71 | impl DerefMut for FieldSettingsByFieldIdMap { 72 | fn deref_mut(&mut self) -> &mut Self::Target { 73 | &mut self.0 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /collab-database/src/views/filter.rs: -------------------------------------------------------------------------------- 1 | use collab::preclude::Any; 2 | use std::collections::HashMap; 3 | 4 | pub type FilterArray = Vec; 5 | pub type FilterMap = HashMap; 6 | pub type FilterMapBuilder = HashMap; 7 | -------------------------------------------------------------------------------- /collab-database/src/views/mod.rs: -------------------------------------------------------------------------------- 1 | mod calculation; 2 | pub mod define; 3 | pub mod field_order; 4 | mod field_settings; 5 | mod filter; 6 | mod group; 7 | mod layout; 8 | mod layout_settings; 9 | mod row_order; 10 | mod sort; 11 | mod view; 12 | mod view_map; 13 | mod view_observer; 14 | 15 | pub use calculation::*; 16 | pub use field_order::*; 17 | pub use field_settings::*; 18 | pub use filter::*; 19 | pub use group::*; 20 | pub use layout::*; 21 | pub use layout_settings::*; 22 | pub use row_order::*; 23 | pub use sort::*; 24 | pub use view::*; 25 | pub use view_map::*; 26 | pub use view_observer::*; 27 | -------------------------------------------------------------------------------- /collab-database/src/views/row_order.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Deref, DerefMut}; 2 | 3 | use collab::preclude::{Any, ArrayRef, ReadTxn, YrsValue}; 4 | use collab::util::deserialize_i32_from_numeric; 5 | use serde::{Deserialize, Serialize}; 6 | 7 | use crate::rows::{Row, RowId}; 8 | use crate::views::{OrderArray, OrderIdentifiable}; 9 | 10 | pub struct RowOrderArray { 11 | array_ref: ArrayRef, 12 | } 13 | 14 | impl OrderArray for RowOrderArray { 15 | type Object = RowOrder; 16 | 17 | fn array_ref(&self) -> &ArrayRef { 18 | &self.array_ref 19 | } 20 | 21 | fn object_from_value(&self, value: YrsValue, txn: &T) -> Option { 22 | row_order_from_value(&value, txn) 23 | } 24 | } 25 | 26 | impl RowOrderArray { 27 | pub fn new(array_ref: ArrayRef) -> Self { 28 | Self { array_ref } 29 | } 30 | } 31 | 32 | impl Deref for RowOrderArray { 33 | type Target = ArrayRef; 34 | 35 | fn deref(&self) -> &Self::Target { 36 | &self.array_ref 37 | } 38 | } 39 | 40 | impl DerefMut for RowOrderArray { 41 | fn deref_mut(&mut self) -> &mut Self::Target { 42 | &mut self.array_ref 43 | } 44 | } 45 | 46 | #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] 47 | pub struct RowOrder { 48 | pub id: RowId, 49 | 50 | #[serde(deserialize_with = "deserialize_i32_from_numeric")] 51 | pub height: i32, 52 | } 53 | 54 | impl OrderIdentifiable for RowOrder { 55 | fn identify_id(&self) -> String { 56 | self.id.to_string() 57 | } 58 | } 59 | 60 | impl RowOrder { 61 | pub fn new(id: RowId, height: i32) -> RowOrder { 62 | Self { id, height } 63 | } 64 | } 65 | 66 | impl From<&Any> for RowOrder { 67 | fn from(any: &Any) -> Self { 68 | let mut json = String::new(); 69 | any.to_json(&mut json); 70 | serde_json::from_str(&json).unwrap() 71 | } 72 | } 73 | 74 | impl From for Any { 75 | fn from(item: RowOrder) -> Self { 76 | let json = serde_json::to_string(&item).unwrap(); 77 | Any::from_json(&json).unwrap() 78 | } 79 | } 80 | 81 | impl From<&Row> for RowOrder { 82 | fn from(row: &Row) -> Self { 83 | Self { 84 | id: row.id.clone(), 85 | height: row.height, 86 | } 87 | } 88 | } 89 | 90 | impl From<&RowOrder> for RowOrder { 91 | fn from(row: &RowOrder) -> Self { 92 | row.clone() 93 | } 94 | } 95 | pub fn row_order_from_value(value: &YrsValue, _txn: &T) -> Option { 96 | if let YrsValue::Any(value) = value { 97 | Some(RowOrder::from(value)) 98 | } else { 99 | None 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /collab-database/src/views/sort.rs: -------------------------------------------------------------------------------- 1 | use collab::preclude::Any; 2 | use std::collections::HashMap; 3 | 4 | pub type SortArray = Vec; 5 | pub type SortMap = HashMap; 6 | pub type SortMapBuilder = HashMap; 7 | -------------------------------------------------------------------------------- /collab-database/src/workspace_database/mod.rs: -------------------------------------------------------------------------------- 1 | pub use body::*; 2 | pub use manager::*; 3 | pub use relation::*; 4 | 5 | mod body; 6 | mod manager; 7 | mod relation; 8 | -------------------------------------------------------------------------------- /collab-database/src/workspace_database/relation/db_relation.rs: -------------------------------------------------------------------------------- 1 | use collab::lock::Mutex; 2 | use collab::preclude::{Collab, Map}; 3 | use std::sync::Arc; 4 | 5 | use crate::workspace_database::relation::RowRelationMap; 6 | 7 | pub struct DatabaseRelation { 8 | #[allow(dead_code)] 9 | inner: Arc>, 10 | row_relation_map: RowRelationMap, 11 | } 12 | 13 | const ROW_RELATION_MAP: &str = "row_relations"; 14 | impl DatabaseRelation { 15 | pub fn new(collab: Arc>) -> DatabaseRelation { 16 | let relation_map = { 17 | let mut lock = collab.blocking_lock(); //FIXME: was that safe before? 18 | let collab = &mut *lock; 19 | let mut txn = collab.context.transact_mut(); 20 | collab.data.get_or_init(&mut txn, ROW_RELATION_MAP) 21 | }; 22 | 23 | Self { 24 | inner: collab, 25 | row_relation_map: RowRelationMap::from_map_ref(relation_map), 26 | } 27 | } 28 | 29 | pub fn row_relations(&self) -> &RowRelationMap { 30 | &self.row_relation_map 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /collab-database/src/workspace_database/relation/mod.rs: -------------------------------------------------------------------------------- 1 | mod db_relation; 2 | mod row_relation; 3 | mod row_relation_map; 4 | 5 | pub use db_relation::*; 6 | pub use row_relation::*; 7 | pub use row_relation_map::*; 8 | -------------------------------------------------------------------------------- /collab-database/tests/database_test/block_test.rs: -------------------------------------------------------------------------------- 1 | use collab_database::rows::CreateRowParams; 2 | use uuid::Uuid; 3 | 4 | use crate::database_test::helper::create_database; 5 | 6 | #[tokio::test] 7 | async fn create_rows_test() { 8 | let database_id = Uuid::new_v4().to_string(); 9 | let mut database_test = create_database(1, &database_id); 10 | for _ in 0..100 { 11 | let row_id = Uuid::new_v4(); 12 | database_test 13 | .create_row_in_view("v1", CreateRowParams::new(row_id, database_id.clone())) 14 | .await 15 | .unwrap(); 16 | } 17 | let rows = database_test.get_rows_for_view("v1").await; 18 | assert_eq!(rows.len(), 100); 19 | } 20 | -------------------------------------------------------------------------------- /collab-database/tests/database_test/cell_type_option_test.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /collab-database/tests/database_test/encode_collab_test.rs: -------------------------------------------------------------------------------- 1 | use crate::database_test::helper::create_database_with_default_data; 2 | use assert_json_diff::assert_json_eq; 3 | use collab::core::collab::CollabOptions; 4 | use collab::core::origin::CollabOrigin; 5 | use collab::preclude::Collab; 6 | 7 | #[tokio::test] 8 | async fn encode_database_collab_test() { 9 | let database_id = uuid::Uuid::new_v4().to_string(); 10 | let database_test = create_database_with_default_data(1, &database_id).await; 11 | 12 | let database_collab = database_test.encode_database_collabs().await.unwrap(); 13 | assert_eq!(database_collab.encoded_row_collabs.len(), 3); 14 | 15 | for (index, encoded_info) in database_collab.encoded_row_collabs.into_iter().enumerate() { 16 | let object_id = database_test.pre_define_row_ids[index].clone(); 17 | let options = CollabOptions::new(object_id.to_string(), database_test.client_id) 18 | .with_data_source(encoded_info.encoded_collab.into()); 19 | let collab = Collab::new_with_options(CollabOrigin::Empty, options).unwrap(); 20 | let json = collab.to_json_value(); 21 | let expected_json = database_test 22 | .get_database_row(&object_id) 23 | .await 24 | .unwrap() 25 | .read() 26 | .await 27 | .to_json_value(); 28 | assert_json_eq!(json, expected_json); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /collab-database/tests/database_test/field_observe_test.rs: -------------------------------------------------------------------------------- 1 | use crate::database_test::helper::{create_database_with_default_data, wait_for_specific_event}; 2 | use crate::helper::setup_log; 3 | use collab_database::fields::FieldChange; 4 | 5 | use collab::lock::Mutex; 6 | use std::sync::Arc; 7 | use std::time::Duration; 8 | use tokio::time::sleep; 9 | 10 | #[tokio::test] 11 | async fn observe_field_update_and_delete_test() { 12 | setup_log(); 13 | let database_id = uuid::Uuid::new_v4().to_string(); 14 | let database_test = create_database_with_default_data(1, &database_id).await; 15 | 16 | let field = database_test.get_fields(None).pop().unwrap(); 17 | 18 | // Update 19 | let cloned_field = field.clone(); 20 | let database_test = Arc::new(Mutex::from(database_test)); 21 | let cloned_database_test = database_test.clone(); 22 | tokio::spawn(async move { 23 | sleep(Duration::from_millis(300)).await; 24 | let mut db = cloned_database_test.lock().await; 25 | db.update_field(&cloned_field.id, |update| { 26 | update.set_name("hello world"); 27 | }); 28 | }); 29 | 30 | let field_change_rx = database_test.lock().await.subscribe_field_change().unwrap(); 31 | wait_for_specific_event(field_change_rx, |event| match event { 32 | FieldChange::DidUpdateField { field } => field.name == "hello world", 33 | _ => false, 34 | }) 35 | .await 36 | .unwrap(); 37 | 38 | // delete 39 | let cloned_field = field.clone(); 40 | let cloned_database_test = database_test.clone(); 41 | tokio::spawn(async move { 42 | sleep(Duration::from_millis(300)).await; 43 | let mut db = cloned_database_test.lock().await; 44 | db.delete_field(&cloned_field.id); 45 | }); 46 | 47 | let cloned_field = field.clone(); 48 | let field_change_rx = database_test.lock().await.subscribe_field_change().unwrap(); 49 | wait_for_specific_event(field_change_rx, |event| match event { 50 | FieldChange::DidDeleteField { field_id } => field_id == &cloned_field.id, 51 | _ => false, 52 | }) 53 | .await 54 | .unwrap(); 55 | } 56 | -------------------------------------------------------------------------------- /collab-database/tests/database_test/mod.rs: -------------------------------------------------------------------------------- 1 | mod block_test; 2 | mod cell_test; 3 | mod cell_type_option_test; 4 | mod encode_collab_test; 5 | mod field_observe_test; 6 | mod field_setting_test; 7 | mod field_test; 8 | mod filter_test; 9 | mod group_test; 10 | pub mod helper; 11 | mod layout_test; 12 | mod restore_test; 13 | mod row_observe_test; 14 | mod row_test; 15 | mod sort_test; 16 | mod type_option_test; 17 | mod view_observe_test; 18 | mod view_test; 19 | -------------------------------------------------------------------------------- /collab-database/tests/helper/mod.rs: -------------------------------------------------------------------------------- 1 | mod util; 2 | pub use util::*; 3 | -------------------------------------------------------------------------------- /collab-database/tests/history_database/020_database.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-database/tests/history_database/020_database.zip -------------------------------------------------------------------------------- /collab-database/tests/history_database/database_020_encode_collab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-database/tests/history_database/database_020_encode_collab -------------------------------------------------------------------------------- /collab-database/tests/main.rs: -------------------------------------------------------------------------------- 1 | pub mod database_test; 2 | pub mod helper; 3 | mod template_test; 4 | pub mod user_test; 5 | -------------------------------------------------------------------------------- /collab-database/tests/template_test/import_csv_test.rs: -------------------------------------------------------------------------------- 1 | use collab::core::collab::default_client_id; 2 | use collab_database::database::Database; 3 | use collab_database::rows::Row; 4 | use collab_database::template::csv::CSVTemplate; 5 | use collab_database::template::entity::CELL_DATA; 6 | use collab_database::workspace_database::NoPersistenceDatabaseCollabService; 7 | use futures::StreamExt; 8 | use std::sync::Arc; 9 | 10 | #[tokio::test] 11 | async fn import_csv_test() { 12 | let csv_data = include_str!("../asset/selected-services-march-2024-quarter-csv.csv"); 13 | let csv_template = CSVTemplate::try_from_reader(csv_data.as_bytes(), false, None).unwrap(); 14 | 15 | let database_template = csv_template.try_into_database_template(None).await.unwrap(); 16 | let service = Arc::new(NoPersistenceDatabaseCollabService { 17 | client_id: default_client_id(), 18 | }); 19 | let database = Database::create_with_template(database_template, service) 20 | .await 21 | .unwrap(); 22 | 23 | let fields = database.get_fields_in_view(&database.get_first_database_view_id().unwrap(), None); 24 | let rows: Vec = database 25 | .get_all_rows(20, None) 26 | .await 27 | .filter_map(|result| async move { result.ok() }) 28 | .collect() 29 | .await; 30 | 31 | let mut reader = csv::Reader::from_reader(csv_data.as_bytes()); 32 | let csv_fields = reader 33 | .headers() 34 | .unwrap() 35 | .iter() 36 | .map(|s| s.to_string()) 37 | .collect::>(); 38 | let csv_rows = reader 39 | .records() 40 | .flat_map(|r| r.ok()) 41 | .map(|record| { 42 | record 43 | .into_iter() 44 | .map(|s| s.to_string()) 45 | .collect::>() 46 | }) 47 | .collect::>>(); 48 | 49 | assert_eq!(rows.len(), csv_rows.len()); 50 | assert_eq!(rows.len(), 1200); 51 | 52 | assert_eq!(fields.len(), csv_fields.len()); 53 | assert_eq!(fields.len(), 14); 54 | 55 | for (index, field) in fields.iter().enumerate() { 56 | assert_eq!(field.name, csv_fields[index]); 57 | } 58 | 59 | for (row_index, row) in rows.iter().enumerate() { 60 | assert_eq!(row.cells.len(), fields.len()); 61 | for (field_index, field) in fields.iter().enumerate() { 62 | let cell = row 63 | .cells 64 | .get(&field.id) 65 | .unwrap() 66 | .get(CELL_DATA) 67 | .cloned() 68 | .unwrap(); 69 | let cell_data = cell.cast::().unwrap(); 70 | assert_eq!( 71 | cell_data, csv_rows[row_index][field_index], 72 | "Row: {}, Field: {}:{}", 73 | row_index, field.name, field_index 74 | ) 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /collab-database/tests/template_test/mod.rs: -------------------------------------------------------------------------------- 1 | mod create_template_test; 2 | mod import_csv_test; 3 | -------------------------------------------------------------------------------- /collab-database/tests/user_test/async_test/mod.rs: -------------------------------------------------------------------------------- 1 | mod script; 2 | 3 | mod flush_test; 4 | mod row_test; 5 | -------------------------------------------------------------------------------- /collab-database/tests/user_test/mod.rs: -------------------------------------------------------------------------------- 1 | mod cell_test; 2 | mod database_test; 3 | pub mod helper; 4 | // mod relation_test; 5 | // mod snapshot_test; 6 | // mod async_test; 7 | mod type_option_test; 8 | -------------------------------------------------------------------------------- /collab-database/tests/user_test/snapshot_test.rs: -------------------------------------------------------------------------------- 1 | use collab_database::rows::CreateRowParams; 2 | use collab_database::views::CreateDatabaseParams; 3 | use collab_plugins::disk::rocksdb::CollabPersistenceConfig; 4 | 5 | use crate::user_test::helper::{user_database_test, user_database_test_with_config}; 6 | 7 | #[tokio::test] 8 | async fn create_database_row_snapshot_test() { 9 | let test = user_database_test_with_config( 10 | 1, 11 | CollabPersistenceConfig::new() 12 | .enable_snapshot(true) 13 | .snapshot_per_update(5), 14 | ); 15 | let database = test 16 | .create_database(CreateDatabaseParams { 17 | database_id: "d1".to_string(), 18 | view_id: "v1".to_string(), 19 | ..Default::default() 20 | }) 21 | .unwrap(); 22 | 23 | let snapshots = test.get_database_snapshots("d1"); 24 | assert!(snapshots.is_empty()); 25 | 26 | for i in 0..10 { 27 | database 28 | .create_row(CreateRowParams { 29 | id: i.into(), 30 | ..Default::default() 31 | }) 32 | .unwrap(); 33 | } 34 | 35 | let snapshots = test.get_database_snapshots("d1"); 36 | assert_eq!(snapshots.len(), 2); 37 | } 38 | 39 | #[tokio::test] 40 | async fn delete_database_snapshot_test() { 41 | let test = user_database_test(1); 42 | let database = test 43 | .create_database(CreateDatabaseParams { 44 | database_id: "d1".to_string(), 45 | view_id: "v1".to_string(), 46 | ..Default::default() 47 | }) 48 | .unwrap(); 49 | 50 | for i in 0..10 { 51 | database 52 | .create_row(CreateRowParams { 53 | id: i.into(), 54 | ..Default::default() 55 | }) 56 | .unwrap(); 57 | } 58 | test.delete_database("d1"); 59 | let snapshots = test.get_database_snapshots("d1"); 60 | assert!(snapshots.is_empty()); 61 | } 62 | -------------------------------------------------------------------------------- /collab-derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "collab-derive" 3 | version = "0.2.0" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | proc-macro2 = "1.0" 10 | quote = "1.0" 11 | syn = { version = "1.0.109", features = ["extra-traits", "visit"] } 12 | yrs.workspace = true 13 | serde_json.workspace = true 14 | 15 | [lib] 16 | name = "collab_derive" 17 | proc-macro = true 18 | -------------------------------------------------------------------------------- /collab-derive/src/collab.rs: -------------------------------------------------------------------------------- 1 | use crate::internal::{ASTContainer, ASTResult}; 2 | use crate::yrs_token::make_yrs_token_steam; 3 | use proc_macro2::TokenStream; 4 | 5 | pub fn expand_derive(input: &syn::DeriveInput) -> Result> { 6 | let ast_result = ASTResult::new(); 7 | let cont = match ASTContainer::from_ast(&ast_result, input) { 8 | Some(cont) => cont, 9 | None => return Err(ast_result.check().unwrap_err()), 10 | }; 11 | 12 | let mut token_stream: TokenStream = TokenStream::default(); 13 | if let Some(yrs_token_stream) = make_yrs_token_steam(&ast_result, &cont) { 14 | token_stream.extend(yrs_token_stream); 15 | } 16 | 17 | ast_result.check()?; 18 | Ok(token_stream) 19 | } 20 | -------------------------------------------------------------------------------- /collab-derive/src/internal/ctxt.rs: -------------------------------------------------------------------------------- 1 | use quote::ToTokens; 2 | use std::{cell::RefCell, fmt::Display, thread}; 3 | 4 | #[derive(Default)] 5 | pub struct ASTResult { 6 | errors: RefCell>>, 7 | } 8 | 9 | impl ASTResult { 10 | pub fn new() -> Self { 11 | ASTResult { 12 | errors: RefCell::new(Some(Vec::new())), 13 | } 14 | } 15 | 16 | pub fn error_spanned_by(&self, obj: A, msg: T) { 17 | self 18 | .errors 19 | .borrow_mut() 20 | .as_mut() 21 | .unwrap() 22 | .push(syn::Error::new_spanned(obj.into_token_stream(), msg)); 23 | } 24 | 25 | pub fn syn_error(&self, err: syn::Error) { 26 | self.errors.borrow_mut().as_mut().unwrap().push(err); 27 | } 28 | 29 | pub fn check(self) -> Result<(), Vec> { 30 | let errors = self.errors.borrow_mut().take().unwrap(); 31 | match errors.len() { 32 | 0 => Ok(()), 33 | _ => Err(errors), 34 | } 35 | } 36 | } 37 | 38 | impl Drop for ASTResult { 39 | fn drop(&mut self) { 40 | if !thread::panicking() && self.errors.borrow().is_some() { 41 | panic!("forgot to check for errors"); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /collab-derive/src/internal/mod.rs: -------------------------------------------------------------------------------- 1 | mod ast; 2 | mod ctxt; 3 | 4 | pub use ast::*; 5 | pub use ctxt::*; 6 | -------------------------------------------------------------------------------- /collab-derive/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod collab; 2 | mod internal; 3 | mod yrs_token; 4 | 5 | #[macro_use] 6 | extern crate quote; 7 | extern crate proc_macro; 8 | 9 | use proc_macro::TokenStream; 10 | use quote::quote; 11 | use syn::parse_macro_input; 12 | use syn::DeriveInput; 13 | 14 | #[proc_macro_derive(Collab, attributes(collab, collab_key))] 15 | pub fn derive_collab(input: TokenStream) -> TokenStream { 16 | let input = parse_macro_input!(input as DeriveInput); 17 | collab::expand_derive(&input) 18 | .unwrap_or_else(to_compile_errors) 19 | .into() 20 | } 21 | 22 | fn to_compile_errors(errors: Vec) -> proc_macro2::TokenStream { 23 | let compile_errors = errors.iter().map(syn::Error::to_compile_error); 24 | quote!(#(#compile_errors)*) 25 | } 26 | -------------------------------------------------------------------------------- /collab-document/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "collab-document" 3 | version = "0.2.0" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | crate-type = ["cdylib", "rlib"] 9 | 10 | [dependencies] 11 | collab = { workspace = true } 12 | collab-entity = { workspace = true } 13 | serde.workspace = true 14 | serde_json.workspace = true 15 | nanoid = "0.4.0" 16 | thiserror = "1.0.30" 17 | anyhow.workspace = true 18 | tracing.workspace = true 19 | arc-swap.workspace = true 20 | tokio = { workspace = true, features = ["sync", "rt"] } 21 | tokio-stream = { version = "0.1.14", features = ["sync"] } 22 | uuid = { version = "1.3.3", features = ["v4", "v5"] } 23 | markdown = "1.0.0-alpha.21" 24 | 25 | [target.'cfg(target_arch = "wasm32")'.dependencies] 26 | getrandom = { version = "0.2", features = ["js"] } 27 | 28 | [dev-dependencies] 29 | tokio = { version = "1.26", features = ["macros", "rt"] } 30 | tempfile = "3.8.0" 31 | tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } 32 | collab-plugins = { workspace = true } 33 | zip = "0.6.6" 34 | futures = "0.3.30" 35 | assert-json-diff = "2.0.2" 36 | yrs.workspace = true 37 | 38 | [features] 39 | verbose_log = [] 40 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod document_parser; 2 | pub mod parsers; 3 | pub mod registry; 4 | pub mod text_utils; 5 | pub mod traits; 6 | 7 | pub use document_parser::*; 8 | pub use parsers::*; 9 | pub use registry::*; 10 | pub use text_utils::*; 11 | pub use traits::*; 12 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/bulleted_list.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{ 2 | BlockParser, DefaultDocumentTextExtractor, DocumentTextExtractor, OutputFormat, ParseContext, 3 | ParseResult, 4 | }; 5 | use crate::blocks::{Block, BlockType}; 6 | use crate::error::DocumentError; 7 | 8 | /// Parse the bulleted list block. 9 | /// 10 | /// Bulleted list block data: 11 | /// delta: delta 12 | pub struct BulletedListParser; 13 | 14 | impl BlockParser for BulletedListParser { 15 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 16 | let text_extractor = DefaultDocumentTextExtractor; 17 | let content = text_extractor.extract_text_from_block(block, context)?; 18 | 19 | let formatted_content = match context.format { 20 | OutputFormat::Markdown => { 21 | let indent = context.get_indent(); 22 | format!("{}* {}", indent, content) 23 | }, 24 | OutputFormat::PlainText => { 25 | let indent = context.get_indent(); 26 | format!("{}{}", indent, content) 27 | }, 28 | }; 29 | 30 | let children_content = self.parse_children(block, context); 31 | 32 | let mut result = formatted_content; 33 | if !children_content.is_empty() { 34 | result.push('\n'); 35 | result.push_str(&children_content); 36 | } 37 | 38 | Ok(ParseResult::new(result)) 39 | } 40 | 41 | fn block_type(&self) -> &'static str { 42 | BlockType::BulletedList.as_str() 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/callout.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use crate::block_parser::{ 4 | BlockParser, DefaultDocumentTextExtractor, DocumentTextExtractor, OutputFormat, ParseContext, 5 | ParseResult, 6 | }; 7 | use crate::blocks::{Block, BlockType}; 8 | use crate::error::DocumentError; 9 | 10 | /// Parse the callout block. 11 | /// 12 | /// Callout block data: 13 | /// icon: string, 14 | /// delta: delta, 15 | pub struct CalloutParser; 16 | 17 | // do not change the key value, it comes from the flutter code. 18 | const ICON_KEY: &str = "icon"; 19 | 20 | impl BlockParser for CalloutParser { 21 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 22 | let text_extractor = DefaultDocumentTextExtractor; 23 | let content = text_extractor.extract_text_from_block(block, context)?; 24 | 25 | let icon = block 26 | .data 27 | .get(ICON_KEY) 28 | .and_then(|v| match v { 29 | Value::String(s) => Some(s.clone()), 30 | _ => None, 31 | }) 32 | .unwrap_or_else(|| "💡".to_string()); // Default icon if none provided 33 | 34 | let formatted_content = match context.format { 35 | OutputFormat::Markdown => { 36 | let indent = context.get_indent(); 37 | format!("{}> {} {}", indent, icon, content) 38 | }, 39 | OutputFormat::PlainText => { 40 | let indent = context.get_indent(); 41 | format!("{}{} {}", indent, icon, content) 42 | }, 43 | }; 44 | 45 | let children_content = self.parse_children(block, context); 46 | 47 | let mut result = formatted_content; 48 | if !children_content.is_empty() { 49 | result.push('\n'); 50 | result.push_str(&children_content); 51 | } 52 | 53 | Ok(ParseResult::new(result)) 54 | } 55 | 56 | fn block_type(&self) -> &'static str { 57 | BlockType::Callout.as_str() 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/code_block.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use crate::block_parser::{ 4 | BlockParser, DefaultDocumentTextExtractor, DocumentTextExtractor, OutputFormat, ParseContext, 5 | ParseResult, 6 | }; 7 | use crate::blocks::{Block, BlockType}; 8 | use crate::error::DocumentError; 9 | 10 | /// Parse the code block. 11 | /// 12 | /// Code block data: 13 | /// language: string 14 | /// delta: delta 15 | pub struct CodeBlockParser; 16 | 17 | // do not change the key value, it comes from the flutter code. 18 | const LANGUAGE_KEY: &str = "language"; 19 | 20 | impl BlockParser for CodeBlockParser { 21 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 22 | let text_extractor = DefaultDocumentTextExtractor; 23 | let content = text_extractor.extract_text_from_block(block, context)?; 24 | 25 | let language = block 26 | .data 27 | .get(LANGUAGE_KEY) 28 | .and_then(|v| match v { 29 | Value::String(s) => Some(s.clone()), 30 | _ => None, 31 | }) 32 | .unwrap_or_default(); 33 | 34 | let formatted_content = match context.format { 35 | OutputFormat::Markdown => { 36 | let indent = context.get_indent(); 37 | format!("{}```{}\n{}\n{}```", indent, language, content, indent) 38 | }, 39 | OutputFormat::PlainText => { 40 | let indent = context.get_indent(); 41 | format!("{}{}", indent, content) 42 | }, 43 | }; 44 | 45 | Ok(ParseResult::new(formatted_content)) 46 | } 47 | 48 | fn block_type(&self) -> &'static str { 49 | BlockType::Code.as_str() 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/divider.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{BlockParser, OutputFormat, ParseContext, ParseResult}; 2 | use crate::blocks::{Block, BlockType}; 3 | use crate::error::DocumentError; 4 | 5 | /// Parse the divider block. 6 | /// 7 | /// Divider blocks have no data and no children. 8 | pub struct DividerParser; 9 | 10 | impl BlockParser for DividerParser { 11 | fn parse(&self, _block: &Block, context: &ParseContext) -> Result { 12 | let formatted_content = match context.format { 13 | OutputFormat::Markdown => { 14 | let indent = context.get_indent(); 15 | format!("{}---", indent) 16 | }, 17 | OutputFormat::PlainText => { 18 | let indent = context.get_indent(); 19 | format!("{}---", indent) 20 | }, 21 | }; 22 | 23 | Ok(ParseResult::new(formatted_content)) 24 | } 25 | 26 | fn block_type(&self) -> &'static str { 27 | BlockType::Divider.as_str() 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/file_block.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use crate::block_parser::{BlockParser, OutputFormat, ParseContext, ParseResult}; 4 | use crate::blocks::{Block, BlockType}; 5 | use crate::error::DocumentError; 6 | 7 | /// Parse the file block. 8 | /// 9 | /// File block data: 10 | /// name: string, 11 | /// url: string 12 | pub struct FileBlockParser; 13 | 14 | // do not change the key values, they come from the flutter code. 15 | const NAME_KEY: &str = "name"; 16 | const URL_KEY: &str = "url"; 17 | 18 | impl BlockParser for FileBlockParser { 19 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 20 | let name = block 21 | .data 22 | .get(NAME_KEY) 23 | .and_then(|v| match v { 24 | Value::String(s) => Some(s.clone()), 25 | _ => None, 26 | }) 27 | .unwrap_or_default(); 28 | 29 | let url = block 30 | .data 31 | .get(URL_KEY) 32 | .and_then(|v| match v { 33 | Value::String(s) => Some(s.clone()), 34 | _ => None, 35 | }) 36 | .unwrap_or_default(); 37 | 38 | let formatted_content = match context.format { 39 | OutputFormat::Markdown => { 40 | let indent = context.get_indent(); 41 | if url.is_empty() { 42 | format!("{}{}", indent, name) 43 | } else { 44 | format!("{}[{}]({})", indent, name, url) 45 | } 46 | }, 47 | OutputFormat::PlainText => { 48 | let indent = context.get_indent(); 49 | if url.is_empty() { 50 | format!("{}{}", indent, name) 51 | } else { 52 | format!("{}{}({})", indent, name, url) 53 | } 54 | }, 55 | }; 56 | 57 | Ok(ParseResult::new(formatted_content)) 58 | } 59 | 60 | fn block_type(&self) -> &'static str { 61 | BlockType::File.as_str() 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/heading.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use crate::block_parser::{ 4 | BlockParser, DefaultDocumentTextExtractor, DocumentTextExtractor, OutputFormat, ParseContext, 5 | ParseResult, 6 | }; 7 | use crate::blocks::{Block, BlockType}; 8 | use crate::error::DocumentError; 9 | 10 | /// Parse the heading block. 11 | /// 12 | /// Heading block data: 13 | /// level: int, 14 | /// delta: delta, 15 | pub struct HeadingParser; 16 | 17 | const MAX_LEVEL: usize = 6; 18 | const MIN_LEVEL: usize = 1; 19 | 20 | // do not change the key value, it comes from the flutter code. 21 | const LEVEL_KEY: &str = "level"; 22 | 23 | impl BlockParser for HeadingParser { 24 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 25 | let text_extractor = DefaultDocumentTextExtractor; 26 | let content = text_extractor.extract_text_from_block(block, context)?; 27 | 28 | let level = block 29 | .data 30 | .get(LEVEL_KEY) 31 | .and_then(|v| match v { 32 | Value::Number(n) => n.as_u64().map(|n| n as usize), 33 | Value::String(s) => s.parse::().ok(), 34 | _ => None, 35 | }) 36 | .unwrap_or(1) 37 | .clamp(MIN_LEVEL, MAX_LEVEL); 38 | 39 | let formatted_content = match context.format { 40 | OutputFormat::Markdown => { 41 | format!("{} {}", "#".repeat(level), content) 42 | }, 43 | OutputFormat::PlainText => content, 44 | }; 45 | 46 | let children_content = self.parse_children(block, context); 47 | 48 | let mut result = formatted_content; 49 | if !children_content.is_empty() { 50 | result.push('\n'); 51 | result.push_str(&children_content); 52 | } 53 | 54 | Ok(ParseResult::new(result)) 55 | } 56 | 57 | fn block_type(&self) -> &'static str { 58 | BlockType::Heading.as_str() 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/image.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{BlockParser, ParseContext, ParseResult}; 2 | use crate::blocks::{Block, BlockType}; 3 | use crate::error::DocumentError; 4 | 5 | /// Parse the image block. 6 | /// 7 | /// Image block data: 8 | /// - url: the image URL 9 | pub struct ImageParser; 10 | 11 | // do not change the key value, it comes from the flutter code. 12 | const URL_KEY: &str = "url"; 13 | 14 | impl BlockParser for ImageParser { 15 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 16 | // Extract the URL from block data 17 | let url = block 18 | .data 19 | .get(URL_KEY) 20 | .and_then(|v| v.as_str()) 21 | .unwrap_or(""); 22 | 23 | let formatted_content = match context.format { 24 | crate::block_parser::OutputFormat::Markdown => { 25 | if url.is_empty() { 26 | "![Image]()".to_string() 27 | } else { 28 | format!("![Image]({})", url) 29 | } 30 | }, 31 | crate::block_parser::OutputFormat::PlainText => url.to_string(), 32 | }; 33 | 34 | let children_content = self.parse_children(block, context); 35 | 36 | let mut result = formatted_content; 37 | if !children_content.is_empty() { 38 | result.push('\n'); 39 | result.push_str(&children_content); 40 | } 41 | 42 | Ok(ParseResult::new(result)) 43 | } 44 | 45 | fn block_type(&self) -> &'static str { 46 | BlockType::Image.as_str() 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/link_preview.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use crate::block_parser::{BlockParser, OutputFormat, ParseContext, ParseResult}; 4 | use crate::blocks::{Block, BlockType}; 5 | use crate::error::DocumentError; 6 | 7 | /// Parse the link preview block. 8 | /// 9 | /// Link preview block data: 10 | /// url: string 11 | pub struct LinkPreviewParser; 12 | 13 | // do not change the key value, it comes from the flutter code. 14 | const URL_KEY: &str = "url"; 15 | 16 | impl BlockParser for LinkPreviewParser { 17 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 18 | let url = block 19 | .data 20 | .get(URL_KEY) 21 | .and_then(|v| match v { 22 | Value::String(s) => Some(s.clone()), 23 | _ => None, 24 | }) 25 | .unwrap_or_default(); 26 | 27 | let formatted_content = match context.format { 28 | OutputFormat::Markdown => { 29 | let indent = context.get_indent(); 30 | if url.is_empty() { 31 | "".to_string() 32 | } else { 33 | format!("{}[{}]({})", indent, url, url) 34 | } 35 | }, 36 | OutputFormat::PlainText => { 37 | let indent = context.get_indent(); 38 | if url.is_empty() { 39 | "".to_string() 40 | } else { 41 | format!("{}{}", indent, url) 42 | } 43 | }, 44 | }; 45 | 46 | Ok(ParseResult::new(formatted_content)) 47 | } 48 | 49 | fn block_type(&self) -> &'static str { 50 | BlockType::LinkPreview.as_str() 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/math_equation.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use crate::block_parser::{BlockParser, OutputFormat, ParseContext, ParseResult}; 4 | use crate::blocks::{Block, BlockType}; 5 | use crate::error::DocumentError; 6 | 7 | /// Parse the math equation block. 8 | /// 9 | /// Math equation block data: 10 | /// formula: string 11 | pub struct MathEquationParser; 12 | 13 | // do not change the key value, it comes from the flutter code. 14 | const FORMULA_KEY: &str = "formula"; 15 | 16 | impl BlockParser for MathEquationParser { 17 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 18 | let formula = block 19 | .data 20 | .get(FORMULA_KEY) 21 | .and_then(|v| match v { 22 | Value::String(s) => Some(s.clone()), 23 | _ => None, 24 | }) 25 | .unwrap_or_default(); 26 | 27 | let formatted_content = match context.format { 28 | OutputFormat::Markdown => { 29 | let indent = context.get_indent(); 30 | if formula.is_empty() { 31 | format!("{}```math\n{}\n{}```", indent, indent, indent) 32 | } else { 33 | format!("{}```math\n{}{}\n{}```", indent, indent, formula, indent) 34 | } 35 | }, 36 | OutputFormat::PlainText => { 37 | let indent = context.get_indent(); 38 | format!("{}{}", indent, formula) 39 | }, 40 | }; 41 | 42 | Ok(ParseResult::new(formatted_content)) 43 | } 44 | 45 | fn block_type(&self) -> &'static str { 46 | BlockType::MathEquation.as_str() 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod bulleted_list; 2 | pub mod callout; 3 | pub mod code_block; 4 | pub mod divider; 5 | pub mod file_block; 6 | pub mod heading; 7 | pub mod image; 8 | pub mod link_preview; 9 | pub mod math_equation; 10 | pub mod numbered_list; 11 | pub mod page; 12 | pub mod paragraph; 13 | pub mod quote_list; 14 | pub mod simple_column; 15 | pub mod simple_columns; 16 | pub mod simple_table; 17 | pub mod simple_table_cell; 18 | pub mod simple_table_row; 19 | pub mod subpage; 20 | pub mod todo_list; 21 | pub mod toggle_list; 22 | 23 | pub use bulleted_list::*; 24 | pub use callout::*; 25 | pub use code_block::*; 26 | pub use divider::*; 27 | pub use file_block::*; 28 | pub use heading::*; 29 | pub use image::*; 30 | pub use link_preview::*; 31 | pub use math_equation::*; 32 | pub use numbered_list::*; 33 | pub use page::*; 34 | pub use paragraph::*; 35 | pub use quote_list::*; 36 | pub use simple_column::*; 37 | pub use simple_columns::*; 38 | pub use simple_table::*; 39 | pub use simple_table_cell::*; 40 | pub use simple_table_row::*; 41 | pub use subpage::*; 42 | pub use todo_list::*; 43 | pub use toggle_list::*; 44 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/numbered_list.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use crate::block_parser::{ 4 | BlockParser, DefaultDocumentTextExtractor, DocumentTextExtractor, OutputFormat, ParseContext, 5 | ParseResult, 6 | }; 7 | use crate::blocks::{Block, BlockType}; 8 | use crate::error::DocumentError; 9 | 10 | /// Parse the numbered list block. 11 | /// 12 | /// Numbered list block data: 13 | /// number: string, 14 | /// delta: delta, 15 | pub struct NumberedListParser; 16 | 17 | // do not change the key value, it comes from the flutter code. 18 | const NUMBER_KEY: &str = "number"; 19 | 20 | impl BlockParser for NumberedListParser { 21 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 22 | let text_extractor = DefaultDocumentTextExtractor; 23 | let content = text_extractor.extract_text_from_block(block, context)?; 24 | 25 | let number = block 26 | .data 27 | .get(NUMBER_KEY) 28 | .and_then(|v| match v { 29 | Value::Number(n) => n.as_u64().map(|n| n as usize), 30 | Value::String(s) => s.parse::().ok(), 31 | _ => None, 32 | }) 33 | .unwrap_or_else(|| context.list_number.unwrap_or(1)); 34 | 35 | let formatted_content = match context.format { 36 | OutputFormat::Markdown => { 37 | let indent = context.get_indent(); 38 | format!("{}{}. {}", indent, number, content) 39 | }, 40 | OutputFormat::PlainText => { 41 | let indent = context.get_indent(); 42 | format!("{}{}. {}", indent, number, content) 43 | }, 44 | }; 45 | 46 | let list_context = context.with_list_context(Some(number + 1)); 47 | let children_content = self.parse_children(block, &list_context); 48 | 49 | let mut result = formatted_content; 50 | if !children_content.is_empty() { 51 | result.push('\n'); 52 | result.push_str(&children_content); 53 | } 54 | 55 | Ok(ParseResult::new(result)) 56 | } 57 | 58 | fn block_type(&self) -> &'static str { 59 | BlockType::NumberedList.as_str() 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/page.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{BlockParser, ParseContext, ParseResult}; 2 | use crate::blocks::{Block, BlockType}; 3 | use crate::error::DocumentError; 4 | 5 | /// Parse the page block. 6 | /// 7 | /// Page block data: 8 | /// - children: blocks 9 | pub struct PageParser; 10 | 11 | impl BlockParser for PageParser { 12 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 13 | let children_content = self.parse_children(block, context); 14 | Ok(ParseResult::new(children_content)) 15 | } 16 | 17 | fn block_type(&self) -> &'static str { 18 | BlockType::Page.as_str() 19 | } 20 | 21 | // Custom parse_children implementation that keeps children at the same depth level (root level) 22 | fn parse_children(&self, block: &Block, context: &ParseContext) -> String { 23 | if block.children.is_empty() { 24 | return "".to_string(); 25 | } 26 | 27 | if let Some(child_ids) = context.document_data.meta.children_map.get(&block.children) { 28 | // Use the same context (same depth) instead of incrementing depth 29 | let child_context = context; 30 | 31 | let result = child_ids 32 | .iter() 33 | .filter_map(|child_id| context.document_data.blocks.get(child_id)) 34 | .filter_map(|child_block| context.parser.parse_block(child_block, child_context).ok()) 35 | .filter(|child_content| !child_content.is_empty()) 36 | .collect::>() 37 | .join("\n"); 38 | 39 | return result; 40 | } 41 | 42 | "".to_string() 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/paragraph.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{ 2 | BlockParser, DefaultDocumentTextExtractor, DocumentTextExtractor, ParseContext, ParseResult, 3 | }; 4 | use crate::blocks::{Block, BlockType}; 5 | use crate::error::DocumentError; 6 | 7 | /// Parse the paragraph block. 8 | /// 9 | /// Paragraph block data: 10 | /// - delta: delta 11 | pub struct ParagraphParser; 12 | 13 | impl BlockParser for ParagraphParser { 14 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 15 | let text_extractor = DefaultDocumentTextExtractor; 16 | let content = text_extractor.extract_text_from_block(block, context)?; 17 | 18 | let children_content = self.parse_children(block, context); 19 | 20 | let mut result = content; 21 | if !children_content.is_empty() { 22 | if !result.is_empty() { 23 | result.push('\n'); 24 | } 25 | result.push_str(&children_content); 26 | } 27 | 28 | Ok(ParseResult::new(result)) 29 | } 30 | 31 | fn block_type(&self) -> &'static str { 32 | BlockType::Paragraph.as_str() 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/quote_list.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{ 2 | BlockParser, DefaultDocumentTextExtractor, DocumentTextExtractor, OutputFormat, ParseContext, 3 | ParseResult, 4 | }; 5 | use crate::blocks::{Block, BlockType}; 6 | use crate::error::DocumentError; 7 | 8 | /// Parse the quote list block. 9 | /// 10 | /// Quote list block is typically used for blockquotes or quoted text. 11 | pub struct QuoteListParser; 12 | 13 | impl BlockParser for QuoteListParser { 14 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 15 | let text_extractor = DefaultDocumentTextExtractor; 16 | let content = text_extractor.extract_text_from_block(block, context)?; 17 | 18 | let formatted_content = match context.format { 19 | OutputFormat::Markdown => { 20 | let indent = context.get_indent(); 21 | format!("{}> {}", indent, content) 22 | }, 23 | OutputFormat::PlainText => { 24 | let indent = context.get_indent(); 25 | format!("{}{}", indent, content) 26 | }, 27 | }; 28 | 29 | let children_content = self.parse_children(block, context); 30 | 31 | let mut result = formatted_content; 32 | if !children_content.is_empty() { 33 | result.push('\n'); 34 | result.push_str(&children_content); 35 | } 36 | 37 | Ok(ParseResult::new(result)) 38 | } 39 | 40 | fn block_type(&self) -> &'static str { 41 | BlockType::Quote.as_str() 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/simple_column.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{BlockParser, ParseContext, ParseResult}; 2 | use crate::blocks::{Block, BlockType}; 3 | use crate::error::DocumentError; 4 | 5 | /// Parse the simple column block. 6 | /// 7 | /// Simple column block: 8 | /// - A container 9 | pub struct SimpleColumnParser; 10 | 11 | impl BlockParser for SimpleColumnParser { 12 | fn parse(&self, _block: &Block, _context: &ParseContext) -> Result { 13 | // simple column block is a container that holds content. 14 | // Return empty content but signal that this block has children. 15 | Ok(ParseResult::container("".to_string())) 16 | } 17 | 18 | fn block_type(&self) -> &'static str { 19 | BlockType::SimpleColumn.as_str() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/simple_columns.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{BlockParser, ParseContext, ParseResult}; 2 | use crate::blocks::{Block, BlockType}; 3 | use crate::error::DocumentError; 4 | 5 | /// Parse the simple columns block. 6 | /// 7 | /// Simple columns block: 8 | /// - A container 9 | pub struct SimpleColumnsParser; 10 | 11 | impl BlockParser for SimpleColumnsParser { 12 | fn parse(&self, _block: &Block, _context: &ParseContext) -> Result { 13 | // simple columns block is a container that holds multiple simple column blocks. 14 | // the children of simple columns are simple column blocks. 15 | Ok(ParseResult::container("".to_string())) 16 | } 17 | 18 | fn block_type(&self) -> &'static str { 19 | BlockType::SimpleColumns.as_str() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/simple_table.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{BlockParser, OutputFormat, ParseContext, ParseResult}; 2 | use crate::blocks::{Block, BlockType}; 3 | use crate::error::DocumentError; 4 | 5 | /// Parse the simple table block. 6 | /// 7 | /// Simple table block: 8 | /// - A container that holds multiple simple table row blocks 9 | pub struct SimpleTableParser; 10 | 11 | impl BlockParser for SimpleTableParser { 12 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 13 | match context.format { 14 | OutputFormat::PlainText => { 15 | // For plain text, just use the default container behavior 16 | Ok(ParseResult::container("".to_string())) 17 | }, 18 | OutputFormat::Markdown => { 19 | // For markdown, we need to handle the table separator row 20 | if block.children.is_empty() { 21 | return Ok(ParseResult::new("".to_string())); 22 | } 23 | 24 | if let Some(child_ids) = context.document_data.meta.children_map.get(&block.children) { 25 | let child_context = context.with_depth(context.depth + 1); 26 | let mut row_contents: Vec = Vec::new(); 27 | 28 | for (row_index, child_id) in child_ids.iter().enumerate() { 29 | if let Some(child_block) = context.document_data.blocks.get(child_id) { 30 | let row_content = context 31 | .parser 32 | .parse_block(child_block, &child_context) 33 | .unwrap_or_default(); 34 | if row_index == 0 && !row_content.is_empty() { 35 | let num_columns = row_content.matches(" | ").count() + 1; 36 | row_contents.push(row_content); 37 | let separator = 38 | format!("|{}|", "------|".repeat(num_columns).trim_end_matches('|')); 39 | row_contents.push(separator); 40 | } else { 41 | row_contents.push(row_content); 42 | } 43 | } 44 | } 45 | 46 | let result = row_contents.join("\n"); 47 | return Ok(ParseResult::new(result)); 48 | } 49 | 50 | Ok(ParseResult::new("".to_string())) 51 | }, 52 | } 53 | } 54 | 55 | fn block_type(&self) -> &'static str { 56 | BlockType::SimpleTable.as_str() 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/simple_table_cell.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{BlockParser, ParseContext, ParseResult}; 2 | use crate::blocks::{Block, BlockType}; 3 | use crate::error::DocumentError; 4 | 5 | /// Parse the simple table cell block. 6 | /// 7 | /// Simple table cell block: 8 | /// - A container that holds content (multiple blocks like paragraphs, headings, etc.) 9 | pub struct SimpleTableCellParser; 10 | 11 | impl BlockParser for SimpleTableCellParser { 12 | fn parse(&self, _block: &Block, _context: &ParseContext) -> Result { 13 | Ok(ParseResult::container("".to_string())) 14 | } 15 | 16 | fn block_type(&self) -> &'static str { 17 | BlockType::SimpleTableCell.as_str() 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/simple_table_row.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{BlockParser, OutputFormat, ParseContext, ParseResult}; 2 | use crate::blocks::{Block, BlockType}; 3 | use crate::error::DocumentError; 4 | 5 | /// Parse the simple table row block. 6 | /// 7 | /// Simple table row block: 8 | /// - A container that holds multiple simple table cell blocks 9 | pub struct SimpleTableRowParser; 10 | 11 | impl BlockParser for SimpleTableRowParser { 12 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 13 | if block.children.is_empty() { 14 | return Ok(ParseResult::new("".to_string())); 15 | } 16 | 17 | if let Some(child_ids) = context.document_data.meta.children_map.get(&block.children) { 18 | let child_context = context.with_depth(context.depth + 1); 19 | 20 | let cell_contents: Vec = child_ids 21 | .iter() 22 | .filter_map(|child_id| context.document_data.blocks.get(child_id)) 23 | .map(|child_block| { 24 | context 25 | .parser 26 | .parse_block(child_block, &child_context) 27 | .unwrap_or_default() 28 | }) 29 | .collect(); 30 | 31 | let result = match context.format { 32 | OutputFormat::PlainText => cell_contents.join("\t"), 33 | OutputFormat::Markdown => { 34 | format!("| {} |", cell_contents.join(" | ")) 35 | }, 36 | }; 37 | 38 | return Ok(ParseResult::new(result)); 39 | } 40 | 41 | Ok(ParseResult::new("".to_string())) 42 | } 43 | 44 | fn block_type(&self) -> &'static str { 45 | BlockType::SimpleTableRow.as_str() 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/subpage.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | 3 | use crate::block_parser::{BlockParser, OutputFormat, ParseContext, ParseResult}; 4 | use crate::blocks::{Block, BlockType}; 5 | use crate::error::DocumentError; 6 | 7 | /// Parse the subpage block. 8 | /// 9 | /// Subpage block data: 10 | /// viewId: string 11 | pub struct SubpageParser; 12 | 13 | // do not change the key value, it comes from the flutter code. 14 | const VIEW_ID_KEY: &str = "viewId"; 15 | 16 | impl BlockParser for SubpageParser { 17 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 18 | let view_id = block 19 | .data 20 | .get(VIEW_ID_KEY) 21 | .and_then(|v| match v { 22 | Value::String(s) => Some(s.clone()), 23 | _ => None, 24 | }) 25 | .unwrap_or_default(); 26 | 27 | let formatted_content = match context.format { 28 | OutputFormat::Markdown => { 29 | let indent = context.get_indent(); 30 | if view_id.is_empty() { 31 | format!("{}[Subpage]", indent) 32 | } else { 33 | format!("{}[Subpage]({})", indent, view_id) 34 | } 35 | }, 36 | OutputFormat::PlainText => { 37 | let indent = context.get_indent(); 38 | if view_id.is_empty() { 39 | "".to_string() 40 | } else { 41 | format!("{}{}", indent, view_id) 42 | } 43 | }, 44 | }; 45 | 46 | Ok(ParseResult::new(formatted_content)) 47 | } 48 | 49 | fn block_type(&self) -> &'static str { 50 | BlockType::SubPage.as_str() 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/todo_list.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{ 2 | BlockParser, DefaultDocumentTextExtractor, DocumentTextExtractor, OutputFormat, ParseContext, 3 | ParseResult, 4 | }; 5 | use crate::blocks::{Block, BlockType}; 6 | use crate::error::DocumentError; 7 | 8 | /// Parse the todo list block. 9 | /// 10 | /// Todo list block data: 11 | /// checked: bool 12 | pub struct TodoListParser; 13 | 14 | // do not change the key value, it comes from the flutter code. 15 | const CHECKED_KEY: &str = "checked"; 16 | 17 | impl BlockParser for TodoListParser { 18 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 19 | let text_extractor = DefaultDocumentTextExtractor; 20 | let content = text_extractor.extract_text_from_block(block, context)?; 21 | 22 | let is_checked = block 23 | .data 24 | .get(CHECKED_KEY) 25 | .and_then(|v| v.as_bool()) 26 | .unwrap_or(false); 27 | 28 | let formatted_content = match context.format { 29 | OutputFormat::Markdown => { 30 | let indent = context.get_indent(); 31 | let checkbox = if is_checked { "[x]" } else { "[ ]" }; 32 | format!("{}- {} {}", indent, checkbox, content) 33 | }, 34 | OutputFormat::PlainText => { 35 | let indent = context.get_indent(); 36 | format!("{}{}", indent, content) 37 | }, 38 | }; 39 | 40 | let children_content = self.parse_children(block, context); 41 | 42 | let mut result = formatted_content; 43 | if !children_content.is_empty() { 44 | result.push('\n'); 45 | result.push_str(&children_content); 46 | } 47 | 48 | Ok(ParseResult::new(result)) 49 | } 50 | 51 | fn block_type(&self) -> &'static str { 52 | BlockType::TodoList.as_str() 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/parsers/toggle_list.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{ 2 | BlockParser, DefaultDocumentTextExtractor, DocumentTextExtractor, OutputFormat, ParseContext, 3 | ParseResult, 4 | }; 5 | use crate::blocks::{Block, BlockType}; 6 | use crate::error::DocumentError; 7 | 8 | /// Parse the toggle list block. 9 | /// 10 | /// Toggle list block data: 11 | /// delta: delta 12 | pub struct ToggleListParser; 13 | 14 | impl BlockParser for ToggleListParser { 15 | fn parse(&self, block: &Block, context: &ParseContext) -> Result { 16 | let text_extractor = DefaultDocumentTextExtractor; 17 | let content = text_extractor.extract_text_from_block(block, context)?; 18 | 19 | let children_content = self.parse_children(block, context); 20 | 21 | let result = match context.format { 22 | OutputFormat::Markdown => { 23 | let indent = context.get_indent(); 24 | if children_content.is_empty() { 25 | format!( 26 | "{}
\n{}{}\n{}
", 27 | indent, indent, content, indent 28 | ) 29 | } else { 30 | format!( 31 | "{}
\n{}{}\n\n{}\n{}
", 32 | indent, indent, content, children_content, indent 33 | ) 34 | } 35 | }, 36 | OutputFormat::PlainText => { 37 | let indent = context.get_indent(); 38 | let mut result = format!("{}{}", indent, content); 39 | if !children_content.is_empty() { 40 | result.push('\n'); 41 | result.push_str(&children_content); 42 | } 43 | result 44 | }, 45 | }; 46 | 47 | Ok(ParseResult::new(result)) 48 | } 49 | 50 | fn block_type(&self) -> &'static str { 51 | BlockType::ToggleList.as_str() 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /collab-document/src/block_parser/registry.rs: -------------------------------------------------------------------------------- 1 | use crate::block_parser::{BlockParser, ParseContext, ParseResult}; 2 | use crate::blocks::Block; 3 | use crate::error::DocumentError; 4 | use std::collections::HashMap; 5 | use std::fmt::{Debug, Formatter}; 6 | use std::sync::Arc; 7 | 8 | #[derive(Clone)] 9 | pub struct BlockParserRegistry { 10 | parsers: HashMap>, 11 | } 12 | 13 | impl Debug for BlockParserRegistry { 14 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 15 | f.debug_struct("BlockParserRegistry") 16 | .field("parsers", &self.parsers.keys().collect::>()) 17 | .finish() 18 | } 19 | } 20 | 21 | impl BlockParserRegistry { 22 | pub fn new() -> Self { 23 | Self { 24 | parsers: HashMap::new(), 25 | } 26 | } 27 | 28 | pub fn register(&mut self, parser: Arc) -> &mut Self { 29 | let block_type = parser.block_type().to_string(); 30 | self.parsers.insert(block_type, parser); 31 | self 32 | } 33 | 34 | pub fn unregister(&mut self, block_type: &str) -> Option> { 35 | self.parsers.remove(block_type) 36 | } 37 | 38 | pub fn get_parser(&self, block_type: &str) -> Option<&Arc> { 39 | self.parsers.get(block_type) 40 | } 41 | 42 | pub fn parse_block( 43 | &self, 44 | block: &Block, 45 | context: &ParseContext, 46 | ) -> Result { 47 | if let Some(parser) = self.get_parser(&block.ty) { 48 | parser.parse(block, context) 49 | } else { 50 | Ok(ParseResult::empty()) 51 | } 52 | } 53 | } 54 | 55 | impl Default for BlockParserRegistry { 56 | fn default() -> Self { 57 | Self::new() 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /collab-document/src/blocks/attr_keys.rs: -------------------------------------------------------------------------------- 1 | use std::str::FromStr; 2 | 3 | #[derive(Debug, Clone, PartialEq, Eq, Hash)] 4 | pub enum AttrKey { 5 | Bold, 6 | Italic, 7 | Strikethrough, 8 | Href, 9 | Code, 10 | Mention, 11 | } 12 | 13 | impl AttrKey { 14 | pub fn as_str(&self) -> &'static str { 15 | match self { 16 | AttrKey::Bold => "bold", 17 | AttrKey::Italic => "italic", 18 | AttrKey::Strikethrough => "strikethrough", 19 | AttrKey::Href => "href", 20 | AttrKey::Code => "code", 21 | AttrKey::Mention => "mention", 22 | } 23 | } 24 | } 25 | 26 | impl FromStr for AttrKey { 27 | type Err = String; 28 | 29 | fn from_str(s: &str) -> Result { 30 | match s { 31 | "bold" => Ok(AttrKey::Bold), 32 | "italic" => Ok(AttrKey::Italic), 33 | "strikethrough" => Ok(AttrKey::Strikethrough), 34 | "href" => Ok(AttrKey::Href), 35 | "code" => Ok(AttrKey::Code), 36 | "mention" => Ok(AttrKey::Mention), 37 | _ => Err(format!("Unknown attribute key: {}", s)), 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /collab-document/src/blocks/mod.rs: -------------------------------------------------------------------------------- 1 | mod attr_keys; 2 | mod block; 3 | mod block_types; 4 | mod children; 5 | mod entities; 6 | mod text; 7 | mod text_entities; 8 | mod utils; 9 | 10 | pub use attr_keys::*; 11 | pub use block::*; 12 | pub use block_types::*; 13 | pub use children::*; 14 | pub use entities::*; 15 | pub use text::*; 16 | pub use text_entities::*; 17 | pub use utils::*; 18 | -------------------------------------------------------------------------------- /collab-document/src/document_awareness.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 4 | pub struct DocumentAwarenessState { 5 | // the fields supported in version 1 contain the user, selection, metadata, and timestamp fields 6 | pub version: i64, 7 | pub user: DocumentAwarenessUser, 8 | pub selection: Option, 9 | // The `metadata` field is an optional field (json string) that can be used to store additional information. 10 | // For example, the user can store the color of the selection in this field 11 | pub metadata: Option, 12 | pub timestamp: i64, 13 | } 14 | 15 | impl DocumentAwarenessState { 16 | pub fn new(version: i64, user: DocumentAwarenessUser) -> Self { 17 | Self { 18 | version, 19 | user, 20 | selection: None, 21 | metadata: None, 22 | timestamp: 0, 23 | } 24 | } 25 | } 26 | 27 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 28 | pub struct DocumentAwarenessUser { 29 | pub uid: i64, 30 | pub device_id: String, 31 | } 32 | 33 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 34 | pub struct DocumentAwarenessSelection { 35 | pub start: DocumentAwarenessPosition, 36 | pub end: DocumentAwarenessPosition, 37 | } 38 | 39 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 40 | pub struct DocumentAwarenessPosition { 41 | pub path: Vec, 42 | pub offset: u64, 43 | } 44 | -------------------------------------------------------------------------------- /collab-document/src/error.rs: -------------------------------------------------------------------------------- 1 | use collab_entity::CollabValidateError; 2 | 3 | #[derive(Debug, thiserror::Error)] 4 | pub enum DocumentError { 5 | #[error(transparent)] 6 | Internal(#[from] anyhow::Error), 7 | 8 | #[error(transparent)] 9 | CollabError(#[from] collab::error::CollabError), 10 | 11 | #[error("Could not create block")] 12 | BlockCreateError, 13 | 14 | #[error("The block already exists")] 15 | BlockAlreadyExists, 16 | 17 | #[error("The block is not found")] 18 | BlockIsNotFound, 19 | 20 | #[error("The page id empty")] 21 | PageIdIsEmpty, 22 | 23 | #[error("Could not convert json to data")] 24 | ConvertDataError, 25 | 26 | #[error("The parent is not found")] 27 | ParentIsNotFound, 28 | 29 | #[error("Could not create the root block due to an unspecified error")] 30 | CreateRootBlockError, 31 | 32 | #[error("Could not delete block")] 33 | DeleteBlockError, 34 | 35 | #[error("text_id or delta is empty")] 36 | TextActionParamsError, 37 | 38 | #[error("Lack of document required data")] 39 | NoRequiredData, 40 | 41 | #[error("The external id is not found")] 42 | ExternalIdIsNotFound, 43 | 44 | #[error("Unable to parse document to plain text")] 45 | ParseDocumentError, 46 | 47 | #[error("Unable to parse markdown to document data")] 48 | ParseMarkdownError, 49 | 50 | #[error("Unable to parse delta json to text delta")] 51 | ParseDeltaJsonToTextDeltaError, 52 | 53 | #[error("No children found")] 54 | NoBlockChildrenFound, 55 | 56 | #[error("Unknown block type: {0}")] 57 | UnknownBlockType(String), 58 | 59 | #[error("Unable to find the page block")] 60 | PageBlockNotFound, 61 | } 62 | 63 | impl From for DocumentError { 64 | fn from(error: CollabValidateError) -> Self { 65 | match error { 66 | CollabValidateError::NoRequiredData(_) => DocumentError::NoRequiredData, 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /collab-document/src/importer/define.rs: -------------------------------------------------------------------------------- 1 | pub const IMAGE_EXTENSIONS: [&str; 6] = ["png", "jpg", "jpeg", "gif", "svg", "webp"]; 2 | 3 | // Data Attribute Keys 4 | pub const DEFAULT_COL_WIDTH: i32 = 150; 5 | pub const DEFAULT_ROW_HEIGHT: i32 = 37; 6 | 7 | // Align 8 | pub const ALIGN_LEFT: &str = "left"; 9 | pub const ALIGN_RIGHT: &str = "right"; 10 | pub const ALIGN_CENTER: &str = "center"; 11 | 12 | // Heading Keys 13 | pub const LEVEL_FIELD: &str = "level"; 14 | 15 | // Code Keys 16 | pub const LANGUAGE_FIELD: &str = "language"; 17 | 18 | // Link Keys 19 | pub const URL_FIELD: &str = "url"; 20 | 21 | // Image Keys 22 | pub const IMAGE_TYPE_FIELD: &str = "image_type"; 23 | pub const EXTERNAL_IMAGE_TYPE: i32 = 2; 24 | 25 | // Math Equation Keys 26 | pub const FORMULA_FIELD: &str = "formula"; 27 | 28 | // Delta Attribute Keys 29 | pub const BOLD_ATTR: &str = "bold"; 30 | pub const ITALIC_ATTR: &str = "italic"; 31 | pub const HREF_ATTR: &str = "href"; 32 | pub const CODE_ATTR: &str = "code"; 33 | pub const FORMULA_ATTR: &str = "formula"; 34 | pub const STRIKETHROUGH_ATTR: &str = "strikethrough"; 35 | pub const INLINE_MATH_SYMBOL: &str = "$"; 36 | 37 | // Table Keys 38 | pub const ROWS_LEN_FIELD: &str = "rowsLen"; 39 | pub const COLS_LEN_FIELD: &str = "colsLen"; 40 | pub const COL_DEFAULT_WIDTH_FIELD: &str = "colDefaultWidth"; 41 | pub const ROW_DEFAULT_HEIGHT_FIELD: &str = "rowDefaultHeight"; 42 | pub const ROW_POSITION_FIELD: &str = "rowPosition"; 43 | pub const COL_POSITION_FIELD: &str = "colPosition"; 44 | 45 | // List Keys 46 | pub const CHECKED_FIELD: &str = "checked"; 47 | pub const START_NUMBER_FIELD: &str = "number"; 48 | 49 | pub const ALIGN_FIELD: &str = "align"; 50 | -------------------------------------------------------------------------------- /collab-document/src/importer/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod define; 2 | mod delta; 3 | pub mod md_importer; 4 | mod util; 5 | -------------------------------------------------------------------------------- /collab-document/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod block_parser; 2 | pub mod blocks; 3 | pub mod document; 4 | pub mod document_awareness; 5 | pub mod document_data; 6 | pub mod error; 7 | pub mod importer; 8 | -------------------------------------------------------------------------------- /collab-document/tests/block_parser/divider_test.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use collab_document::block_parser::parsers::divider::DividerParser; 4 | use collab_document::block_parser::{BlockParser, DocumentParser, OutputFormat, ParseContext}; 5 | use collab_document::blocks::{Block, BlockType}; 6 | 7 | use crate::blocks::block_test_core::{BlockTestCore, generate_id}; 8 | 9 | fn create_divider_block(test: &mut BlockTestCore, parent_id: &str) -> Block { 10 | let data = HashMap::new(); 11 | 12 | let actual_parent_id = if parent_id.is_empty() { 13 | test.get_page().id 14 | } else { 15 | parent_id.to_string() 16 | }; 17 | 18 | let block = Block { 19 | id: generate_id(), 20 | ty: BlockType::Divider.as_str().to_string(), 21 | parent: actual_parent_id, 22 | children: generate_id(), 23 | external_id: None, 24 | external_type: None, 25 | data, 26 | }; 27 | 28 | test.document.insert_block(block, None).unwrap() 29 | } 30 | 31 | #[test] 32 | fn test_divider_parser_markdown_format() { 33 | let mut test = BlockTestCore::new(); 34 | let parser = DividerParser; 35 | 36 | let block = create_divider_block(&mut test, ""); 37 | 38 | let document_data = test.get_document_data(); 39 | let document_parser = DocumentParser::with_default_parsers(); 40 | let context = ParseContext::new(&document_data, &document_parser, OutputFormat::Markdown); 41 | let result = parser.parse(&block, &context).unwrap(); 42 | 43 | assert_eq!(result.content, "---"); 44 | } 45 | 46 | #[test] 47 | fn test_divider_parser_plain_text_format() { 48 | let mut test = BlockTestCore::new(); 49 | let parser = DividerParser; 50 | 51 | let block = create_divider_block(&mut test, ""); 52 | 53 | let document_data = test.get_document_data(); 54 | let document_parser = DocumentParser::with_default_parsers(); 55 | let context = ParseContext::new(&document_data, &document_parser, OutputFormat::PlainText); 56 | let result = parser.parse(&block, &context).unwrap(); 57 | 58 | assert_eq!(result.content, "---"); 59 | } 60 | 61 | #[test] 62 | fn test_divider_parser_with_indent() { 63 | let mut test = BlockTestCore::new(); 64 | let parser = DividerParser; 65 | 66 | let block = create_divider_block(&mut test, ""); 67 | 68 | let document_data = test.get_document_data(); 69 | let document_parser = DocumentParser::with_default_parsers(); 70 | let context = 71 | ParseContext::new(&document_data, &document_parser, OutputFormat::Markdown).with_depth(2); 72 | let result = parser.parse(&block, &context).unwrap(); 73 | 74 | assert_eq!(result.content, " ---"); 75 | } 76 | -------------------------------------------------------------------------------- /collab-document/tests/block_parser/document_parser_test.rs: -------------------------------------------------------------------------------- 1 | use collab_document::block_parser::{ 2 | DocumentParser, DocumentParserDelegate, OutputFormat, ParseContext, 3 | }; 4 | use collab_document::blocks::{Block, BlockType, mention_block_delta}; 5 | use serde_json::json; 6 | use std::collections::HashMap; 7 | use std::sync::Arc; 8 | use yrs::{Any, types::Attrs}; 9 | 10 | use crate::blocks::block_test_core::{BlockTestCore, generate_id}; 11 | 12 | #[derive(Debug)] 13 | struct MentionDelegate; 14 | 15 | impl DocumentParserDelegate for MentionDelegate { 16 | fn handle_text_delta( 17 | &self, 18 | text: &str, 19 | attributes: Option<&Attrs>, 20 | _context: &ParseContext, 21 | ) -> Option { 22 | if text != "$" { 23 | return None; 24 | } 25 | 26 | if let Some(attrs) = attributes { 27 | if let Some(Any::Map(values)) = attrs.get("mention") { 28 | if let Some(Any::String(page_id)) = values.get("page_id") { 29 | return Some(format!("[[{}]]", page_id)); 30 | } 31 | } 32 | } 33 | 34 | None 35 | } 36 | } 37 | 38 | #[test] 39 | fn test_document_parser_with_mention_delegate() { 40 | let delegate = Arc::new(MentionDelegate); 41 | let parser = DocumentParser::with_default_parsers().with_delegate(delegate); 42 | 43 | let mut test = BlockTestCore::new(); 44 | let page = test.get_page(); 45 | let page_id = page.id.as_str(); 46 | 47 | let view_id = "test_page_id"; 48 | let mention_delta = mention_block_delta(view_id); 49 | 50 | let delta_json = json!([ 51 | {"insert": "Mention a page: "}, 52 | mention_delta 53 | ]) 54 | .to_string(); 55 | 56 | let external_id = test.create_text(delta_json); 57 | 58 | let data = HashMap::new(); 59 | let block = Block { 60 | id: generate_id(), 61 | ty: BlockType::Paragraph.as_str().to_string(), 62 | parent: page_id.to_string(), 63 | children: generate_id(), 64 | external_id: Some(external_id), 65 | external_type: Some("text".to_string()), 66 | data, 67 | }; 68 | 69 | test.document.insert_block(block, None).unwrap(); 70 | 71 | let document_data = test.get_document_data(); 72 | let result = parser 73 | .parse_document(&document_data, OutputFormat::PlainText) 74 | .unwrap(); 75 | 76 | let expected = format!("Mention a page: [[{}]]", view_id); 77 | assert_eq!(result.trim(), expected); 78 | 79 | let result_md = parser 80 | .parse_document(&document_data, OutputFormat::Markdown) 81 | .unwrap(); 82 | assert_eq!(result_md.trim(), expected); 83 | } 84 | -------------------------------------------------------------------------------- /collab-document/tests/block_parser/mod.rs: -------------------------------------------------------------------------------- 1 | mod bulleted_list_test; 2 | mod callout_test; 3 | mod code_block_test; 4 | mod divider_test; 5 | mod document_parser_test; 6 | mod file_block_test; 7 | mod heading_test; 8 | mod image_test; 9 | mod link_preview_test; 10 | mod math_equation_test; 11 | mod numbered_list_test; 12 | mod paragraph_test; 13 | mod parser_test; 14 | mod quote_list_test; 15 | mod simple_columns_test; 16 | mod simple_table_test; 17 | mod subpage_test; 18 | mod text_utils_test; 19 | mod todo_list_test; 20 | mod toggle_list_test; 21 | -------------------------------------------------------------------------------- /collab-document/tests/block_parser/paragraph_test.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use collab_document::block_parser::parsers::paragraph::ParagraphParser; 4 | use collab_document::block_parser::{BlockParser, DocumentParser, OutputFormat, ParseContext}; 5 | use collab_document::blocks::{Block, BlockType}; 6 | use serde_json::json; 7 | 8 | use crate::blocks::block_test_core::{BlockTestCore, generate_id}; 9 | 10 | fn create_paragraph_block(test: &mut BlockTestCore, text: String, parent_id: &str) -> Block { 11 | let data = HashMap::new(); 12 | 13 | let delta = json!([{ "insert": text }]).to_string(); 14 | let external_id = test.create_text(delta); 15 | 16 | let actual_parent_id = if parent_id.is_empty() { 17 | test.get_page().id 18 | } else { 19 | parent_id.to_string() 20 | }; 21 | 22 | let block = Block { 23 | id: generate_id(), 24 | ty: BlockType::Paragraph.as_str().to_string(), 25 | parent: actual_parent_id, 26 | children: generate_id(), 27 | external_id: Some(external_id), 28 | external_type: Some("text".to_string()), 29 | data, 30 | }; 31 | 32 | test.document.insert_block(block, None).unwrap() 33 | } 34 | 35 | #[test] 36 | fn test_paragraph_parser_basic_text() { 37 | let mut test = BlockTestCore::new(); 38 | let parser = ParagraphParser; 39 | 40 | let block = create_paragraph_block(&mut test, "Hello AppFlowy".to_string(), ""); 41 | 42 | let document_data = test.get_document_data(); 43 | let document_parser = DocumentParser::with_default_parsers(); 44 | let context = ParseContext::new(&document_data, &document_parser, OutputFormat::Markdown); 45 | let result = parser.parse(&block, &context).unwrap(); 46 | 47 | assert_eq!(result.content, "Hello AppFlowy"); 48 | } 49 | 50 | #[test] 51 | fn test_paragraph_parser_empty_content() { 52 | let mut test = BlockTestCore::new(); 53 | let parser = ParagraphParser; 54 | 55 | let block = create_paragraph_block(&mut test, "".to_string(), ""); 56 | 57 | let document_data = test.get_document_data(); 58 | let document_parser = DocumentParser::with_default_parsers(); 59 | let context = ParseContext::new(&document_data, &document_parser, OutputFormat::Markdown); 60 | let result = parser.parse(&block, &context).unwrap(); 61 | 62 | assert!(result.content.is_empty()); 63 | } 64 | -------------------------------------------------------------------------------- /collab-document/tests/block_parser/text_utils_test.rs: -------------------------------------------------------------------------------- 1 | use collab::preclude::{Any, Attrs}; 2 | use collab_document::block_parser::*; 3 | use std::sync::Arc; 4 | 5 | #[test] 6 | fn test_format_text_with_attributes() { 7 | let mut attrs = Attrs::new(); 8 | attrs.insert(Arc::from("bold"), Any::Bool(true)); 9 | 10 | let result = format_text_with_attributes("test", &attrs); 11 | assert_eq!(result, "**test**"); 12 | 13 | let mut attrs = Attrs::new(); 14 | attrs.insert(Arc::from("italic"), Any::Bool(true)); 15 | let result = format_text_with_attributes("test", &attrs); 16 | assert_eq!(result, "*test*"); 17 | 18 | let mut attrs = Attrs::new(); 19 | attrs.insert(Arc::from("strikethrough"), Any::Bool(true)); 20 | let result = format_text_with_attributes("test", &attrs); 21 | assert_eq!(result, "~~test~~"); 22 | 23 | let mut attrs = Attrs::new(); 24 | attrs.insert( 25 | Arc::from("href"), 26 | Any::String(Arc::from("https://appflowy.io")), 27 | ); 28 | let result = format_text_with_attributes("test", &attrs); 29 | assert_eq!(result, "[test](https://appflowy.io)"); 30 | 31 | let mut attrs = Attrs::new(); 32 | attrs.insert(Arc::from("code"), Any::Bool(true)); 33 | let result = format_text_with_attributes("test", &attrs); 34 | assert_eq!(result, "`test`"); 35 | } 36 | 37 | #[test] 38 | fn test_text_extractor_basic() { 39 | let delta_json = r#"[{"insert": "Hello AppFlowy"}]"#; 40 | let result = DefaultDocumentTextExtractor 41 | .extract_plain_text_from_delta_with_context(delta_json, None) 42 | .unwrap(); 43 | assert_eq!(result, "Hello AppFlowy"); 44 | } 45 | 46 | #[test] 47 | fn test_text_extractor_delta_parsing() { 48 | let delta_json = r#"[ 49 | {"insert": "Hello", "attributes": {"bold": true}}, 50 | {"insert": " "}, 51 | {"insert": "AppFlowy", "attributes": {"italic": true}} 52 | ]"#; 53 | 54 | let plain_result = DefaultDocumentTextExtractor 55 | .extract_plain_text_from_delta_with_context(delta_json, None) 56 | .unwrap(); 57 | assert_eq!(plain_result, "Hello AppFlowy"); 58 | 59 | let markdown_result = DefaultDocumentTextExtractor 60 | .extract_markdown_text_from_delta_with_context(delta_json, None) 61 | .unwrap(); 62 | assert_eq!(markdown_result, "**Hello** *AppFlowy*"); 63 | } 64 | 65 | #[test] 66 | fn test_text_extractor_mentions() { 67 | // Test delta with mentions - mentions should be filtered out in plain text 68 | let delta_json = r#"[ 69 | {"insert": "Mention a page "}, 70 | {"insert": "$", "attributes": {"mention": "@page_id"}} 71 | ]"#; 72 | 73 | let plain_result = DefaultDocumentTextExtractor 74 | .extract_plain_text_from_delta_with_context(delta_json, None) 75 | .unwrap(); 76 | assert_eq!(plain_result, "Mention a page $"); 77 | } 78 | -------------------------------------------------------------------------------- /collab-document/tests/blocks/mod.rs: -------------------------------------------------------------------------------- 1 | mod block_test; 2 | pub mod block_test_core; 3 | mod text_test; 4 | -------------------------------------------------------------------------------- /collab-document/tests/conversions/mod.rs: -------------------------------------------------------------------------------- 1 | mod plain_text_test; 2 | -------------------------------------------------------------------------------- /collab-document/tests/conversions/plain_text_test.rs: -------------------------------------------------------------------------------- 1 | use collab_document::{blocks::Block, document::Document}; 2 | use nanoid::nanoid; 3 | 4 | use crate::util::DocumentTest; 5 | 6 | #[test] 7 | fn plain_text_1_test() { 8 | let doc_id = "1"; 9 | let test = DocumentTest::new(1, doc_id); 10 | let mut document = test.document; 11 | let paragraphs = vec![ 12 | "Welcome to AppFlowy!".to_string(), 13 | "Here are the basics".to_string(), 14 | "Click anywhere and just start typing.".to_string(), 15 | "Highlight any text, and use the editing menu to _style_ **your** writing `however` you ~~like.~~".to_string(), 16 | "As soon as you type `/` a menu will pop up. Select different types of content blocks you can add.".to_string(), 17 | "Type `/` followed by `/bullet` or `/num` to create a list.".to_string(), 18 | "Click `+ New Page `button at the bottom of your sidebar to add a new page.".to_string(), 19 | "Click `+` next to any page title in the sidebar to quickly add a new subpage, `Document`, `Grid`, or `Kanban Board`.".to_string(), 20 | ]; 21 | insert_paragraphs(&mut document, paragraphs.clone()); 22 | 23 | let splitted = document.to_plain_text(); 24 | // remove the empty lines at the beginning and the end 25 | // the first one and the last one are empty 26 | assert_eq!(splitted.len(), 8); 27 | 28 | for i in 0..splitted.len() { 29 | assert_eq!(splitted[i], paragraphs[i]); 30 | } 31 | } 32 | 33 | fn insert_paragraphs(document: &mut Document, paragraphs: Vec) { 34 | let page_id = document.get_page_id().unwrap(); 35 | let mut prev_id = "".to_string(); 36 | for paragraph in paragraphs { 37 | let block_id = nanoid!(6); 38 | let text_id = nanoid!(6); 39 | let block = Block { 40 | id: block_id.clone(), 41 | ty: "paragraph".to_owned(), 42 | parent: page_id.clone(), 43 | children: "".to_string(), 44 | external_id: Some(text_id.clone()), 45 | external_type: Some("text".to_owned()), 46 | data: Default::default(), 47 | }; 48 | 49 | document.insert_block(block, Some(prev_id.clone())).unwrap(); 50 | prev_id.clone_from(&block_id); 51 | 52 | document.apply_text_delta(&text_id, format!(r#"[{{"insert": "{}"}}]"#, paragraph)); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /collab-document/tests/document/document_data_test.rs: -------------------------------------------------------------------------------- 1 | use collab::core::collab::{CollabOptions, default_client_id}; 2 | use collab::core::origin::CollabOrigin; 3 | use collab::preclude::Collab; 4 | use collab_document::document::Document; 5 | use collab_document::document_data::default_document_data; 6 | 7 | #[test] 8 | fn get_default_data_test() { 9 | let document_id = "1"; 10 | let data = default_document_data(document_id); 11 | assert!(!data.page_id.is_empty()); 12 | assert!(!data.blocks.is_empty()); 13 | assert!(!data.meta.children_map.is_empty()); 14 | assert!(data.meta.text_map.is_some()); 15 | assert!(data.meta.text_map.is_some()); 16 | assert_eq!(data.meta.text_map.unwrap().len(), 1); 17 | 18 | let document_id = "2"; 19 | let data = default_document_data(document_id); 20 | println!("{:?}", data); 21 | assert!(!data.page_id.is_empty()); 22 | assert_eq!(data.blocks.len(), 2); 23 | assert_eq!(data.meta.children_map.len(), 2); 24 | assert!(data.meta.text_map.is_some()); 25 | assert_eq!(data.meta.text_map.unwrap().len(), 1); 26 | } 27 | 28 | #[test] 29 | fn validate_document_data() { 30 | let document_id = "1"; 31 | let document_data = default_document_data(document_id); 32 | let document = Document::create(document_id, document_data, default_client_id()).unwrap(); 33 | assert!(document.validate().is_ok()); 34 | 35 | let options = CollabOptions::new(document_id.to_string(), default_client_id()); 36 | let new_collab = Collab::new_with_options(CollabOrigin::Empty, options).unwrap(); 37 | let result = Document::open(new_collab); 38 | assert!(result.is_err()) 39 | } 40 | -------------------------------------------------------------------------------- /collab-document/tests/document/mod.rs: -------------------------------------------------------------------------------- 1 | mod awareness_test; 2 | mod document_data_test; 3 | mod document_test; 4 | mod redo_undo_test; 5 | mod restore_test; 6 | -------------------------------------------------------------------------------- /collab-document/tests/history_document/020_document.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-document/tests/history_document/020_document.zip -------------------------------------------------------------------------------- /collab-document/tests/importer/mod.rs: -------------------------------------------------------------------------------- 1 | mod md_importer_customer_test; 2 | mod md_importer_test; 3 | pub mod util; 4 | -------------------------------------------------------------------------------- /collab-document/tests/importer/util.rs: -------------------------------------------------------------------------------- 1 | use collab_document::blocks::{Block, DocumentData}; 2 | use collab_document::importer::md_importer::MDImporter; 3 | use serde_json::Value; 4 | 5 | pub(crate) fn markdown_to_document_data(md: T) -> DocumentData { 6 | let importer = MDImporter::new(None); 7 | let result = importer.import("test_document", md.to_string()); 8 | result.unwrap() 9 | } 10 | 11 | pub(crate) fn parse_json(s: &str) -> Value { 12 | serde_json::from_str(s).unwrap() 13 | } 14 | 15 | pub(crate) fn get_page_block(document_data: &DocumentData) -> Block { 16 | document_data 17 | .blocks 18 | .values() 19 | .find(|b| b.ty == "page") 20 | .unwrap() 21 | .clone() 22 | } 23 | 24 | pub(crate) fn get_block(document_data: &DocumentData, block_id: &str) -> Block { 25 | document_data.blocks.get(block_id).unwrap().clone() 26 | } 27 | 28 | pub(crate) fn get_block_by_type(document_data: &DocumentData, block_type: &str) -> Block { 29 | document_data 30 | .blocks 31 | .values() 32 | .find(|b| b.ty == block_type) 33 | .unwrap() 34 | .clone() 35 | } 36 | 37 | pub(crate) fn get_children_blocks(document_data: &DocumentData, block_id: &str) -> Vec { 38 | let block = get_block(document_data, block_id); 39 | let children_ids = document_data.meta.children_map.get(&block.id).unwrap(); 40 | children_ids 41 | .iter() 42 | .map(|id| get_block(document_data, id)) 43 | .collect() 44 | } 45 | 46 | pub(crate) fn get_delta(document_data: &DocumentData, block_id: &str) -> String { 47 | let delta = document_data 48 | .meta 49 | .text_map 50 | .as_ref() 51 | .unwrap() 52 | .get(block_id) 53 | .unwrap(); 54 | delta.clone() 55 | } 56 | 57 | pub(crate) fn get_delta_json(document_data: &DocumentData, block_id: &str) -> Value { 58 | let delta = get_delta(document_data, block_id); 59 | parse_json(&delta) 60 | } 61 | 62 | // Prints all child blocks of the page block for debugging purposes. 63 | #[allow(dead_code)] 64 | pub(crate) fn dump_page_blocks(document_data: &DocumentData) { 65 | let page_block = get_page_block(document_data); 66 | let children_blocks = get_children_blocks(document_data, &page_block.id); 67 | for block in children_blocks { 68 | println!("{:?}", block); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /collab-document/tests/main.rs: -------------------------------------------------------------------------------- 1 | #[cfg(not(target_arch = "wasm32"))] 2 | mod block_parser; 3 | #[cfg(not(target_arch = "wasm32"))] 4 | mod blocks; 5 | #[cfg(not(target_arch = "wasm32"))] 6 | mod document; 7 | #[cfg(not(target_arch = "wasm32"))] 8 | mod util; 9 | 10 | #[cfg(not(target_arch = "wasm32"))] 11 | mod conversions; 12 | 13 | #[cfg(not(target_arch = "wasm32"))] 14 | mod importer; 15 | -------------------------------------------------------------------------------- /collab-entity/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "collab-entity" 3 | version = "0.2.0" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | crate-type = ["cdylib", "rlib"] 9 | 10 | [dependencies] 11 | uuid = { version = "1.3.3", features = ["v4"] } 12 | serde.workspace = true 13 | serde_json.workspace = true 14 | serde_repr = "0.1" 15 | collab = { workspace = true } 16 | anyhow.workspace = true 17 | bytes = { workspace = true, features = ["serde"] } 18 | prost = "0.13.3" 19 | thiserror = "1.0.61" 20 | 21 | [build-dependencies] 22 | prost-build = "0.12" 23 | walkdir = ">=2.0.0" 24 | protoc-bin-vendored = "3.0.0" 25 | 26 | [target.'cfg(target_arch = "wasm32")'.dependencies] 27 | getrandom = { version = "0.2", features = ["js"] } 28 | 29 | 30 | -------------------------------------------------------------------------------- /collab-entity/build.rs: -------------------------------------------------------------------------------- 1 | use std::io::Result; 2 | use std::process::Command; 3 | use walkdir::WalkDir; 4 | 5 | fn compile_proto_files(proto_files: &[String]) -> Result<()> { 6 | prost_build::Config::new() 7 | .protoc_arg("--experimental_allow_proto3_optional") 8 | .out_dir("src/proto") 9 | .compile_protos(proto_files, &["proto/"]) 10 | } 11 | 12 | fn main() -> Result<()> { 13 | let mut proto_files = Vec::new(); 14 | for e in WalkDir::new("proto").into_iter().filter_map(|e| e.ok()) { 15 | if e.metadata().unwrap().is_file() { 16 | proto_files.push(e.path().display().to_string()); 17 | } 18 | } 19 | 20 | for proto_file in &proto_files { 21 | println!("cargo:rerun-if-changed={}", proto_file); 22 | } 23 | 24 | // If the `PROTOC` environment variable is set, don't use vendored `protoc` 25 | std::env::var("PROTOC").map(|_| ()).unwrap_or_else(|_| { 26 | let protoc_path = protoc_bin_vendored::protoc_bin_path().expect("protoc bin path"); 27 | let protoc_path_str = protoc_path.to_str().expect("protoc path to str"); 28 | 29 | // Set the `PROTOC` environment variable to the path of the `protoc` binary. 30 | unsafe { 31 | std::env::set_var("PROTOC", protoc_path_str); 32 | } 33 | }); 34 | 35 | compile_proto_files(&proto_files).expect("unable to compile proto files"); 36 | 37 | let generated_files = std::fs::read_dir("src/proto")? 38 | .filter_map(Result::ok) 39 | .filter(|entry| { 40 | entry 41 | .path() 42 | .extension() 43 | .map(|ext| ext == "rs") 44 | .unwrap_or(false) 45 | }) 46 | .map(|entry| entry.path().display().to_string()); 47 | for generated_file in generated_files { 48 | Command::new("rustfmt").arg(generated_file).status()?; 49 | } 50 | Ok(()) 51 | } 52 | -------------------------------------------------------------------------------- /collab-entity/proto/collab/common.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package collab; 4 | 5 | // Originating from an AppFlowy Client. 6 | message ClientOrigin { 7 | // User id. 8 | int64 uid = 1; 9 | // Device id. 10 | string device_id = 2; 11 | } 12 | 13 | // Unknown origin. 14 | message EmptyOrigin {} 15 | 16 | // Originating from the AppFlowy Server. 17 | message ServerOrigin {} 18 | 19 | // Origin of a collab message. 20 | message CollabOrigin { 21 | oneof origin { 22 | EmptyOrigin empty = 1; 23 | ClientOrigin client = 2; 24 | ServerOrigin server = 3; 25 | } 26 | } 27 | 28 | // Collab Type. 29 | enum CollabType { 30 | COLLAB_TYPE_UNKNOWN = 0; 31 | COLLAB_TYPE_DOCUMENT = 1; 32 | COLLAB_TYPE_DATABASE = 2; 33 | COLLAB_TYPE_WORKSPACE_DATABASE = 3; 34 | COLLAB_TYPE_FOLDER = 4; 35 | COLLAB_TYPE_DATABASE_ROW = 5; 36 | COLLAB_TYPE_USER_AWARENESS = 6; 37 | } 38 | -------------------------------------------------------------------------------- /collab-entity/proto/collab/encoding.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package collab; 4 | 5 | // yrs encoder version. 6 | enum EncoderVersion { 7 | ENCODER_VERSION_UNKNOWN = 0; 8 | ENCODER_VERSION_V1 = 1; 9 | ENCODER_VERSION_V2 = 2; 10 | } 11 | 12 | // Encoded collaborative document. 13 | message EncodedCollab { 14 | // yrs state vector 15 | bytes state_vector = 1; 16 | // yrs document state 17 | bytes doc_state = 2; 18 | // yrs encoder version used for the state vector and doc state 19 | EncoderVersion encoder_version = 3; 20 | } 21 | -------------------------------------------------------------------------------- /collab-entity/proto/collab/params.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | import "collab/common.proto"; 3 | 4 | package collab; 5 | 6 | // Types of embeddings content. 7 | enum EmbeddingContentType { 8 | // Unknown content type. 9 | EMBEDDING_CONTENT_TYPE_UNKNOWN = 0; 10 | // Plain text 11 | EMBEDDING_CONTENT_TYPE_PLAIN_TEXT = 1; 12 | } 13 | 14 | // Embeddings and the associated collab metadata. 15 | message CollabEmbeddingsParams { 16 | // Fragment id. 17 | string fragment_id = 1; 18 | // Collab object id. 19 | string object_id = 2; 20 | // Collab type. 21 | CollabType collab_type = 3; 22 | // Embedding content type. 23 | EmbeddingContentType content_type = 4; 24 | // Embedding content as string. 25 | string content = 5; 26 | // Embedding as float array. 27 | repeated float embedding = 6; 28 | } 29 | 30 | // Wrapper over a collection of embeddings, together with metadata associated on the collection level. 31 | message CollabEmbeddings { 32 | // OpenAPI tokens consumed. 33 | uint32 tokens_consumed = 1; 34 | // List of embeddings. 35 | repeated CollabEmbeddingsParams embeddings = 2; 36 | } 37 | 38 | // Payload for sending and receive collab over http. 39 | message CollabParams { 40 | string object_id = 1; 41 | // Serialized EncodedCollab object, which could either be in bincode or protobuf serialization format. 42 | bytes encoded_collab = 2; 43 | // Collab type. 44 | CollabType collab_type = 3; 45 | // Document embeddings. 46 | optional CollabEmbeddings embeddings = 4; 47 | } 48 | 49 | // Payload for creating batch of collab over http. 50 | message BatchCreateCollabParams { 51 | // Workspace id. 52 | string workspace_id = 1; 53 | // List of collab params. 54 | repeated CollabParams params_list = 2; 55 | } 56 | 57 | // Payload for creating new collab or update existing collab over http. 58 | message CreateCollabParams { 59 | // Workspace id. 60 | string workspace_id = 1; 61 | // Object id. 62 | string object_id = 2; 63 | // Serialized EncodedCollab object, which could either be in bincode or protobuf serialization format. 64 | bytes encoded_collab = 3; 65 | // Collab type. 66 | CollabType collab_type = 4; 67 | } 68 | -------------------------------------------------------------------------------- /collab-entity/proto/collab/protocol.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | import "collab/common.proto"; 3 | 4 | package collab; 5 | 6 | // Message sent when the origin attempt to sync the payload with a collab document. 7 | message InitSync { 8 | // Message origin. 9 | CollabOrigin origin = 1; 10 | // Object id for the collab. 11 | string object_id = 2; 12 | // Collab type. 13 | CollabType collab_type = 3; 14 | // Workspace which the collab belongs to. 15 | string workspace_id = 4; 16 | // Message id for the sync. 17 | uint64 msg_id = 5; 18 | // Encoded yrs document state vector. 19 | bytes payload = 6; 20 | } 21 | 22 | // Update message sent from the origin to the collab. 23 | message UpdateSync { 24 | // Message origin. 25 | CollabOrigin origin = 1; 26 | // Object id for the collab. 27 | string object_id = 2; 28 | // Message id for the sync. 29 | uint64 msg_id = 3; 30 | // Encoded yrs updates. 31 | bytes payload = 4; 32 | } 33 | 34 | // Metadata for ack message, to be deprecated. 35 | message AckMeta { 36 | string data = 1; 37 | uint64 msg_id = 2; 38 | } 39 | 40 | message CollabAck { 41 | CollabOrigin origin = 1; 42 | string object_id = 2; 43 | // deprecated 44 | AckMeta meta = 3; 45 | bytes payload = 4; 46 | uint32 code = 5; 47 | uint64 msg_id = 6; 48 | uint32 seq_num = 7; 49 | } 50 | 51 | message ServerInit { 52 | CollabOrigin origin = 1; 53 | string object_id = 2; 54 | uint64 msg_id = 3; 55 | bytes payload = 4; 56 | } 57 | 58 | message AwarenessSync { 59 | CollabOrigin origin = 1; 60 | string object_id = 2; 61 | bytes payload = 3; 62 | } 63 | 64 | message BroadcastSync { 65 | CollabOrigin origin = 1; 66 | string object_id = 2; 67 | bytes payload = 3; 68 | uint32 seq_num = 4; 69 | } 70 | 71 | // Wrapper for init sync, for the case when the client is the origin. 72 | message ClientInitSync { 73 | InitSync data = 1; 74 | } 75 | 76 | // Wrapper for update sync, for the case when the client is the origin. 77 | message ClientUpdateSync { 78 | UpdateSync data = 1; 79 | } 80 | -------------------------------------------------------------------------------- /collab-entity/proto/collab/pubsub.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package collab; 4 | 5 | // Identifier of an active collab document sent over pubsub 6 | message ActiveCollabID { 7 | // Workspace id the active collab belongs to. 8 | string workspace_id = 1; 9 | // Object id 10 | string oid = 2; 11 | } 12 | 13 | // Update content sent over pubsub 14 | message CollabUpdateEvent { 15 | oneof update { 16 | // yrs update in encoded form v1 17 | bytes update_v1 = 1; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /collab-entity/src/define.rs: -------------------------------------------------------------------------------- 1 | // Document 2 | pub const DOCUMENT_ROOT: &str = "document"; 3 | 4 | // Folder 5 | pub const FOLDER: &str = "folder"; 6 | pub const FOLDER_META: &str = "meta"; 7 | pub const FOLDER_WORKSPACE_ID: &str = "current_workspace"; 8 | 9 | // Database 10 | pub const WORKSPACE_DATABASES: &str = "databases"; 11 | pub const DATABASE: &str = "database"; 12 | pub const DATABASE_ID: &str = "id"; 13 | pub const DATABASE_METAS: &str = "metas"; 14 | pub const DATABASE_INLINE_VIEW: &str = "iid"; 15 | pub const DATABASE_ROW_DATA: &str = "data"; 16 | pub const DATABASE_ROW_ID: &str = "id"; 17 | 18 | // User Awareness 19 | pub const USER_AWARENESS: &str = "user_awareness"; 20 | -------------------------------------------------------------------------------- /collab-entity/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub use collab_object::*; 2 | 3 | mod collab_object; 4 | pub mod define; 5 | pub mod proto; 6 | pub mod reminder; 7 | 8 | pub use collab::entity::*; 9 | -------------------------------------------------------------------------------- /collab-entity/src/proto/mod.rs: -------------------------------------------------------------------------------- 1 | #[rustfmt::skip] 2 | pub mod collab; 3 | -------------------------------------------------------------------------------- /collab-folder/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | edition = "2024" 3 | name = "collab-folder" 4 | version = "0.2.0" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | crate-type = ["cdylib", "rlib"] 9 | 10 | [dependencies] 11 | anyhow.workspace = true 12 | chrono.workspace = true 13 | collab = { workspace = true } 14 | collab-entity.workspace = true 15 | serde.workspace = true 16 | serde_json.workspace = true 17 | serde_repr = "0.1" 18 | thiserror = "1.0.30" 19 | tokio = { workspace = true, features = ["rt", "sync"] } 20 | tokio-stream = { version = "0.1.14", features = ["sync"] } 21 | tracing.workspace = true 22 | dashmap = "5" 23 | arc-swap = "1.7" 24 | uuid = "1.10" 25 | 26 | [target.'cfg(target_arch = "wasm32")'.dependencies] 27 | getrandom = { version = "0.2", features = ["js"] } 28 | 29 | [dev-dependencies] 30 | assert-json-diff = "2.0.2" 31 | collab-plugins = { workspace = true } 32 | fs_extra = "1.2.0" 33 | nanoid = "0.4.0" 34 | tempfile = "3.8.0" 35 | tokio = { version = "1.26", features = ["rt", "macros"] } 36 | tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } 37 | walkdir = "2.3.2" 38 | zip = "0.6.6" 39 | uuid = { version = "1.6.1", features = ["v4"] } 40 | futures = "0.3.30" 41 | -------------------------------------------------------------------------------- /collab-folder/src/entities.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | use crate::{SectionsByUid, View, Workspace}; 4 | 5 | #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] 6 | pub struct FolderData { 7 | pub workspace: Workspace, 8 | pub current_view: String, 9 | pub views: Vec, 10 | #[serde(default)] 11 | pub favorites: SectionsByUid, 12 | #[serde(default)] 13 | pub recent: SectionsByUid, 14 | #[serde(default)] 15 | pub trash: SectionsByUid, 16 | #[serde(default)] 17 | pub private: SectionsByUid, 18 | } 19 | 20 | impl FolderData { 21 | pub fn new(workspace: Workspace) -> Self { 22 | Self { 23 | workspace, 24 | current_view: "".to_string(), 25 | views: vec![], 26 | favorites: SectionsByUid::new(), 27 | recent: SectionsByUid::new(), 28 | trash: SectionsByUid::new(), 29 | private: SectionsByUid::new(), 30 | } 31 | } 32 | } 33 | 34 | #[derive(Clone, Debug)] 35 | pub struct TrashInfo { 36 | pub id: String, 37 | pub name: String, 38 | pub created_at: i64, 39 | } 40 | impl AsRef for TrashInfo { 41 | fn as_ref(&self) -> &str { 42 | &self.id 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /collab-folder/src/error.rs: -------------------------------------------------------------------------------- 1 | use collab_entity::CollabValidateError; 2 | 3 | #[derive(Debug, thiserror::Error)] 4 | pub enum FolderError { 5 | #[error(transparent)] 6 | Internal(#[from] anyhow::Error), 7 | 8 | #[error(transparent)] 9 | CollabError(#[from] collab::error::CollabError), 10 | 11 | #[error("Lack of folder required data:{0}")] 12 | NoRequiredData(String), 13 | } 14 | 15 | impl From for FolderError { 16 | fn from(error: CollabValidateError) -> Self { 17 | match error { 18 | CollabValidateError::NoRequiredData(data) => FolderError::NoRequiredData(data), 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /collab-folder/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub use entities::*; 2 | pub use folder::*; 3 | pub use folder_migration::*; 4 | pub use folder_observe::*; 5 | pub use relation::*; 6 | pub use section::*; 7 | // pub use trash::*; 8 | pub use space_info::*; 9 | pub use view::*; 10 | pub use workspace::*; 11 | 12 | mod entities; 13 | mod folder; 14 | mod relation; 15 | mod section; 16 | // mod trash; 17 | mod view; 18 | mod workspace; 19 | 20 | #[macro_use] 21 | mod macros; 22 | pub mod error; 23 | pub mod folder_diff; 24 | mod folder_migration; 25 | mod folder_observe; 26 | pub mod hierarchy_builder; 27 | pub mod space_info; 28 | -------------------------------------------------------------------------------- /collab-folder/src/space_info.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | use crate::timestamp; 4 | 5 | pub const SPACE_IS_SPACE_KEY: &str = "is_space"; 6 | pub const SPACE_PERMISSION_KEY: &str = "space_permission"; 7 | pub const SPACE_ICON_KEY: &str = "space_icon"; 8 | pub const SPACE_ICON_COLOR_KEY: &str = "space_icon_color"; 9 | pub const SPACE_CREATED_AT_KEY: &str = "space_created_at"; 10 | 11 | /// Represents the space info of a view 12 | /// 13 | /// Two view types are supported: 14 | /// 15 | /// - Space view: A view associated with a space info. Parent view that can contain normal views. 16 | /// Child views inherit the space's permissions. 17 | /// 18 | /// - Normal view: Cannot contain space views and has no direct permission controls. 19 | #[derive(Serialize, Deserialize, Clone, Debug)] 20 | pub struct SpaceInfo { 21 | /// Whether the view is a space view. 22 | pub is_space: bool, 23 | 24 | /// The permission of the space view. 25 | /// 26 | /// If the space_permission is none, the space view will use the SpacePermission::PublicToAll. 27 | #[serde(default)] 28 | pub space_permission: SpacePermission, 29 | 30 | /// The created time of the space view. 31 | pub space_created_at: i64, 32 | 33 | /// The space icon. 34 | /// 35 | /// If the space_icon is none, the space view will use the default icon. 36 | pub space_icon: Option, 37 | 38 | /// The space icon color. 39 | /// 40 | /// If the space_icon_color is none, the space view will use the default icon color. 41 | /// The value should be a valid hex color code: 0xFFA34AFD 42 | pub space_icon_color: Option, 43 | } 44 | 45 | impl Default for SpaceInfo { 46 | /// Default space info is a public space 47 | /// 48 | /// The permission is public to all 49 | /// The created time is the current timestamp 50 | fn default() -> Self { 51 | Self { 52 | is_space: true, 53 | space_permission: SpacePermission::PublicToAll, 54 | space_created_at: timestamp(), 55 | space_icon: None, 56 | space_icon_color: None, 57 | } 58 | } 59 | } 60 | 61 | #[derive( 62 | Debug, Clone, Default, serde_repr::Serialize_repr, serde_repr::Deserialize_repr, PartialEq, Eq, 63 | )] 64 | #[repr(u8)] 65 | pub enum SpacePermission { 66 | #[default] 67 | PublicToAll = 0, 68 | Private = 1, 69 | } 70 | -------------------------------------------------------------------------------- /collab-folder/src/workspace.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | use crate::{RepeatedViewIdentifier, View, ViewLayout, timestamp}; 4 | 5 | #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] 6 | pub struct Workspace { 7 | pub id: String, 8 | pub name: String, 9 | pub child_views: RepeatedViewIdentifier, 10 | pub created_at: i64, 11 | pub created_by: Option, 12 | pub last_edited_time: i64, 13 | pub last_edited_by: Option, 14 | } 15 | 16 | impl Workspace { 17 | pub fn new(id: String, name: String, uid: i64) -> Self { 18 | debug_assert!(!id.is_empty()); 19 | let time = timestamp(); 20 | Self { 21 | id, 22 | name, 23 | child_views: Default::default(), 24 | created_at: time, 25 | last_edited_time: time, 26 | created_by: Some(uid), 27 | last_edited_by: Some(uid), 28 | } 29 | } 30 | } 31 | 32 | impl From<&View> for Workspace { 33 | fn from(value: &View) -> Self { 34 | Self { 35 | id: value.id.clone(), 36 | name: value.name.clone(), 37 | child_views: value.children.clone(), 38 | created_at: value.created_at, 39 | created_by: value.created_by, 40 | last_edited_time: value.last_edited_time, 41 | last_edited_by: value.last_edited_by, 42 | } 43 | } 44 | } 45 | impl From for View { 46 | fn from(value: Workspace) -> Self { 47 | Self { 48 | id: value.id, 49 | parent_view_id: "".to_string(), 50 | name: value.name, 51 | children: value.child_views, 52 | created_at: value.created_at, 53 | is_favorite: false, 54 | layout: ViewLayout::Document, 55 | icon: None, 56 | created_by: value.created_by, 57 | last_edited_time: value.last_edited_time, 58 | last_edited_by: value.last_edited_by, 59 | is_locked: None, 60 | extra: None, 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /collab-folder/tests/folder_test/custom_section.rs: -------------------------------------------------------------------------------- 1 | use assert_json_diff::assert_json_include; 2 | use collab::preclude::Any; 3 | use collab_folder::{Section, SectionItem, UserId, timestamp}; 4 | use serde_json::json; 5 | use std::collections::HashMap; 6 | use std::sync::Arc; 7 | 8 | use crate::util::create_folder_with_workspace; 9 | 10 | #[test] 11 | fn custom_section_test() { 12 | let uid = UserId::from(1); 13 | let folder_test = create_folder_with_workspace(uid.clone(), "w1"); 14 | 15 | let mut folder = folder_test.folder; 16 | let mut txn = folder.collab.transact_mut(); 17 | 18 | // By default, the folder has a favorite section 19 | let op = folder 20 | .body 21 | .section 22 | .section_op(&txn, Section::Favorite) 23 | .unwrap(); 24 | op.add_sections_item(&mut txn, vec![SectionItem::new("1".to_string())]); 25 | 26 | let _ = folder 27 | .body 28 | .section 29 | .create_section(&mut txn, Section::Custom("private".to_string())); 30 | let op = folder 31 | .body 32 | .section 33 | .section_op(&txn, Section::Custom("private".to_string())) 34 | .unwrap(); 35 | op.add_sections_item(&mut txn, vec![SectionItem::new("2".to_string())]); 36 | 37 | drop(txn); 38 | 39 | let json = folder.to_json_value(); 40 | assert_json_include!( 41 | actual: json, 42 | expected: json!({"section": { 43 | "favorite": { 44 | "1": [ 45 | { 46 | "id": "1" 47 | } 48 | ] 49 | }, 50 | "private": { 51 | "1": [ 52 | { 53 | "id": "2" 54 | } 55 | ] 56 | } 57 | }}) 58 | ); 59 | } 60 | 61 | #[test] 62 | fn section_serde_test() { 63 | let mut data: HashMap = HashMap::new(); 64 | data.insert("id".to_string(), uuid::Uuid::new_v4().to_string().into()); 65 | data.insert("timestamp".to_string(), timestamp().into()); 66 | let any = Any::Map(Arc::new(data)); 67 | println!("Any: {:?}", any); 68 | let start = std::time::Instant::now(); 69 | let item = SectionItem::try_from(&any).unwrap(); 70 | let elapsed = start.elapsed(); 71 | println!( 72 | "Time to convert Any to SectionItem: {:?} {:?}", 73 | item, elapsed 74 | ); 75 | } 76 | -------------------------------------------------------------------------------- /collab-folder/tests/folder_test/history_folder/folder_with_fav_v1.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-folder/tests/folder_test/history_folder/folder_with_fav_v1.zip -------------------------------------------------------------------------------- /collab-folder/tests/folder_test/history_folder/folder_without_fav.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-folder/tests/folder_test/history_folder/folder_without_fav.zip -------------------------------------------------------------------------------- /collab-folder/tests/folder_test/load_disk.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use fs_extra::file; 4 | use nanoid::nanoid; 5 | use walkdir::WalkDir; 6 | 7 | // #[test] 8 | // fn test_set_current_view() { 9 | // let uid: i64 = 185579439403307008; 10 | // let source = "./tests/folder_test/dbs".to_string(); 11 | // duplicate_db(source, &uid.to_string(), |duplicate_db| { 12 | // let folder = create_folder_with_object_id(uid, duplicate_db); 13 | // 14 | // // set current view 15 | // folder.set_current_view("abc"); 16 | // let json1 = folder.to_json_value(); 17 | // drop(folder); 18 | // 19 | // // reopen 20 | // let folder = create_folder_with_object_id(uid, duplicate_db); 21 | // let json2 = folder.to_json_value(); 22 | // assert_json_diff::assert_json_eq!(json1, json2); 23 | // }) 24 | // } 25 | 26 | #[allow(dead_code)] 27 | fn duplicate_db(source: String, folder: &str, f: impl FnOnce(&str)) { 28 | let dest = format!("temp/{}", nanoid!()); 29 | let dest_path = format!("{}/{}", source, dest); 30 | copy_folder_recursively(&source, folder, &dest).unwrap(); 31 | f(&dest_path); 32 | std::fs::remove_dir_all(dest_path).unwrap(); 33 | } 34 | 35 | #[allow(dead_code)] 36 | fn copy_folder_recursively( 37 | parent_folder: &str, 38 | src_folder: &str, 39 | dest_folder: &str, 40 | ) -> std::io::Result<()> { 41 | let src_path = Path::new(parent_folder).join(src_folder); 42 | let dest_path = Path::new(parent_folder).join(dest_folder); 43 | 44 | for entry in WalkDir::new(&src_path) { 45 | let entry = entry?; 46 | let entry_path = entry.path(); 47 | 48 | let relative_entry_path = entry_path.strip_prefix(&src_path).unwrap(); 49 | let target_path = dest_path.join(relative_entry_path); 50 | 51 | if entry.file_type().is_dir() { 52 | std::fs::create_dir_all(target_path)?; 53 | } else { 54 | let options = file::CopyOptions::new().overwrite(true); 55 | file::copy(entry_path, target_path, &options).unwrap(); 56 | } 57 | } 58 | Ok(()) 59 | } 60 | -------------------------------------------------------------------------------- /collab-folder/tests/folder_test/main.rs: -------------------------------------------------------------------------------- 1 | mod child_views_test; 2 | mod custom_section; 3 | mod favorite_test; 4 | mod load_disk; 5 | mod recent_views_test; 6 | mod serde_test; 7 | mod space_info_test; 8 | mod trash_test; 9 | mod util; 10 | mod view_test; 11 | mod workspace_test; 12 | -------------------------------------------------------------------------------- /collab-folder/tests/folder_test/space_info_test.rs: -------------------------------------------------------------------------------- 1 | use collab_folder::{ 2 | SPACE_CREATED_AT_KEY, SPACE_ICON_COLOR_KEY, SPACE_ICON_KEY, SPACE_IS_SPACE_KEY, 3 | SPACE_PERMISSION_KEY, SpacePermission, hierarchy_builder::ViewExtraBuilder, timestamp, 4 | }; 5 | use serde_json::json; 6 | 7 | #[test] 8 | fn create_public_space_test() { 9 | let builder = ViewExtraBuilder::new(); 10 | let timestamp = timestamp(); 11 | let space_info = builder 12 | .is_space(true) 13 | .with_space_permission(SpacePermission::PublicToAll) 14 | .with_space_icon(Some("interface_essential/home-3")) 15 | .with_space_icon_color(Some("0xFFA34AFD")) 16 | .build(); 17 | let space_info_json = serde_json::to_value(space_info).unwrap(); 18 | assert_json_diff::assert_json_eq!( 19 | space_info_json, 20 | json!({ 21 | SPACE_IS_SPACE_KEY: true, 22 | SPACE_PERMISSION_KEY: 0, 23 | SPACE_ICON_KEY: "interface_essential/home-3", 24 | SPACE_ICON_COLOR_KEY: "0xFFA34AFD", 25 | SPACE_CREATED_AT_KEY: timestamp 26 | }), 27 | ); 28 | } 29 | 30 | #[test] 31 | fn create_private_space_test() { 32 | let builder = ViewExtraBuilder::new(); 33 | let timestamp = timestamp(); 34 | let space_info = builder 35 | .is_space(true) 36 | .with_space_permission(SpacePermission::Private) 37 | .with_space_icon(Some("interface_essential/lock")) 38 | .with_space_icon_color(Some("0xFF4A4AFD")) 39 | .build(); 40 | let space_info_json = serde_json::to_value(space_info).unwrap(); 41 | assert_json_diff::assert_json_eq!( 42 | space_info_json, 43 | json!({ 44 | SPACE_IS_SPACE_KEY: true, 45 | SPACE_PERMISSION_KEY: 1, 46 | SPACE_ICON_KEY: "interface_essential/lock", 47 | SPACE_ICON_COLOR_KEY: "0xFF4A4AFD", 48 | SPACE_CREATED_AT_KEY: timestamp 49 | }), 50 | ); 51 | } 52 | 53 | #[test] 54 | fn create_space_without_icon_and_color_test() { 55 | let builder = ViewExtraBuilder::new(); 56 | let timestamp = timestamp(); 57 | let space_info = builder 58 | .is_space(true) 59 | .with_space_permission(SpacePermission::PublicToAll) 60 | .build(); 61 | let space_info_json = serde_json::to_value(space_info).unwrap(); 62 | assert_json_diff::assert_json_eq!( 63 | space_info_json, 64 | json!({ 65 | SPACE_IS_SPACE_KEY: true, 66 | SPACE_PERMISSION_KEY: 0, 67 | SPACE_CREATED_AT_KEY: timestamp 68 | }), 69 | ); 70 | } 71 | 72 | #[test] 73 | fn create_non_space_test() { 74 | let builder = ViewExtraBuilder::new(); 75 | let space_info = builder.build(); 76 | let space_info_json = serde_json::to_value(space_info).unwrap(); 77 | assert_json_diff::assert_json_eq!(space_info_json, json!({}),); 78 | } 79 | -------------------------------------------------------------------------------- /collab-folder/tests/folder_test/workspace_test.rs: -------------------------------------------------------------------------------- 1 | use collab::core::collab::{CollabOptions, default_client_id}; 2 | use collab::core::origin::CollabOrigin; 3 | use collab::preclude::Collab; 4 | use collab_folder::{Folder, FolderData, UserId, Workspace, check_folder_is_valid}; 5 | 6 | #[test] 7 | fn test_workspace_is_ready() { 8 | let uid = UserId::from(1); 9 | let object_id = "1"; 10 | 11 | let workspace = Workspace::new("w1".to_string(), "".to_string(), uid.as_i64()); 12 | let folder_data = FolderData::new(workspace); 13 | let options = CollabOptions::new(object_id.to_string(), default_client_id()); 14 | let collab = Collab::new_with_options(CollabOrigin::Empty, options).unwrap(); 15 | let folder = Folder::create(uid, collab, None, folder_data); 16 | 17 | let workspace_id = check_folder_is_valid(&folder.collab).unwrap(); 18 | assert_eq!(workspace_id, "w1".to_string()); 19 | } 20 | 21 | #[test] 22 | fn validate_folder_data() { 23 | let options = CollabOptions::new("1".to_string(), default_client_id()); 24 | let collab = Collab::new_with_options(CollabOrigin::Empty, options).unwrap(); 25 | let result = Folder::open(1, collab, None); 26 | assert!(result.is_err()); 27 | } 28 | -------------------------------------------------------------------------------- /collab-importer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "collab-importer" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | collab-folder = { workspace = true } 10 | collab-database = { workspace = true } 11 | collab-document = { workspace = true } 12 | collab = { workspace = true } 13 | collab-entity = { workspace = true } 14 | thiserror.workspace = true 15 | anyhow.workspace = true 16 | chrono.workspace = true 17 | walkdir = "2.5.0" 18 | uuid = "1.6.1" 19 | serde = { version = "1.0.204", features = ["derive"] } 20 | serde_json = "1.0.120" 21 | markdown = "1.0.0-alpha.21" 22 | tracing.workspace = true 23 | percent-encoding = "2.3.1" 24 | fancy-regex = "0.13.0" 25 | fxhash = "0.2.1" 26 | tokio = { workspace = true, features = ["io-util", "fs"] } 27 | tokio-util = "0.7" 28 | rayon = "1.10.0" 29 | sha2 = "0.10.8" 30 | base64 = "0.22.1" 31 | hex = "0.4.3" 32 | async_zip = { version = "0.0.17", features = ["full"] } 33 | async-trait.workspace = true 34 | futures = "0.3.30" 35 | async-recursion = "1.1" 36 | futures-util = "0.3.30" 37 | futures-lite = "2.3.0" 38 | sanitize-filename = "0.5.0" 39 | zip = "0.6.6" 40 | csv = { version = "1.3.0" } 41 | 42 | [dev-dependencies] 43 | tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } 44 | tokio = { workspace = true, features = ["full"] } 45 | nanoid = "0.4.0" 46 | assert-json-diff = "2.0.2" 47 | tempfile = "3.10.1" 48 | -------------------------------------------------------------------------------- /collab-importer/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::str::Utf8Error; 2 | 3 | #[derive(Debug, thiserror::Error)] 4 | pub enum ImporterError { 5 | #[error("Invalid path: {0}")] 6 | InvalidPath(String), 7 | 8 | #[error("Invalid path format")] 9 | InvalidPathFormat, 10 | 11 | #[error("{0}")] 12 | InvalidFileType(String), 13 | 14 | #[error(transparent)] 15 | ImportMarkdownError(#[from] collab_document::error::DocumentError), 16 | 17 | #[error(transparent)] 18 | ImportCsvError(#[from] collab_database::error::DatabaseError), 19 | 20 | #[error("Parse markdown error: {0}")] 21 | ParseMarkdownError(markdown::message::Message), 22 | 23 | #[error(transparent)] 24 | Utf8Error(#[from] Utf8Error), 25 | 26 | #[error(transparent)] 27 | IOError(#[from] std::io::Error), 28 | 29 | #[error("File not found")] 30 | FileNotFound, 31 | 32 | #[error("Can not import file")] 33 | CannotImport, 34 | 35 | #[error(transparent)] 36 | Internal(#[from] anyhow::Error), 37 | } 38 | -------------------------------------------------------------------------------- /collab-importer/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod error; 2 | pub mod imported_collab; 3 | pub mod notion; 4 | mod space_view; 5 | pub mod util; 6 | pub mod zip_tool; 7 | -------------------------------------------------------------------------------- /collab-importer/src/notion/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod file; 2 | pub mod importer; 3 | pub mod page; 4 | mod walk_dir; 5 | 6 | pub use importer::*; 7 | -------------------------------------------------------------------------------- /collab-importer/src/space_view.rs: -------------------------------------------------------------------------------- 1 | use crate::error::ImporterError; 2 | use collab::core::collab::{CollabOptions, DataSource, default_client_id}; 3 | use collab::core::origin::CollabOrigin; 4 | use collab::preclude::Collab; 5 | use collab_document::document_data::default_document_collab_data; 6 | use collab_folder::hierarchy_builder::{NestedChildViewBuilder, ParentChildViews}; 7 | use collab_folder::{SpaceInfo, ViewLayout}; 8 | 9 | #[allow(dead_code)] 10 | pub fn create_space_view( 11 | uid: i64, 12 | workspace_id: &str, 13 | name: &str, 14 | view_id: &str, 15 | child_views: Vec, 16 | space_info: SpaceInfo, 17 | ) -> Result<(ParentChildViews, Collab), ImporterError> { 18 | let client_id = default_client_id(); 19 | let import_container_doc_state = default_document_collab_data(view_id, client_id) 20 | .map_err(|err| ImporterError::Internal(err.into()))? 21 | .doc_state 22 | .to_vec(); 23 | 24 | let options = CollabOptions::new(view_id.to_string(), client_id) 25 | .with_data_source(DataSource::DocStateV1(import_container_doc_state)); 26 | let collab = Collab::new_with_options(CollabOrigin::Empty, options) 27 | .map_err(|err| ImporterError::Internal(err.into()))?; 28 | 29 | let view = NestedChildViewBuilder::new(uid, workspace_id.to_string()) 30 | .with_view_id(view_id) 31 | .with_layout(ViewLayout::Document) 32 | .with_name(name) 33 | .with_children(child_views) 34 | .with_extra(|extra| extra.with_space_info(space_info).build()) 35 | .build(); 36 | Ok((view, collab)) 37 | } 38 | -------------------------------------------------------------------------------- /collab-importer/src/zip_tool/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod async_zip; 2 | pub mod sync_zip; 3 | pub mod util; 4 | -------------------------------------------------------------------------------- /collab-importer/tests/asset/all_md_files.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/all_md_files.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/blog_post.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/blog_post.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/blog_post_duplicate_name.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/blog_post_duplicate_name.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/blog_post_no_subpages.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/blog_post_no_subpages.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/csv_relation.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/csv_relation.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/design.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/design.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/empty_spaces.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/empty_spaces.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/empty_zip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/empty_zip.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/import_test.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/import_test.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/multi_part_zip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/multi_part_zip.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/project&task.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/project&task.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/project&task_contain_zip_attachment.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/project&task_contain_zip_attachment.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/project&task_no_subpages.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/project&task_no_subpages.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/project.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/project.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/row_page_with_headings.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/row_page_with_headings.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/two_spaces.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/two_spaces.zip -------------------------------------------------------------------------------- /collab-importer/tests/asset/two_spaces_with_other_files.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/collab-importer/tests/asset/two_spaces_with_other_files.zip -------------------------------------------------------------------------------- /collab-importer/tests/main.rs: -------------------------------------------------------------------------------- 1 | mod notion_test; 2 | mod util; 3 | -------------------------------------------------------------------------------- /collab-importer/tests/notion_test/mod.rs: -------------------------------------------------------------------------------- 1 | mod customer_import_test; 2 | mod import_test; 3 | -------------------------------------------------------------------------------- /collab-plugins/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "collab-plugins" 3 | version = "0.2.0" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | crate-type = ["cdylib", "rlib"] 9 | 10 | [dependencies] 11 | yrs.workspace = true 12 | collab-entity = { workspace = true } 13 | 14 | futures-util = { version = "0.3", features = ["sink"] } 15 | tokio = { workspace = true, features = ["sync", "rt", "macros"] } 16 | tracing.workspace = true 17 | anyhow.workspace = true 18 | 19 | tokio-retry = "0.3" 20 | async-trait.workspace = true 21 | thiserror.workspace = true 22 | serde.workspace = true 23 | serde_json.workspace = true 24 | similar = { version = "2.2.1" } 25 | tokio-stream = { version = "0.1.14", features = ["sync"] } 26 | uuid = { version = "1.3.3", features = ["v4"] } 27 | bytes.workspace = true 28 | rand = { version = "0.8", optional = true } 29 | lazy_static = "1.4.0" 30 | smallvec = { version = "1.10", features = ["write", "union", "const_generics", "const_new"] } 31 | chrono = { version = "0.4.22", default-features = false, features = ["clock"] } 32 | bincode = "1.3.3" 33 | 34 | [target.'cfg(not(target_arch = "wasm32"))'.dependencies] 35 | collab = { workspace = true } 36 | rocksdb = { version = "0.22.0", default-features = false, features = ["zstd"] } 37 | 38 | 39 | [dev-dependencies] 40 | tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } 41 | tokio = { version = "1.26.0", features = ["macros"] } 42 | rand = { version = "0.8" } 43 | tempfile = "3.8.0" 44 | assert-json-diff = "2.0.2" 45 | tokio-util = { version = "0.7", features = ["codec"] } 46 | futures = "0.3" 47 | 48 | [target.'cfg(target_arch = "wasm32")'.dependencies] 49 | getrandom = { version = "0.2", features = ["js"] } 50 | collab = { workspace = true } 51 | indexed_db_futures = { version = "0.4" } 52 | js-sys = "0.3" 53 | async-stream = "0.3" 54 | futures = "0.3" 55 | wasm-bindgen = "0.2" 56 | web-sys = { version = "0.3", features = ["console", "Window"] } 57 | wasm-bindgen-futures = "0.4" 58 | tracing-wasm = "0.2" 59 | 60 | [target.'cfg(target_arch = "wasm32")'.dev-dependencies] 61 | wasm-bindgen-test = "0.3.40" 62 | 63 | [features] 64 | default = [] 65 | postgres_plugin = ["rand"] 66 | verbose_log = [] -------------------------------------------------------------------------------- /collab-plugins/src/cloud_storage/channel.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Debug; 2 | use std::pin::Pin; 3 | use std::task::{Context, Poll}; 4 | 5 | use futures_util::{Sink, Stream}; 6 | use tokio::sync::mpsc::UnboundedSender; 7 | 8 | use crate::cloud_storage::error::SyncError; 9 | 10 | #[allow(dead_code)] 11 | pub trait CollabConnect: Sink + Stream {} 12 | 13 | pub struct TokioUnboundedSink(pub UnboundedSender); 14 | 15 | impl Sink for TokioUnboundedSink 16 | where 17 | T: Send + Sync + 'static + Debug, 18 | { 19 | type Error = SyncError; 20 | 21 | fn poll_ready(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { 22 | // An unbounded channel can always accept messages without blocking, so we always return Ready. 23 | Poll::Ready(Ok(())) 24 | } 25 | 26 | fn start_send(self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> { 27 | let _ = self.0.send(item); 28 | Ok(()) 29 | } 30 | 31 | fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { 32 | // There is no buffering in an unbounded channel, so we always return Ready. 33 | Poll::Ready(Ok(())) 34 | } 35 | 36 | fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { 37 | // An unbounded channel is closed by dropping the sender, so we don't need to do anything here. 38 | Poll::Ready(Ok(())) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /collab-plugins/src/cloud_storage/error.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, thiserror::Error)] 2 | pub enum SyncError { 3 | #[error("failed to deserialize message: {0}")] 4 | DecodingError(#[from] yrs::encoding::read::Error), 5 | 6 | #[error(transparent)] 7 | SerdeError(#[from] serde_json::Error), 8 | 9 | #[error(transparent)] 10 | TokioTask(#[from] tokio::task::JoinError), 11 | 12 | #[error(transparent)] 13 | IO(#[from] std::io::Error), 14 | 15 | #[error("Internal failure: {0}")] 16 | Internal(#[from] Box), 17 | } 18 | -------------------------------------------------------------------------------- /collab-plugins/src/cloud_storage/mod.rs: -------------------------------------------------------------------------------- 1 | pub use remote_collab::{ 2 | RemoteCollabSnapshot, RemoteCollabState, RemoteCollabStorage, RemoteUpdateReceiver, 3 | RemoteUpdateSender, 4 | }; 5 | pub use yrs::Update as YrsUpdate; 6 | pub use yrs::merge_updates_v1; 7 | pub use yrs::updates::decoder::Decode; 8 | 9 | pub mod postgres; 10 | 11 | mod channel; 12 | mod error; 13 | mod msg; 14 | mod remote_collab; 15 | mod sink; 16 | -------------------------------------------------------------------------------- /collab-plugins/src/cloud_storage/postgres/mod.rs: -------------------------------------------------------------------------------- 1 | pub use plugin::*; 2 | 3 | mod plugin; 4 | -------------------------------------------------------------------------------- /collab-plugins/src/connect_state.rs: -------------------------------------------------------------------------------- 1 | use std::sync::atomic::{AtomicU8, Ordering}; 2 | use tokio::sync::broadcast; 3 | 4 | #[repr(u8)] 5 | #[derive(Debug, Clone, Copy, Eq, PartialEq)] 6 | pub enum CollabConnectState { 7 | Connected = CollabConnectState::CONNECTED, 8 | Disconnected = CollabConnectState::DISCONNECTED, 9 | } 10 | 11 | impl CollabConnectState { 12 | const CONNECTED: u8 = 0; 13 | const DISCONNECTED: u8 = 1; 14 | } 15 | 16 | impl TryFrom for CollabConnectState { 17 | type Error = u8; 18 | 19 | fn try_from(value: u8) -> Result { 20 | match value { 21 | Self::CONNECTED => Ok(Self::Connected), 22 | Self::DISCONNECTED => Ok(Self::Disconnected), 23 | unknown => Err(unknown), 24 | } 25 | } 26 | } 27 | 28 | pub struct CollabConnectReachability { 29 | state: AtomicU8, 30 | state_sender: broadcast::Sender, 31 | } 32 | 33 | impl Default for CollabConnectReachability { 34 | fn default() -> Self { 35 | let (state_sender, _) = broadcast::channel(1000); 36 | let state = AtomicU8::new(CollabConnectState::Connected as u8); 37 | Self { 38 | state, 39 | state_sender, 40 | } 41 | } 42 | } 43 | 44 | impl CollabConnectReachability { 45 | pub fn new() -> Self { 46 | Self::default() 47 | } 48 | 49 | pub fn state(&self) -> CollabConnectState { 50 | CollabConnectState::try_from(self.state.load(Ordering::Acquire)).unwrap() 51 | } 52 | 53 | pub fn set_state(&self, new_state: CollabConnectState) { 54 | let old = self.state.swap(new_state as u8, Ordering::AcqRel); 55 | if old != new_state as u8 { 56 | let _ = self.state_sender.send(new_state); 57 | } 58 | } 59 | 60 | pub fn subscribe(&self) -> broadcast::Receiver { 61 | self.state_sender.subscribe() 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /collab-plugins/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod local_storage; 2 | 3 | #[macro_export] 4 | macro_rules! if_native { 5 | ($($item:item)*) => {$( 6 | #[cfg(not(target_arch = "wasm32"))] 7 | $item 8 | )*} 9 | } 10 | 11 | #[macro_export] 12 | macro_rules! if_wasm { 13 | ($($item:item)*) => {$( 14 | #[cfg(target_arch = "wasm32")] 15 | $item 16 | )*} 17 | } 18 | 19 | #[cfg(all(feature = "postgres_plugin", not(target_arch = "wasm32")))] 20 | pub mod cloud_storage; 21 | pub mod connect_state; 22 | 23 | if_native! { 24 | pub type CollabKVDB = local_storage::rocksdb::kv_impl::KVTransactionDBRocksdbImpl; 25 | } 26 | 27 | if_wasm! { 28 | pub type CollabKVDB = local_storage::indexeddb::CollabIndexeddb; 29 | } 30 | -------------------------------------------------------------------------------- /collab-plugins/src/local_storage/indexeddb/mod.rs: -------------------------------------------------------------------------------- 1 | mod indexeddb_plugin; 2 | mod kv_impl; 3 | 4 | pub use indexeddb_plugin::*; 5 | pub use kv_impl::*; 6 | -------------------------------------------------------------------------------- /collab-plugins/src/local_storage/kv/error.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, thiserror::Error)] 2 | pub enum PersistenceError { 3 | #[cfg(not(target_arch = "wasm32"))] 4 | #[error("Rocksdb corruption:{0}")] 5 | RocksdbCorruption(String), 6 | 7 | #[cfg(not(target_arch = "wasm32"))] 8 | #[error("Rocksdb repair:{0}")] 9 | RocksdbRepairFail(String), 10 | 11 | #[cfg(not(target_arch = "wasm32"))] 12 | #[error("{0}")] 13 | RocksdbBusy(String), 14 | 15 | // If the database is already locked by another process, it will return an IO error. It 16 | // happens when the database is already opened by another process. 17 | #[cfg(not(target_arch = "wasm32"))] 18 | #[error("{0}")] 19 | RocksdbIOError(String), 20 | 21 | #[error(transparent)] 22 | Bincode(#[from] bincode::Error), 23 | 24 | #[error("{0}")] 25 | RecordNotFound(String), 26 | 27 | #[error("The document already exist")] 28 | DocumentAlreadyExist, 29 | 30 | #[error("Unexpected empty updates")] 31 | UnexpectedEmptyUpdates, 32 | 33 | #[error(transparent)] 34 | Yrs(#[from] yrs::encoding::read::Error), 35 | 36 | #[error("Failed to apply update from persistent store: {0}")] 37 | Update(#[from] yrs::error::UpdateError), 38 | 39 | #[error("invalid data: {0}")] 40 | InvalidData(String), 41 | 42 | #[error("Duplicate update key")] 43 | DuplicateUpdateKey, 44 | 45 | #[error("Can't find the latest update key")] 46 | LatestUpdateKeyNotExist, 47 | 48 | #[error(transparent)] 49 | Collab(#[from] collab::error::CollabError), 50 | 51 | #[error(transparent)] 52 | Internal(#[from] anyhow::Error), 53 | } 54 | 55 | impl PersistenceError { 56 | pub fn is_record_not_found(&self) -> bool { 57 | matches!(self, PersistenceError::RecordNotFound(_)) 58 | } 59 | } 60 | 61 | #[cfg(target_arch = "wasm32")] 62 | impl From for PersistenceError { 63 | fn from(value: indexed_db_futures::web_sys::DomException) -> Self { 64 | PersistenceError::Internal(anyhow::anyhow!("DOMException: {:?}", value)) 65 | } 66 | } 67 | 68 | #[cfg(not(target_arch = "wasm32"))] 69 | impl From for PersistenceError { 70 | fn from(value: rocksdb::Error) -> Self { 71 | match value.kind() { 72 | rocksdb::ErrorKind::NotFound => PersistenceError::UnexpectedEmptyUpdates, 73 | rocksdb::ErrorKind::Corruption => PersistenceError::RocksdbCorruption(value.into_string()), 74 | rocksdb::ErrorKind::IOError => PersistenceError::RocksdbIOError(value.into_string()), 75 | rocksdb::ErrorKind::Busy => PersistenceError::RocksdbBusy(value.into_string()), 76 | _ => PersistenceError::Internal(value.into()), 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /collab-plugins/src/local_storage/kv/mod.rs: -------------------------------------------------------------------------------- 1 | pub use db::*; 2 | pub use error::*; 3 | pub use range::*; 4 | 5 | mod db; 6 | pub mod doc; 7 | pub mod error; 8 | pub mod keys; 9 | pub mod oid; 10 | mod range; 11 | pub mod snapshot; 12 | -------------------------------------------------------------------------------- /collab-plugins/src/local_storage/kv/range.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Range, RangeInclusive, RangeToInclusive}; 2 | 3 | #[derive(Clone)] 4 | pub struct CLRange { 5 | pub(crate) start: i64, 6 | pub(crate) end: i64, 7 | } 8 | 9 | impl CLRange { 10 | /// Construct a new `RevRange` representing the range [start..end). 11 | /// It is an invariant that `start <= end`. 12 | pub fn new(start: i64, end: i64) -> CLRange { 13 | debug_assert!(start <= end); 14 | CLRange { start, end } 15 | } 16 | } 17 | 18 | impl From> for CLRange { 19 | fn from(src: RangeInclusive) -> CLRange { 20 | CLRange::new(*src.start(), src.end().saturating_add(1)) 21 | } 22 | } 23 | 24 | impl From> for CLRange { 25 | fn from(src: RangeToInclusive) -> CLRange { 26 | CLRange::new(0, src.end.saturating_add(1)) 27 | } 28 | } 29 | 30 | impl From> for CLRange { 31 | fn from(src: Range) -> CLRange { 32 | let Range { start, end } = src; 33 | CLRange { start, end } 34 | } 35 | } 36 | 37 | impl Iterator for CLRange { 38 | type Item = i64; 39 | 40 | fn next(&mut self) -> Option { 41 | if self.start > self.end { 42 | return None; 43 | } 44 | let val = self.start; 45 | self.start += 1; 46 | Some(val) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /collab-plugins/src/local_storage/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod kv; 2 | 3 | #[cfg(not(target_arch = "wasm32"))] 4 | pub mod rocksdb; 5 | 6 | #[cfg(target_arch = "wasm32")] 7 | pub mod indexeddb; 8 | 9 | mod storage_config; 10 | 11 | pub use storage_config::*; 12 | -------------------------------------------------------------------------------- /collab-plugins/src/local_storage/rocksdb/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod kv_impl; 2 | pub mod rocksdb_plugin; 3 | // pub mod snapshot_plugin; 4 | pub mod util; 5 | -------------------------------------------------------------------------------- /collab-plugins/src/local_storage/rocksdb/util.rs: -------------------------------------------------------------------------------- 1 | use crate::CollabKVDB; 2 | use crate::local_storage::kv::KVTransactionDB; 3 | use crate::local_storage::kv::doc::CollabKVAction; 4 | use anyhow::anyhow; 5 | use collab::core::collab::DataSource; 6 | use collab::core::collab_plugin::CollabPersistence; 7 | use collab::error::CollabError; 8 | use collab::preclude::Collab; 9 | use std::sync::Weak; 10 | use tracing::error; 11 | 12 | pub struct KVDBCollabPersistenceImpl { 13 | pub db: Weak, 14 | pub uid: i64, 15 | pub workspace_id: String, 16 | } 17 | 18 | impl KVDBCollabPersistenceImpl { 19 | pub fn new(db: Weak, uid: i64, workspace_id: String) -> Self { 20 | Self { 21 | db, 22 | uid, 23 | workspace_id, 24 | } 25 | } 26 | 27 | pub fn into_data_source(self) -> DataSource { 28 | DataSource::Disk(Some(Box::new(self))) 29 | } 30 | } 31 | 32 | impl From for DataSource { 33 | fn from(persistence: KVDBCollabPersistenceImpl) -> Self { 34 | persistence.into_data_source() 35 | } 36 | } 37 | 38 | impl CollabPersistence for KVDBCollabPersistenceImpl { 39 | fn load_collab_from_disk(&self, collab: &mut Collab) -> Result<(), CollabError> { 40 | let collab_db = self 41 | .db 42 | .upgrade() 43 | .ok_or_else(|| CollabError::Internal(anyhow!("collab_db is dropped")))?; 44 | let object_id = collab.object_id().to_string(); 45 | let rocksdb_read = collab_db.read_txn(); 46 | 47 | if rocksdb_read.is_exist(self.uid, &self.workspace_id, &object_id) { 48 | let mut txn = collab.transact_mut(); 49 | if let Err(err) = 50 | rocksdb_read.load_doc_with_txn(self.uid, self.workspace_id.as_str(), &object_id, &mut txn) 51 | { 52 | error!("🔴 load doc:{} failed: {}", object_id, err); 53 | } 54 | drop(rocksdb_read); 55 | txn.commit(); 56 | drop(txn); 57 | } 58 | Ok(()) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /collab-plugins/src/local_storage/storage_config.rs: -------------------------------------------------------------------------------- 1 | #[derive(Clone)] 2 | pub struct CollabPersistenceConfig { 3 | /// Enable snapshot. Default is [false]. 4 | pub enable_snapshot: bool, 5 | /// Generate a snapshot every N updates 6 | /// Default is 100. The value must be greater than 0. 7 | pub snapshot_per_update: u32, 8 | } 9 | 10 | impl CollabPersistenceConfig { 11 | pub fn new() -> Self { 12 | Self::default() 13 | } 14 | 15 | pub fn enable_snapshot(mut self, enable_snapshot: bool) -> Self { 16 | self.enable_snapshot = enable_snapshot; 17 | self 18 | } 19 | 20 | pub fn snapshot_per_update(mut self, snapshot_per_update: u32) -> Self { 21 | debug_assert!(snapshot_per_update > 0); 22 | self.snapshot_per_update = snapshot_per_update; 23 | self 24 | } 25 | } 26 | 27 | impl Default for CollabPersistenceConfig { 28 | fn default() -> Self { 29 | Self { 30 | enable_snapshot: true, 31 | snapshot_per_update: 100, 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /collab-plugins/tests/disk/delete_test.rs: -------------------------------------------------------------------------------- 1 | use crate::disk::script::CollabPersistenceTest; 2 | use collab_plugins::local_storage::CollabPersistenceConfig; 3 | 4 | #[tokio::test] 5 | async fn delete_single_doc_test() { 6 | let mut test = CollabPersistenceTest::new(CollabPersistenceConfig::default()); 7 | let doc_id = "1".to_string(); 8 | 9 | // Replacing Script variants with function calls 10 | test 11 | .create_document_with_collab_db(doc_id.clone(), test.db.clone()) 12 | .await; 13 | test.assert_ids(vec![1.to_string()]).await; 14 | test.delete_document(doc_id).await; 15 | test.assert_ids(vec![]).await; 16 | } 17 | 18 | #[tokio::test] 19 | async fn delete_multiple_docs_test() { 20 | let mut test = CollabPersistenceTest::new(CollabPersistenceConfig::default()); 21 | let db = test.db.clone(); 22 | 23 | // Replacing Script variants with function calls 24 | test 25 | .create_document_with_collab_db("1".to_string(), db.clone()) 26 | .await; 27 | test 28 | .create_document_with_collab_db("2".to_string(), db.clone()) 29 | .await; 30 | test 31 | .create_document_with_collab_db("3".to_string(), db.clone()) 32 | .await; 33 | test.delete_document("1".to_string()).await; 34 | test.delete_document("2".to_string()).await; 35 | test.assert_ids(vec![3.to_string()]).await; 36 | } 37 | -------------------------------------------------------------------------------- /collab-plugins/tests/disk/mod.rs: -------------------------------------------------------------------------------- 1 | mod delete_test; 2 | mod insert_test; 3 | mod range_test; 4 | mod restore_test; 5 | mod script; 6 | mod undo_test; 7 | mod util; 8 | -------------------------------------------------------------------------------- /collab-plugins/tests/disk/util.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | use collab_plugins::CollabKVDB; 4 | use tempfile::TempDir; 5 | 6 | pub fn rocks_db() -> (PathBuf, CollabKVDB) { 7 | let tempdir = TempDir::new().unwrap(); 8 | let path = tempdir.into_path(); 9 | let cloned_path = path.clone(); 10 | (path, CollabKVDB::open(cloned_path).unwrap()) 11 | } 12 | -------------------------------------------------------------------------------- /collab-plugins/tests/main.rs: -------------------------------------------------------------------------------- 1 | #[cfg(not(target_arch = "wasm32"))] 2 | mod disk; 3 | 4 | #[cfg(target_arch = "wasm32")] 5 | mod web; 6 | 7 | #[cfg(not(target_arch = "wasm32"))] 8 | pub fn setup_log() { 9 | use tracing_subscriber::util::SubscriberInitExt; 10 | static START: std::sync::Once = std::sync::Once::new(); 11 | START.call_once(|| { 12 | let level = "trace"; 13 | let mut filters = vec![]; 14 | filters.push(format!("collab_persistence={}", level)); 15 | filters.push(format!("collab={}", level)); 16 | filters.push(format!("collab_sync={}", level)); 17 | filters.push(format!("collab_plugins={}", level)); 18 | unsafe { 19 | std::env::set_var("RUST_LOG", filters.join(",")); 20 | } 21 | 22 | let subscriber = tracing_subscriber::fmt::Subscriber::builder() 23 | .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) 24 | .with_ansi(true) 25 | .finish(); 26 | subscriber.try_init().unwrap(); 27 | }); 28 | } 29 | -------------------------------------------------------------------------------- /collab-plugins/tests/web/mod.rs: -------------------------------------------------------------------------------- 1 | use wasm_bindgen_test::wasm_bindgen_test_configure; 2 | wasm_bindgen_test_configure!(run_in_browser); 3 | 4 | mod edit_collab_test; 5 | mod indexeddb_test; 6 | -------------------------------------------------------------------------------- /collab-plugins/tests/web/setup_tests.js: -------------------------------------------------------------------------------- 1 | function get_current_timestamp() { 2 | return Date.now(); 3 | } 4 | 5 | // Expose the function to the global scope so it's accessible to the WASM module 6 | global.get_current_timestamp = get_current_timestamp; 7 | -------------------------------------------------------------------------------- /collab-plugins/tests/web/test.md: -------------------------------------------------------------------------------- 1 | 2 | ## Run clippy for web 3 | 4 | ```shell 5 | cargo clippy --target=wasm32-unknown-unknown --fix --allow-dirty --features="wasm_build" 6 | ``` 7 | 8 | ## Run tests in Chrome 9 | ```shell 10 | wasm-pack test --chrome 11 | ``` 12 | 13 | ## Build for web 14 | 15 | ```shell 16 | wasm-pack build 17 | ``` -------------------------------------------------------------------------------- /collab-user/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "collab-user" 3 | version = "0.2.0" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | crate-type = ["cdylib", "rlib"] 9 | 10 | [dependencies] 11 | serde.workspace = true 12 | serde_json.workspace = true 13 | anyhow.workspace = true 14 | collab = { workspace = true } 15 | collab-entity = { workspace = true } 16 | tokio = { workspace = true, features = ["rt", "sync"] } 17 | tokio-stream = { version = "0.1.14", features = ["sync"] } 18 | tracing.workspace = true 19 | 20 | [target.'cfg(target_arch = "wasm32")'.dependencies] 21 | getrandom = { version = "0.2", features = ["js"] } 22 | 23 | [dev-dependencies] 24 | assert-json-diff = "2.0.2" 25 | collab-plugins = { workspace = true } 26 | fs_extra = "1.2.0" 27 | nanoid = "0.4.0" 28 | tempfile = "3.8.0" 29 | tokio = { version = "1.26", features = ["rt", "macros"] } 30 | uuid = "1.3.3" 31 | -------------------------------------------------------------------------------- /collab-user/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod reminder; 2 | mod user_awareness; 3 | 4 | pub mod core { 5 | pub use crate::reminder::*; 6 | pub use crate::user_awareness::*; 7 | } 8 | -------------------------------------------------------------------------------- /collab-user/tests/main.rs: -------------------------------------------------------------------------------- 1 | mod reminder_test; 2 | mod util; 3 | -------------------------------------------------------------------------------- /collab-user/tests/reminder_test/mod.rs: -------------------------------------------------------------------------------- 1 | mod subscribe_test; 2 | mod test; 3 | -------------------------------------------------------------------------------- /collab-user/tests/reminder_test/subscribe_test.rs: -------------------------------------------------------------------------------- 1 | use collab::lock::Mutex; 2 | use collab_entity::reminder::{ObjectType, Reminder}; 3 | use collab_user::core::ReminderChange; 4 | use std::sync::Arc; 5 | 6 | use crate::util::{UserAwarenessTest, receive_with_timeout}; 7 | 8 | #[tokio::test] 9 | async fn subscribe_insert_reminder_test() { 10 | let test = UserAwarenessTest::new(1); 11 | let mut rx = test.reminder_change_tx.subscribe(); 12 | let reminder = Reminder::new("1".to_string(), "o1".to_string(), 123, ObjectType::Document); 13 | let test = Arc::new(Mutex::from(test)); 14 | let cloned_test = test.clone(); 15 | let cloned_reminder = reminder.clone(); 16 | tokio::spawn(async move { 17 | let mut lock = cloned_test.lock().await; 18 | lock.user_awareness.add_reminder(cloned_reminder); 19 | }); 20 | 21 | let change = receive_with_timeout(&mut rx, std::time::Duration::from_secs(2)) 22 | .await 23 | .unwrap(); 24 | match change { 25 | ReminderChange::DidCreateReminders { reminders } => { 26 | assert_eq!(reminders.len(), 1); 27 | assert_eq!(reminders[0], reminder); 28 | }, 29 | _ => panic!("Expected DidCreateReminders"), 30 | } 31 | } 32 | 33 | #[tokio::test] 34 | async fn subscribe_delete_reminder_test() { 35 | let mut test = UserAwarenessTest::new(1); 36 | let mut rx = test.reminder_change_tx.subscribe(); 37 | for i in 0..5 { 38 | let reminder = Reminder::new( 39 | format!("{}", i), 40 | "o1".to_string(), 41 | 123, 42 | ObjectType::Document, 43 | ); 44 | test.add_reminder(reminder); 45 | } 46 | 47 | let test = Arc::new(Mutex::from(test)); 48 | let cloned_test = test.clone(); 49 | tokio::spawn(async move { 50 | let mut lock = cloned_test.lock().await; 51 | lock.remove_reminder("1"); 52 | }); 53 | 54 | // Continuously receive changes until the change we want is received. 55 | while let Ok(change) = rx.recv().await { 56 | if let ReminderChange::DidDeleteReminder { index } = change { 57 | assert_eq!(index, 1); 58 | break; 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /collab-user/tests/util.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Deref, DerefMut}; 2 | use std::sync::Arc; 3 | use std::time::Duration; 4 | 5 | use anyhow::Result; 6 | use collab::core::collab::{CollabOptions, DataSource, default_client_id}; 7 | use collab::core::origin::{CollabClient, CollabOrigin}; 8 | use collab::preclude::Collab; 9 | use collab_entity::CollabType; 10 | use collab_plugins::CollabKVDB; 11 | use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; 12 | use collab_user::core::{RemindersChangeSender, UserAwareness, UserAwarenessNotifier}; 13 | use tempfile::TempDir; 14 | use tokio::sync::broadcast::Receiver; 15 | use tokio::time::timeout; 16 | use uuid::Uuid; 17 | 18 | pub struct UserAwarenessTest { 19 | pub user_awareness: UserAwareness, 20 | #[allow(dead_code)] 21 | pub reminder_change_tx: RemindersChangeSender, 22 | } 23 | 24 | impl Deref for UserAwarenessTest { 25 | type Target = UserAwareness; 26 | 27 | fn deref(&self) -> &Self::Target { 28 | &self.user_awareness 29 | } 30 | } 31 | 32 | impl DerefMut for UserAwarenessTest { 33 | fn deref_mut(&mut self) -> &mut Self::Target { 34 | &mut self.user_awareness 35 | } 36 | } 37 | 38 | impl UserAwarenessTest { 39 | pub fn new(uid: i64) -> Self { 40 | let workspace_id = Uuid::new_v4().to_string(); 41 | let tempdir = TempDir::new().unwrap(); 42 | 43 | let path = tempdir.into_path(); 44 | let db = Arc::new(CollabKVDB::open(path.clone()).unwrap()); 45 | let id = uuid::Uuid::new_v4().to_string(); 46 | let disk_plugin = RocksdbDiskPlugin::new( 47 | uid, 48 | workspace_id, 49 | id.clone(), 50 | CollabType::UserAwareness, 51 | Arc::downgrade(&db), 52 | ); 53 | 54 | let options = CollabOptions::new(uid.to_string(), default_client_id()) 55 | .with_data_source(DataSource::Disk(None)); 56 | let client = CollabClient::new(uid, "1"); 57 | let mut collab = Collab::new_with_options(CollabOrigin::Client(client), options).unwrap(); 58 | collab.add_plugin(Box::new(disk_plugin)); 59 | collab.initialize(); 60 | 61 | let (reminder_change_tx, _) = tokio::sync::broadcast::channel(100); 62 | let notifier = UserAwarenessNotifier { 63 | reminder_change_tx: reminder_change_tx.clone(), 64 | }; 65 | let user_awareness = UserAwareness::create(collab, Some(notifier)).unwrap(); 66 | Self { 67 | user_awareness, 68 | reminder_change_tx, 69 | } 70 | } 71 | } 72 | 73 | pub async fn receive_with_timeout(receiver: &mut Receiver, duration: Duration) -> Result 74 | where 75 | T: Clone, 76 | { 77 | let res = timeout(duration, receiver.recv()).await??; 78 | Ok(res) 79 | } 80 | -------------------------------------------------------------------------------- /collab/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "collab" 3 | version = "0.2.0" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | crate-type = ["cdylib", "rlib"] 9 | 10 | [dependencies] 11 | yrs.workspace = true 12 | anyhow.workspace = true 13 | thiserror.workspace = true 14 | serde = { workspace = true, features = ["rc"] } 15 | serde_json.workspace = true 16 | bytes = { workspace = true, features = ["serde"] } 17 | tracing.workspace = true 18 | tokio = { workspace = true, features = ["sync", "rt"] } 19 | tokio-stream = { version = "0.1.14", features = ["sync"] } 20 | async-trait.workspace = true 21 | arc-swap.workspace = true 22 | bincode = "1.3.3" 23 | serde_repr = "0.1" 24 | chrono = "0.4.22" 25 | unicode-segmentation = "1.10.1" 26 | lazy_static = "1.4.0" 27 | fastrand = "2.1.0" 28 | 29 | [target.'cfg(target_arch = "wasm32")'.dependencies] 30 | web-sys = { version = "0.3" } 31 | js-sys = "0.3" 32 | 33 | [dev-dependencies] 34 | tokio = { workspace = true, features = ["macros", "sync", "rt"] } 35 | tempfile = "3.8.0" 36 | collab = { path = "", features = ["default"] } 37 | nanoid = "0.4.0" 38 | chrono.workspace = true 39 | assert-json-diff = "2.0.2" 40 | tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } 41 | assert_matches2 = "0.1.2" 42 | 43 | [features] 44 | default = [] 45 | verbose_log = [] 46 | trace_transact = [] 47 | lock_timeout = [] 48 | -------------------------------------------------------------------------------- /collab/src/any_mut.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::sync::Arc; 3 | use yrs::Any; 4 | 5 | #[derive(Debug, Clone, PartialEq, Default)] 6 | pub enum AnyMut { 7 | #[default] 8 | Null, 9 | Bool(bool), 10 | Number(f64), 11 | BigInt(i64), 12 | String(String), 13 | Bytes(bytes::BytesMut), 14 | Array(Vec), 15 | Map(HashMap), 16 | } 17 | 18 | impl From for AnyMut { 19 | fn from(value: Any) -> Self { 20 | match value { 21 | Any::Null => AnyMut::Null, 22 | Any::Undefined => AnyMut::Null, 23 | Any::Bool(bool) => AnyMut::Bool(bool), 24 | Any::Number(num) => AnyMut::Number(num), 25 | Any::BigInt(num) => AnyMut::BigInt(num), 26 | Any::String(str) => AnyMut::String(str.to_string()), 27 | Any::Buffer(buf) => AnyMut::Bytes(bytes::BytesMut::from(&*buf)), 28 | Any::Array(array) => { 29 | let array: Vec = array.iter().map(|any| AnyMut::from(any.clone())).collect(); 30 | AnyMut::Array(array) 31 | }, 32 | Any::Map(map) => { 33 | let owned = Arc::try_unwrap(map).unwrap_or_else(|map| (*map).clone()); 34 | let map: HashMap = owned 35 | .into_iter() 36 | .map(|(k, v)| (k, AnyMut::from(v))) 37 | .collect(); 38 | AnyMut::Map(map) 39 | }, 40 | } 41 | } 42 | } 43 | 44 | impl From for Any { 45 | fn from(value: AnyMut) -> Self { 46 | match value { 47 | AnyMut::Null => Any::Null, 48 | AnyMut::Bool(bool) => Any::Bool(bool), 49 | AnyMut::Number(num) => Any::Number(num), 50 | AnyMut::BigInt(num) => Any::BigInt(num), 51 | AnyMut::String(str) => Any::String(str.into()), 52 | AnyMut::Bytes(bytes) => Any::Buffer(bytes.freeze().to_vec().into()), 53 | AnyMut::Array(array) => Any::Array(array.into_iter().map(Any::from).collect()), 54 | AnyMut::Map(map) => { 55 | let map: HashMap = map.into_iter().map(|(k, v)| (k, Any::from(v))).collect(); 56 | Any::Map(map.into()) 57 | }, 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /collab/src/core/collab_search.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /collab/src/core/fill.rs: -------------------------------------------------------------------------------- 1 | use crate::util::ArrayExt; 2 | 3 | use yrs::types::TypeRef; 4 | use yrs::{Any, Array, ArrayPrelim, ArrayRef, Map, MapPrelim, MapRef, SharedRef, TransactionMut}; 5 | 6 | #[derive(Debug, thiserror::Error)] 7 | pub enum FillError { 8 | #[error("cannot fill {0} with: {0}")] 9 | InvalidData(TypeRef, String), 10 | } 11 | 12 | /// Trait that allows to fill shared refs with data. 13 | pub trait FillRef 14 | where 15 | R: SharedRef, 16 | { 17 | fn fill(self, txn: &mut TransactionMut, shared_ref: &R) -> Result<(), FillError>; 18 | } 19 | 20 | impl FillRef for Any { 21 | fn fill(self, txn: &mut TransactionMut, shared_ref: &MapRef) -> Result<(), FillError> { 22 | match self { 23 | Any::Map(map) => { 24 | for (key, value) in map.iter() { 25 | let value = value.clone(); 26 | match value { 27 | Any::Array(values) => { 28 | let nested_ref: ArrayRef = 29 | shared_ref.insert(txn, key.as_str(), ArrayPrelim::default()); 30 | nested_ref.insert_range(txn, 0, values.to_vec()); 31 | }, 32 | value @ Any::Map(_) => { 33 | let nested_ref: MapRef = shared_ref.get_or_init(txn, key.as_str()); 34 | value.fill(txn, &nested_ref)?; 35 | }, 36 | other => { 37 | shared_ref.try_update(txn, key.as_str(), other); 38 | }, 39 | } 40 | } 41 | Ok(()) 42 | }, 43 | _ => Err(FillError::InvalidData(TypeRef::Map, self.to_string())), 44 | } 45 | } 46 | } 47 | 48 | impl FillRef for Any { 49 | fn fill(self, txn: &mut TransactionMut, shared_ref: &ArrayRef) -> Result<(), FillError> { 50 | match self { 51 | Any::Array(array) => { 52 | shared_ref.clear(txn); 53 | for value in array.iter().cloned() { 54 | let map_ref = shared_ref.push_back(txn, MapPrelim::default()); 55 | value.fill(txn, &map_ref)?; 56 | } 57 | Ok(()) 58 | }, 59 | _ => Err(FillError::InvalidData(TypeRef::Array, self.to_string())), 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /collab/src/core/mod.rs: -------------------------------------------------------------------------------- 1 | pub use yrs::sync::awareness; 2 | pub mod collab; 3 | pub mod collab_plugin; 4 | mod collab_search; 5 | pub mod collab_state; 6 | pub mod fill; 7 | pub mod origin; 8 | pub mod transaction; 9 | pub mod value; 10 | -------------------------------------------------------------------------------- /collab/src/core/text_wrapper.rs: -------------------------------------------------------------------------------- 1 | use crate::preclude::{CollabContext, YrsDelta}; 2 | use std::ops::{Deref, DerefMut}; 3 | use std::sync::Arc; 4 | use yrs::types::text::{TextEvent, YChange}; 5 | use yrs::types::Delta; 6 | use yrs::{In, ReadTxn, Subscription, Text, TextRef, Transaction, TransactionMut}; 7 | pub type TextSubscriptionCallback = Arc; 8 | pub type TextSubscription = Subscription; 9 | 10 | pub struct TextRefWrapper { 11 | text_ref: TextRef, 12 | collab_ctx: CollabContext, 13 | } 14 | 15 | impl TextRefWrapper { 16 | pub fn new(text_ref: TextRef, collab_ctx: CollabContext) -> Self { 17 | Self { 18 | text_ref, 19 | collab_ctx, 20 | } 21 | } 22 | 23 | pub fn transact(&self) -> Transaction { 24 | self.collab_ctx.transact() 25 | } 26 | 27 | pub fn with_transact_mut(&self, f: F) -> T 28 | where 29 | F: FnOnce(&mut TransactionMut) -> T, 30 | { 31 | self.collab_ctx.with_transact_mut(f) 32 | } 33 | 34 | pub fn get_delta_with_txn(&self, txn: &T) -> Vec { 35 | let changes = self.text_ref.diff(txn, YChange::identity); 36 | let mut deltas = vec![]; 37 | for change in changes { 38 | let delta = YrsDelta::Inserted(change.insert, change.attributes); 39 | deltas.push(delta); 40 | } 41 | deltas 42 | } 43 | 44 | pub fn apply_delta_with_txn(&self, txn: &mut TransactionMut, delta: Vec>) { 45 | self.text_ref.apply_delta(txn, delta); 46 | } 47 | } 48 | 49 | impl Deref for TextRefWrapper { 50 | type Target = TextRef; 51 | 52 | fn deref(&self) -> &Self::Target { 53 | &self.text_ref 54 | } 55 | } 56 | 57 | impl DerefMut for TextRefWrapper { 58 | fn deref_mut(&mut self) -> &mut Self::Target { 59 | &mut self.text_ref 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /collab/src/core/transaction.rs: -------------------------------------------------------------------------------- 1 | use yrs::updates::encoder::Encode; 2 | use yrs::{ReadTxn, StateVector, Transaction, TransactionMut}; 3 | 4 | use crate::entity::EncodedCollab; 5 | 6 | pub trait DocTransactionExtension: ReadTxn { 7 | fn get_encoded_collab_v1(&self) -> EncodedCollab { 8 | EncodedCollab::new_v1( 9 | self.state_vector().encode_v1(), 10 | self.encode_state_as_update_v1(&StateVector::default()), 11 | ) 12 | } 13 | 14 | fn get_encoded_collab_v2(&self) -> EncodedCollab { 15 | EncodedCollab::new_v2( 16 | self.state_vector().encode_v2(), 17 | self.encode_state_as_update_v2(&StateVector::default()), 18 | ) 19 | } 20 | } 21 | 22 | impl DocTransactionExtension for Transaction<'_> {} 23 | impl DocTransactionExtension for TransactionMut<'_> {} 24 | -------------------------------------------------------------------------------- /collab/src/core/value.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | use yrs::block::{ItemContent, Prelim, Unused}; 3 | use yrs::branch::{Branch, BranchPtr}; 4 | use yrs::types::TypeRef; 5 | use yrs::{Any, Array, ArrayRef, Map, MapRef, TransactionMut}; 6 | 7 | #[derive(Debug, Clone, Eq, PartialEq)] 8 | pub struct Entity(Value); 9 | 10 | impl From for Entity { 11 | fn from(value: Value) -> Self { 12 | Entity(value) 13 | } 14 | } 15 | 16 | impl Prelim for Entity { 17 | type Return = Unused; 18 | 19 | fn into_content(self, _txn: &mut TransactionMut) -> (ItemContent, Option) { 20 | match &self.0 { 21 | Value::Null => (ItemContent::Any(vec![Any::Null]), None), 22 | Value::Bool(value) => (ItemContent::Any(vec![Any::from(*value)]), None), 23 | Value::String(value) => (ItemContent::Any(vec![Any::from(value.clone())]), None), 24 | Value::Number(value) => { 25 | let any = if value.is_f64() { 26 | Any::from(value.as_f64().unwrap()) 27 | } else { 28 | Any::from(value.as_i64().unwrap()) 29 | }; 30 | (ItemContent::Any(vec![any]), None) 31 | }, 32 | Value::Array(_) => { 33 | let yarray = ItemContent::Type(Branch::new(TypeRef::Array)); 34 | (yarray, Some(self)) 35 | }, 36 | Value::Object(_) => { 37 | let yarray = ItemContent::Type(Branch::new(TypeRef::Map)); 38 | (yarray, Some(self)) 39 | }, 40 | } 41 | } 42 | 43 | fn integrate(self, txn: &mut TransactionMut, inner_ref: BranchPtr) { 44 | match self.0 { 45 | Value::Array(array) => { 46 | let yarray = ArrayRef::from(inner_ref); 47 | for value in array { 48 | yarray.push_back(txn, Entity::from(value)); 49 | } 50 | }, 51 | Value::Object(map) => { 52 | let ymap = MapRef::from(inner_ref); 53 | for (key, value) in map { 54 | ymap.insert(txn, key, Entity::from(value)); 55 | } 56 | }, 57 | _ => { /* not used */ }, 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /collab/src/error.rs: -------------------------------------------------------------------------------- 1 | use anyhow::anyhow; 2 | use yrs::TransactionAcqError; 3 | 4 | #[derive(Debug, thiserror::Error)] 5 | pub enum CollabError { 6 | #[error(transparent)] 7 | SerdeJson(#[from] serde_json::Error), 8 | 9 | #[error("Unexpected empty: {0}")] 10 | UnexpectedEmpty(String), 11 | 12 | #[error("Get write txn failed")] 13 | AcquiredWriteTxnFail, 14 | 15 | #[error("Get read txn failed")] 16 | AcquiredReadTxnFail, 17 | 18 | #[error("Try apply update failed: {0}")] 19 | YrsTransactionError(String), 20 | 21 | #[error("Try encode update failed: {0}")] 22 | YrsEncodeStateError(String), 23 | 24 | #[error("UndoManager is not enabled")] 25 | UndoManagerNotEnabled, 26 | 27 | #[error(transparent)] 28 | DecodeUpdate(#[from] yrs::encoding::read::Error), 29 | 30 | #[error("{0}")] 31 | NoRequiredData(String), 32 | 33 | #[error(transparent)] 34 | Awareness(#[from] crate::core::awareness::Error), 35 | 36 | #[error("Failed to apply update: {0}")] 37 | UpdateFailed(#[from] yrs::error::UpdateError), 38 | 39 | #[error("Internal failure: {0}")] 40 | Internal(#[from] anyhow::Error), 41 | } 42 | 43 | impl From for CollabError { 44 | fn from(value: TransactionAcqError) -> Self { 45 | match value { 46 | TransactionAcqError::SharedAcqFailed => Self::AcquiredReadTxnFail, 47 | TransactionAcqError::ExclusiveAcqFailed => Self::AcquiredWriteTxnFail, 48 | TransactionAcqError::DocumentDropped => Self::Internal(anyhow!("Document dropped")), 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /collab/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[macro_export] 2 | macro_rules! if_native { 3 | ($($item:item)*) => {$( 4 | #[cfg(not(target_arch = "wasm32"))] 5 | $item 6 | )*} 7 | } 8 | 9 | #[macro_export] 10 | macro_rules! if_wasm { 11 | ($($item:item)*) => {$( 12 | #[cfg(target_arch = "wasm32")] 13 | $item 14 | )*} 15 | } 16 | 17 | mod any_mut; 18 | pub mod core; 19 | pub mod entity; 20 | pub mod error; 21 | pub mod lock; 22 | pub mod util; 23 | 24 | pub mod preclude { 25 | pub use serde_json::value::Value as JsonValue; 26 | pub use yrs::In as YrsInput; 27 | pub use yrs::Out as YrsValue; 28 | pub use yrs::block::ClientID; 29 | pub use yrs::block::Prelim; 30 | pub use yrs::types::{ 31 | AsPrelim, Attrs, Delta as YrsDelta, EntryChange, GetString, Observable, ToJson, array::Array, *, 32 | }; 33 | pub use yrs::*; 34 | 35 | pub use crate::any_mut::AnyMut; 36 | pub use crate::core::collab::Collab; 37 | pub use crate::core::collab_plugin::CollabPlugin; 38 | pub use crate::core::fill::{FillError, FillRef}; 39 | pub use crate::util::MapExt; 40 | pub use crate::util::deserialize_i32_from_numeric; 41 | pub use crate::util::deserialize_i64_from_numeric; 42 | } 43 | -------------------------------------------------------------------------------- /collab/src/lock/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(not(feature = "lock_timeout"))] 2 | pub type Mutex = tokio::sync::Mutex; 3 | #[cfg(not(feature = "lock_timeout"))] 4 | pub type RwLock = tokio::sync::RwLock; 5 | 6 | #[cfg(feature = "lock_timeout")] 7 | mod lock_timeout; 8 | 9 | #[cfg(feature = "lock_timeout")] 10 | pub use lock_timeout::Mutex; 11 | #[cfg(feature = "lock_timeout")] 12 | pub use lock_timeout::RwLock; 13 | -------------------------------------------------------------------------------- /collab/tests/edit_test/mod.rs: -------------------------------------------------------------------------------- 1 | mod awareness_test; 2 | mod insert_test; 3 | mod observer_test; 4 | mod restore_test; 5 | mod state_vec_test; 6 | -------------------------------------------------------------------------------- /collab/tests/main.rs: -------------------------------------------------------------------------------- 1 | mod edit_test; 2 | mod util; 3 | -------------------------------------------------------------------------------- /docs/collab_object-CollabPlugins.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/docs/collab_object-CollabPlugins.png -------------------------------------------------------------------------------- /docs/collab_object-Create_Document.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/docs/collab_object-Create_Document.png -------------------------------------------------------------------------------- /docs/collab_object-Edit_Document.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/docs/collab_object-Edit_Document.png -------------------------------------------------------------------------------- /docs/collab_object-Open_Document.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/docs/collab_object-Open_Document.png -------------------------------------------------------------------------------- /docs/collab_object-Sync_Document.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/docs/collab_object-Sync_Document.png -------------------------------------------------------------------------------- /docs/collab_object.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/docs/collab_object.png -------------------------------------------------------------------------------- /docs/create_collab_object-CreateReminder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/docs/create_collab_object-CreateReminder.png -------------------------------------------------------------------------------- /docs/create_collab_object-UserAwareness.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/docs/create_collab_object-UserAwareness.png -------------------------------------------------------------------------------- /docs/create_collab_object.plantuml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title UserAwareness 3 | left to right direction 4 | class UserAwareness { 5 | appearance_settings: Map, 6 | reminders: Vec, 7 | } 8 | 9 | struct Reminder { 10 | id: String, 11 | scheduled_at: i64, 12 | is_ack: bool, 13 | ty: i64, 14 | title: String, 15 | message: String, 16 | reminder_object_id: String, 17 | } 18 | 19 | UserAwareness "1" -- "0..*" Reminder 20 | @enduml 21 | 22 | @startuml 23 | title CreateReminder 24 | actor User 25 | entity "User Device 1" as Device1 26 | entity "User Device 2" as Device2 27 | entity "User Device 3" as Device3 28 | database Server 29 | entity "User Interface" as UI 30 | 31 | User -> Device1: Logs in 32 | User -> Device2: Logs in with the same User ID 33 | User -> Device3: Logs in with the same User ID 34 | Device1 -> Server: Sends login information 35 | Device2 -> Server: Sends login information 36 | Device3 -> Server: Sends login information 37 | Server -> Device1: Synchronizes `UserAwareness` object 38 | Server -> Device2: Synchronizes `UserAwareness` object 39 | Server -> Device3: Synchronizes `UserAwareness` object 40 | 41 | User -> Device1: Creates a new reminder 42 | Device1 -> Server: Sends reminder update 43 | Server -> Device1: Broadcasts reminder update 44 | Server -> Device2: Broadcasts reminder update 45 | Server -> Device3: Broadcasts reminder update 46 | Device1 -> UI: Updates reminder list 47 | Device2 -> UI: Updates reminder list 48 | Device3 -> UI: Updates reminder list 49 | @enduml 50 | -------------------------------------------------------------------------------- /resources/crate_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppFlowy-IO/AppFlowy-Collab/b03aed27f6c4cd563a5863c928b8e5cd66d572a7/resources/crate_arch.png -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.85" 3 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # https://rust-lang.github.io/rustfmt/?version=master&search= 2 | max_width = 100 3 | tab_spaces = 2 4 | newline_style = "Auto" 5 | match_block_trailing_comma = true 6 | use_field_init_shorthand = true 7 | use_try_shorthand = true 8 | reorder_imports = true 9 | reorder_modules = true 10 | remove_nested_parens = true 11 | merge_derives = true 12 | edition = "2024" --------------------------------------------------------------------------------