├── .github └── workflows │ └── main.yml ├── .gitignore ├── Cargo.toml ├── LICENSE ├── README.md └── src ├── awareness.rs ├── lib.rs ├── net ├── broadcast.rs ├── conn.rs └── mod.rs └── sync.rs /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Main 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | build: 14 | runs-on: ${{ matrix.os }} 15 | strategy: 16 | matrix: 17 | os: [ubuntu-latest] 18 | 19 | steps: 20 | 21 | - name: checkout sources 22 | uses: actions/checkout@v2 23 | 24 | - name: install Rust toolchain 25 | uses: actions-rs/toolchain@v1 26 | with: 27 | toolchain: stable 28 | override: true 29 | 30 | - name: build default 31 | run: cargo build --verbose --release 32 | 33 | test-linux: 34 | runs-on: ubuntu-latest 35 | needs: build 36 | steps: 37 | - name: checkout sources 38 | uses: actions/checkout@v2 39 | 40 | - name: test 41 | run: cargo test --release --features=net -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | .idea -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "y-sync" 3 | version = "0.4.0" 4 | edition = "2021" 5 | description = "Yrs synchronization protocol" 6 | license-file = "LICENSE" 7 | authors = ["Bartosz Sypytkowski "] 8 | keywords = ["crdt", "yrs", "y-sync"] 9 | homepage = "https://github.com/y-crdt/y-sync/" 10 | repository = "https://github.com/y-crdt/y-sync/" 11 | readme = "./README.md" 12 | 13 | [features] 14 | net = ["dep:tokio", "dep:futures-util"] 15 | 16 | [dependencies] 17 | yrs = "0.17" 18 | thiserror = "1.0" 19 | tokio = { version = "1.26.0", features = ["net", "sync"], optional = true } 20 | futures-util = { version = "0.3", features = ["sink"], optional = true } 21 | 22 | [dev-dependencies] 23 | tokio = { version = "1.26.0", features = ["full"] } 24 | tokio-util = { version ="0.7", features = ["codec"] } 25 | bytes = "1.4" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2022 4 | - Bartosz Sypytkowski 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Yjs/Yrs sync protocol implementation 2 | 3 | This crate comes with the default implementation of [Yjs/Yrs synchronization protocol](https://github.com/yjs/y-protocols/blob/master/PROTOCOL.md) - it's shared between the projects and can be used to interop between eg. Yjs client and Yrs server. 4 | 5 | It also implements [awareness protocol](https://docs.yjs.dev/api/about-awareness) and corresponding `Awareness` structure. 6 | 7 | The implementation just like the protocol itself is flexible and can be extended by custom implementation of `Protocol` trait. If that's not necessary, the `DefaultProtocol` is provided. 8 | 9 | ## Sponsors 10 | 11 | [![NLNET](https://nlnet.nl/image/logo_nlnet.svg)](https://nlnet.nl/) 12 | -------------------------------------------------------------------------------- /src/awareness.rs: -------------------------------------------------------------------------------- 1 | use std::collections::hash_map::Entry; 2 | use std::collections::HashMap; 3 | use std::fmt::Formatter; 4 | use std::sync::Arc; 5 | use std::time::Instant; 6 | use thiserror::Error; 7 | use yrs::block::ClientID; 8 | use yrs::encoding::read; 9 | use yrs::updates::decoder::{Decode, Decoder}; 10 | use yrs::updates::encoder::{Encode, Encoder}; 11 | use yrs::{Doc, Observer, Subscription}; 12 | 13 | const NULL_STR: &str = "null"; 14 | 15 | /// The Awareness class implements a simple shared state protocol that can be used for non-persistent 16 | /// data like awareness information (cursor, username, status, ..). Each client can update its own 17 | /// local state and listen to state changes of remote clients. 18 | /// 19 | /// Each client is identified by a unique client id (something we borrow from `doc.clientID`). 20 | /// A client can override its own state by propagating a message with an increasing timestamp 21 | /// (`clock`). If such a message is received, it is applied if the known state of that client is 22 | /// older than the new state (`clock < new_clock`). If a client thinks that a remote client is 23 | /// offline, it may propagate a message with `{ clock, state: null, client }`. If such a message is 24 | /// received, and the known clock of that client equals the received clock, it will clean the state. 25 | /// 26 | /// Before a client disconnects, it should propagate a `null` state with an updated clock. 27 | pub struct Awareness { 28 | doc: Doc, 29 | states: HashMap, 30 | meta: HashMap, 31 | on_update: Option () + 'static>>>, 32 | } 33 | 34 | unsafe impl Send for Awareness {} 35 | unsafe impl Sync for Awareness {} 36 | 37 | impl Awareness { 38 | /// Creates a new instance of [Awareness] struct, which operates over a given document. 39 | /// Awareness instance has full ownership of that document. If necessary it can be accessed 40 | /// using either [Awareness::doc] or [Awareness::doc_mut] methods. 41 | pub fn new(doc: Doc) -> Self { 42 | Awareness { 43 | doc, 44 | on_update: None, 45 | states: HashMap::new(), 46 | meta: HashMap::new(), 47 | } 48 | } 49 | 50 | /// Returns a channel receiver for an incoming awareness events. This channel can be cloned. 51 | pub fn on_update(&mut self, f: F) -> UpdateSubscription 52 | where 53 | F: Fn(&Awareness, &Event) -> () + 'static, 54 | { 55 | let eh = self.on_update.get_or_insert_with(Observer::default); 56 | eh.subscribe(Arc::new(f)) 57 | } 58 | 59 | /// Returns a read-only reference to an underlying [Doc]. 60 | pub fn doc(&self) -> &Doc { 61 | &self.doc 62 | } 63 | 64 | /// Returns a read-write reference to an underlying [Doc]. 65 | pub fn doc_mut(&mut self) -> &mut Doc { 66 | &mut self.doc 67 | } 68 | 69 | /// Returns a globally unique client ID of an underlying [Doc]. 70 | pub fn client_id(&self) -> ClientID { 71 | self.doc.client_id() 72 | } 73 | 74 | /// Returns a state map of all of the clients tracked by current [Awareness] instance. Those 75 | /// states are identified by their corresponding [ClientID]s. The associated state is 76 | /// represented and replicated to other clients as a JSON string. 77 | pub fn clients(&self) -> &HashMap { 78 | &self.states 79 | } 80 | 81 | /// Returns a JSON string state representation of a current [Awareness] instance. 82 | pub fn local_state(&self) -> Option<&str> { 83 | Some(self.states.get(&self.doc.client_id())?.as_str()) 84 | } 85 | 86 | /// Sets a current [Awareness] instance state to a corresponding JSON string. This state will 87 | /// be replicated to other clients as part of the [AwarenessUpdate] and it will trigger an event 88 | /// to be emitted if current instance was created using [Awareness::with_observer] method. 89 | /// 90 | pub fn set_local_state>(&mut self, json: S) { 91 | let client_id = self.doc.client_id(); 92 | self.update_meta(client_id); 93 | let new: String = json.into(); 94 | match self.states.entry(client_id) { 95 | Entry::Occupied(mut e) => { 96 | e.insert(new); 97 | if let Some(eh) = self.on_update.as_ref() { 98 | let e = Event::new(vec![], vec![client_id], vec![]); 99 | for cb in eh.callbacks() { 100 | cb(self, &e); 101 | } 102 | } 103 | } 104 | Entry::Vacant(e) => { 105 | e.insert(new); 106 | if let Some(eh) = self.on_update.as_ref() { 107 | let e = Event::new(vec![client_id], vec![], vec![]); 108 | for cb in eh.callbacks() { 109 | cb(self, &e); 110 | } 111 | } 112 | } 113 | } 114 | } 115 | 116 | /// Clears out a state of a given client, effectively marking it as disconnected. 117 | pub fn remove_state(&mut self, client_id: ClientID) { 118 | let prev_state = self.states.remove(&client_id); 119 | self.update_meta(client_id); 120 | if let Some(eh) = self.on_update.as_ref() { 121 | if prev_state.is_some() { 122 | let e = Event::new(Vec::default(), Vec::default(), vec![client_id]); 123 | for cb in eh.callbacks() { 124 | cb(self, &e); 125 | } 126 | } 127 | } 128 | } 129 | 130 | /// Clears out a state of a current client (see: [Awareness::client_id]), 131 | /// effectively marking it as disconnected. 132 | pub fn clean_local_state(&mut self) { 133 | let client_id = self.doc.client_id(); 134 | self.remove_state(client_id); 135 | } 136 | 137 | fn update_meta(&mut self, client_id: ClientID) { 138 | match self.meta.entry(client_id) { 139 | Entry::Occupied(mut e) => { 140 | let clock = e.get().clock + 1; 141 | let meta = MetaClientState::new(clock, Instant::now()); 142 | e.insert(meta); 143 | } 144 | Entry::Vacant(e) => { 145 | e.insert(MetaClientState::new(1, Instant::now())); 146 | } 147 | } 148 | } 149 | 150 | /// Returns a serializable update object which is representation of a current Awareness state. 151 | pub fn update(&self) -> Result { 152 | let clients = self.states.keys().cloned(); 153 | self.update_with_clients(clients) 154 | } 155 | 156 | /// Returns a serializable update object which is representation of a current Awareness state. 157 | /// Unlike [Awareness::update], this method variant allows to prepare update only for a subset 158 | /// of known clients. These clients must all be known to a current [Awareness] instance, 159 | /// otherwise a [Error::ClientNotFound] error will be returned. 160 | pub fn update_with_clients>( 161 | &self, 162 | clients: I, 163 | ) -> Result { 164 | let mut res = HashMap::new(); 165 | for client_id in clients { 166 | let clock = if let Some(meta) = self.meta.get(&client_id) { 167 | meta.clock 168 | } else { 169 | return Err(Error::ClientNotFound(client_id)); 170 | }; 171 | let json = if let Some(json) = self.states.get(&client_id) { 172 | json.clone() 173 | } else { 174 | String::from(NULL_STR) 175 | }; 176 | res.insert(client_id, AwarenessUpdateEntry { clock, json }); 177 | } 178 | Ok(AwarenessUpdate { clients: res }) 179 | } 180 | 181 | /// Applies an update (incoming from remote channel or generated using [Awareness::update] / 182 | /// [Awareness::update_with_clients] methods) and modifies a state of a current instance. 183 | /// 184 | /// If current instance has an observer channel (see: [Awareness::with_observer]), applied 185 | /// changes will also be emitted as events. 186 | pub fn apply_update(&mut self, update: AwarenessUpdate) -> Result<(), Error> { 187 | let now = Instant::now(); 188 | 189 | let mut added = Vec::new(); 190 | let mut updated = Vec::new(); 191 | let mut removed = Vec::new(); 192 | 193 | for (client_id, entry) in update.clients { 194 | let mut clock = entry.clock; 195 | let is_null = entry.json.as_str() == NULL_STR; 196 | match self.meta.entry(client_id) { 197 | Entry::Occupied(mut e) => { 198 | let prev = e.get(); 199 | let is_removed = 200 | prev.clock == clock && is_null && self.states.contains_key(&client_id); 201 | let is_new = prev.clock < clock; 202 | if is_new || is_removed { 203 | if is_null { 204 | // never let a remote client remove this local state 205 | if client_id == self.doc.client_id() 206 | && self.states.get(&client_id).is_some() 207 | { 208 | // remote client removed the local state. Do not remote state. Broadcast a message indicating 209 | // that this client still exists by increasing the clock 210 | clock += 1; 211 | } else { 212 | self.states.remove(&client_id); 213 | if self.on_update.is_some() { 214 | removed.push(client_id); 215 | } 216 | } 217 | } else { 218 | match self.states.entry(client_id) { 219 | Entry::Occupied(mut e) => { 220 | if self.on_update.is_some() { 221 | updated.push(client_id); 222 | } 223 | e.insert(entry.json); 224 | } 225 | Entry::Vacant(e) => { 226 | e.insert(entry.json); 227 | if self.on_update.is_some() { 228 | updated.push(client_id); 229 | } 230 | } 231 | } 232 | } 233 | e.insert(MetaClientState::new(clock, now)); 234 | true 235 | } else { 236 | false 237 | } 238 | } 239 | Entry::Vacant(e) => { 240 | e.insert(MetaClientState::new(clock, now)); 241 | self.states.insert(client_id, entry.json); 242 | if self.on_update.is_some() { 243 | added.push(client_id); 244 | } 245 | true 246 | } 247 | }; 248 | } 249 | 250 | if let Some(eh) = self.on_update.as_ref() { 251 | if !added.is_empty() || !updated.is_empty() || !removed.is_empty() { 252 | let e = Event::new(added, updated, removed); 253 | for cb in eh.callbacks() { 254 | cb(self, &e); 255 | } 256 | } 257 | } 258 | 259 | Ok(()) 260 | } 261 | } 262 | 263 | impl Default for Awareness { 264 | fn default() -> Self { 265 | Awareness::new(Doc::new()) 266 | } 267 | } 268 | 269 | impl std::fmt::Debug for Awareness { 270 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 271 | f.debug_struct("Awareness") 272 | .field("state", &self.states) 273 | .field("meta", &self.meta) 274 | .field("doc", &self.doc) 275 | .finish() 276 | } 277 | } 278 | 279 | /// Whenever a new callback is being registered, a [Subscription] is made. Whenever this 280 | /// subscription a registered callback is cancelled and will not be called any more. 281 | pub type UpdateSubscription = Subscription () + 'static>>; 282 | 283 | /// A structure that represents an encodable state of an [Awareness] struct. 284 | #[derive(Debug, Eq, PartialEq)] 285 | pub struct AwarenessUpdate { 286 | pub(crate) clients: HashMap, 287 | } 288 | 289 | impl Encode for AwarenessUpdate { 290 | fn encode(&self, encoder: &mut E) { 291 | encoder.write_var(self.clients.len()); 292 | for (&client_id, e) in self.clients.iter() { 293 | encoder.write_var(client_id); 294 | encoder.write_var(e.clock); 295 | encoder.write_string(&e.json); 296 | } 297 | } 298 | } 299 | 300 | impl Decode for AwarenessUpdate { 301 | fn decode(decoder: &mut D) -> Result { 302 | let len: usize = decoder.read_var()?; 303 | let mut clients = HashMap::with_capacity(len); 304 | for _ in 0..len { 305 | let client_id: ClientID = decoder.read_var()?; 306 | let clock: u32 = decoder.read_var()?; 307 | let json = decoder.read_string()?.to_string(); 308 | clients.insert(client_id, AwarenessUpdateEntry { clock, json }); 309 | } 310 | 311 | Ok(AwarenessUpdate { clients }) 312 | } 313 | } 314 | 315 | /// A single client entry of an [AwarenessUpdate]. It consists of logical clock and JSON client 316 | /// state represented as a string. 317 | #[derive(Debug, Eq, PartialEq)] 318 | pub struct AwarenessUpdateEntry { 319 | pub(crate) clock: u32, 320 | pub(crate) json: String, 321 | } 322 | 323 | /// Errors generated by an [Awareness] struct methods. 324 | #[derive(Error, Debug)] 325 | pub enum Error { 326 | /// Client ID was not found in [Awareness] metadata. 327 | #[error("client ID `{0}` not found")] 328 | ClientNotFound(ClientID), 329 | } 330 | 331 | #[derive(Debug, Clone, PartialEq, Eq)] 332 | struct MetaClientState { 333 | clock: u32, 334 | last_updated: Instant, 335 | } 336 | 337 | impl MetaClientState { 338 | fn new(clock: u32, last_updated: Instant) -> Self { 339 | MetaClientState { 340 | clock, 341 | last_updated, 342 | } 343 | } 344 | } 345 | 346 | /// Event type emitted by an [Awareness] struct. 347 | #[derive(Debug, Default, Clone, Eq, PartialEq)] 348 | pub struct Event { 349 | added: Vec, 350 | updated: Vec, 351 | removed: Vec, 352 | } 353 | 354 | impl Event { 355 | pub fn new(added: Vec, updated: Vec, removed: Vec) -> Self { 356 | Event { 357 | added, 358 | updated, 359 | removed, 360 | } 361 | } 362 | 363 | /// Collection of new clients that have been added to an [Awareness] struct, that was not known 364 | /// before. Actual client state can be accessed via `awareness.clients().get(client_id)`. 365 | pub fn added(&self) -> &[ClientID] { 366 | &self.added 367 | } 368 | 369 | /// Collection of new clients that have been updated within an [Awareness] struct since the last 370 | /// update. Actual client state can be accessed via `awareness.clients().get(client_id)`. 371 | pub fn updated(&self) -> &[ClientID] { 372 | &self.updated 373 | } 374 | 375 | /// Collection of new clients that have been removed from [Awareness] struct since the last 376 | /// update. 377 | pub fn removed(&self) -> &[ClientID] { 378 | &self.removed 379 | } 380 | } 381 | 382 | #[cfg(test)] 383 | mod test { 384 | use crate::awareness::{Awareness, Event}; 385 | use std::sync::mpsc::{channel, Receiver}; 386 | use yrs::Doc; 387 | 388 | fn update( 389 | recv: &mut Receiver, 390 | from: &Awareness, 391 | to: &mut Awareness, 392 | ) -> Result> { 393 | let e = recv.try_recv()?; 394 | let u = from.update_with_clients([e.added(), e.updated(), e.removed()].concat())?; 395 | to.apply_update(u)?; 396 | Ok(e) 397 | } 398 | 399 | #[test] 400 | fn awareness() -> Result<(), Box> { 401 | let (s1, mut o_local) = channel(); 402 | let mut local = Awareness::new(Doc::with_client_id(1)); 403 | let _sub_local = local.on_update(move |_, e| { 404 | s1.send(e.clone()).unwrap(); 405 | }); 406 | 407 | let (s2, o_remote) = channel(); 408 | let mut remote = Awareness::new(Doc::with_client_id(2)); 409 | let _sub_remote = local.on_update(move |_, e| { 410 | s2.send(e.clone()).unwrap(); 411 | }); 412 | 413 | local.set_local_state("{x:3}"); 414 | let _e_local = update(&mut o_local, &local, &mut remote)?; 415 | assert_eq!(remote.clients()[&1], "{x:3}"); 416 | assert_eq!(remote.meta[&1].clock, 1); 417 | assert_eq!(o_remote.try_recv()?.added, &[1]); 418 | 419 | local.set_local_state("{x:4}"); 420 | let e_local = update(&mut o_local, &local, &mut remote)?; 421 | let e_remote = o_remote.try_recv()?; 422 | assert_eq!(remote.clients()[&1], "{x:4}"); 423 | assert_eq!(e_remote, Event::new(vec![], vec![1], vec![])); 424 | assert_eq!(e_remote, e_local); 425 | 426 | local.clean_local_state(); 427 | let e_local = update(&mut o_local, &local, &mut remote)?; 428 | let e_remote = o_remote.try_recv()?; 429 | assert_eq!(e_remote.removed.len(), 1); 430 | assert_eq!(local.clients().get(&1), None); 431 | assert_eq!(e_remote, e_local); 432 | Ok(()) 433 | } 434 | } 435 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod awareness; 2 | pub mod sync; 3 | 4 | #[cfg(feature = "net")] 5 | pub mod net; 6 | -------------------------------------------------------------------------------- /src/net/broadcast.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | use crate::awareness; 3 | use crate::awareness::Awareness; 4 | use crate::net::conn::handle_msg; 5 | use crate::sync::{DefaultProtocol, Error, Message, Protocol, MSG_SYNC, MSG_SYNC_UPDATE}; 6 | use futures_util::{SinkExt, StreamExt}; 7 | use std::sync::Arc; 8 | use tokio::select; 9 | use tokio::sync::broadcast::error::SendError; 10 | use tokio::sync::broadcast::{channel, Receiver, Sender}; 11 | use tokio::sync::{Mutex, RwLock}; 12 | use tokio::task::JoinHandle; 13 | use yrs::encoding::write::Write; 14 | use yrs::updates::decoder::Decode; 15 | use yrs::updates::encoder::{Encode, Encoder, EncoderV1}; 16 | use yrs::UpdateSubscription; 17 | 18 | /// A broadcast group can be used to propagate updates produced by yrs [yrs::Doc] and [Awareness] 19 | /// structures in a binary form that conforms to a y-sync protocol. 20 | /// 21 | /// New receivers can subscribe to a broadcasting group via [BroadcastGroup::subscribe] method. 22 | pub struct BroadcastGroup { 23 | awareness_sub: awareness::UpdateSubscription, 24 | doc_sub: UpdateSubscription, 25 | awareness_ref: Arc>, 26 | sender: Sender>, 27 | receiver: Receiver>, 28 | } 29 | 30 | unsafe impl Send for BroadcastGroup {} 31 | unsafe impl Sync for BroadcastGroup {} 32 | 33 | impl BroadcastGroup { 34 | /// Creates a new [BroadcastGroup] over a provided `awareness` instance. All changes triggered 35 | /// by this awareness structure or its underlying document will be propagated to all subscribers 36 | /// which have been registered via [BroadcastGroup::subscribe] method. 37 | /// 38 | /// The overflow of the incoming events that needs to be propagates will be buffered up to a 39 | /// provided `buffer_capacity` size. 40 | pub async fn new(awareness: Arc>, buffer_capacity: usize) -> Self { 41 | let (sender, receiver) = channel(buffer_capacity); 42 | let (doc_sub, awareness_sub) = { 43 | let mut awareness = awareness.write().await; 44 | let sink = sender.clone(); 45 | let doc_sub = awareness 46 | .doc_mut() 47 | .observe_update_v1(move |_txn, u| { 48 | // we manually construct msg here to avoid update data copying 49 | let mut encoder = EncoderV1::new(); 50 | encoder.write_var(MSG_SYNC); 51 | encoder.write_var(MSG_SYNC_UPDATE); 52 | encoder.write_buf(&u.update); 53 | let msg = encoder.to_vec(); 54 | if let Err(_e) = sink.send(msg) { 55 | // current broadcast group is being closed 56 | } 57 | }) 58 | .unwrap(); 59 | let sink = sender.clone(); 60 | let awareness_sub = awareness.on_update(move |awareness, e| { 61 | let added = e.added(); 62 | let updated = e.updated(); 63 | let removed = e.removed(); 64 | let mut changed = Vec::with_capacity(added.len() + updated.len() + removed.len()); 65 | changed.extend_from_slice(added); 66 | changed.extend_from_slice(updated); 67 | changed.extend_from_slice(removed); 68 | 69 | if let Ok(u) = awareness.update_with_clients(changed) { 70 | let msg = Message::Awareness(u).encode_v1(); 71 | if let Err(_e) = sink.send(msg) { 72 | // current broadcast group is being closed 73 | } 74 | } 75 | }); 76 | (doc_sub, awareness_sub) 77 | }; 78 | BroadcastGroup { 79 | awareness_ref: awareness, 80 | sender, 81 | receiver, 82 | awareness_sub, 83 | doc_sub, 84 | } 85 | } 86 | 87 | /// Returns a reference to an underlying [Awareness] instance. 88 | pub fn awareness(&self) -> &Arc> { 89 | &self.awareness_ref 90 | } 91 | 92 | /// Broadcasts user message to all active subscribers. Returns error if message could not have 93 | /// been broadcasted. 94 | pub fn broadcast(&self, msg: Vec) -> Result<(), SendError>> { 95 | self.sender.send(msg)?; 96 | Ok(()) 97 | } 98 | 99 | /// Subscribes a new connection - represented by `sink`/`stream` pair implementing a futures 100 | /// Sink and Stream protocols - to a current broadcast group. 101 | /// 102 | /// Returns a subscription structure, which can be dropped in order to unsubscribe or awaited 103 | /// via [Subscription::completed] method in order to complete of its own volition (due to 104 | /// an internal connection error or closed connection). 105 | pub fn subscribe(&self, sink: Arc>, stream: Stream) -> Subscription 106 | where 107 | Sink: SinkExt> + Send + Sync + Unpin + 'static, 108 | Stream: StreamExt, E>> + Send + Sync + Unpin + 'static, 109 | >>::Error: std::error::Error + Send + Sync, 110 | E: std::error::Error + Send + Sync + 'static, 111 | { 112 | self.subscribe_with(sink, stream, DefaultProtocol) 113 | } 114 | 115 | /// Subscribes a new connection - represented by `sink`/`stream` pair implementing a futures 116 | /// Sink and Stream protocols - to a current broadcast group. 117 | /// 118 | /// Returns a subscription structure, which can be dropped in order to unsubscribe or awaited 119 | /// via [Subscription::completed] method in order to complete of its own volition (due to 120 | /// an internal connection error or closed connection). 121 | /// 122 | /// Unlike [BroadcastGroup::subscribe], this method can take [Protocol] parameter that allows to 123 | /// customize the y-sync protocol behavior. 124 | pub fn subscribe_with( 125 | &self, 126 | sink: Arc>, 127 | mut stream: Stream, 128 | protocol: P, 129 | ) -> Subscription 130 | where 131 | Sink: SinkExt> + Send + Sync + Unpin + 'static, 132 | Stream: StreamExt, E>> + Send + Sync + Unpin + 'static, 133 | >>::Error: std::error::Error + Send + Sync, 134 | E: std::error::Error + Send + Sync + 'static, 135 | P: Protocol + Send + Sync + 'static, 136 | { 137 | let sink_task = { 138 | let sink = sink.clone(); 139 | let mut receiver = self.sender.subscribe(); 140 | tokio::spawn(async move { 141 | while let Ok(msg) = receiver.recv().await { 142 | let mut sink = sink.lock().await; 143 | if let Err(e) = sink.send(msg).await { 144 | println!("broadcast failed to sent sync message"); 145 | return Err(Error::Other(Box::new(e))); 146 | } 147 | } 148 | Ok(()) 149 | }) 150 | }; 151 | let stream_task = { 152 | let awareness = self.awareness().clone(); 153 | tokio::spawn(async move { 154 | while let Some(res) = stream.next().await { 155 | let msg = Message::decode_v1(&res.map_err(|e| Error::Other(Box::new(e)))?)?; 156 | let reply = handle_msg(&protocol, &awareness, msg).await?; 157 | match reply { 158 | None => {} 159 | Some(reply) => { 160 | let mut sink = sink.lock().await; 161 | sink.send(reply.encode_v1()) 162 | .await 163 | .map_err(|e| Error::Other(Box::new(e)))?; 164 | } 165 | } 166 | } 167 | Ok(()) 168 | }) 169 | }; 170 | 171 | Subscription { 172 | sink_task, 173 | stream_task, 174 | } 175 | } 176 | } 177 | 178 | /// A subscription structure returned from [BroadcastGroup::subscribe], which represents a 179 | /// subscribed connection. It can be dropped in order to unsubscribe or awaited via 180 | /// [Subscription::completed] method in order to complete of its own volition (due to an internal 181 | /// connection error or closed connection). 182 | #[derive(Debug)] 183 | pub struct Subscription { 184 | sink_task: JoinHandle>, 185 | stream_task: JoinHandle>, 186 | } 187 | 188 | impl Subscription { 189 | /// Consumes current subscription, waiting for it to complete. If an underlying connection was 190 | /// closed because of failure, an error which caused it to happen will be returned. 191 | /// 192 | /// This method doesn't invoke close procedure. If you need that, drop current subscription instead. 193 | pub async fn completed(self) -> Result<(), Error> { 194 | let res = select! { 195 | r1 = self.sink_task => r1?, 196 | r2 = self.stream_task => r2?, 197 | }; 198 | res 199 | } 200 | } 201 | 202 | #[cfg(test)] 203 | mod test { 204 | use crate::awareness::{Awareness, AwarenessUpdate, AwarenessUpdateEntry}; 205 | use crate::net::broadcast::BroadcastGroup; 206 | use crate::sync::{Error, Message, SyncMessage}; 207 | use futures_util::{ready, SinkExt, StreamExt}; 208 | use std::collections::HashMap; 209 | use std::pin::Pin; 210 | use std::sync::Arc; 211 | use std::task::{Context, Poll}; 212 | use tokio::sync::{Mutex, RwLock}; 213 | use tokio_util::sync::PollSender; 214 | use yrs::updates::decoder::Decode; 215 | use yrs::updates::encoder::Encode; 216 | use yrs::{Doc, StateVector, Text, Transact}; 217 | 218 | #[derive(Debug)] 219 | pub struct ReceiverStream { 220 | inner: tokio::sync::mpsc::Receiver, 221 | } 222 | 223 | impl ReceiverStream { 224 | /// Create a new `ReceiverStream`. 225 | pub fn new(recv: tokio::sync::mpsc::Receiver) -> Self { 226 | Self { inner: recv } 227 | } 228 | } 229 | 230 | impl futures_util::Stream for ReceiverStream { 231 | type Item = Result; 232 | 233 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 234 | match ready!(self.inner.poll_recv(cx)) { 235 | None => Poll::Ready(None), 236 | Some(v) => Poll::Ready(Some(Ok(v))), 237 | } 238 | } 239 | } 240 | 241 | fn test_channel(capacity: usize) -> (PollSender>, ReceiverStream>) { 242 | let (s, r) = tokio::sync::mpsc::channel::>(capacity); 243 | let s = PollSender::new(s); 244 | let r = ReceiverStream::new(r); 245 | (s, r) 246 | } 247 | 248 | #[tokio::test] 249 | async fn broadcast_changes() -> Result<(), Box> { 250 | let doc = Doc::with_client_id(1); 251 | let text = doc.get_or_insert_text("test"); 252 | let awareness = Arc::new(RwLock::new(Awareness::new(doc))); 253 | let group = BroadcastGroup::new(awareness.clone(), 1).await; 254 | 255 | let (server_sender, mut client_receiver) = test_channel(1); 256 | let (mut client_sender, server_receiver) = test_channel(1); 257 | let _sub1 = group.subscribe(Arc::new(Mutex::new(server_sender)), server_receiver); 258 | 259 | // check update propagation 260 | { 261 | let a = awareness.write().await; 262 | text.push(&mut a.doc().transact_mut(), "a"); 263 | } 264 | let msg = client_receiver.next().await; 265 | let msg = msg.map(|x| Message::decode_v1(&x.unwrap()).unwrap()); 266 | assert_eq!( 267 | msg, 268 | Some(Message::Sync(SyncMessage::Update(vec![ 269 | 1, 1, 1, 0, 4, 1, 4, 116, 101, 115, 116, 1, 97, 0, 270 | ]))) 271 | ); 272 | 273 | // check awareness update propagation 274 | { 275 | let mut a = awareness.write().await; 276 | a.set_local_state(r#"{"key":"value"}"#) 277 | } 278 | 279 | let msg = client_receiver.next().await; 280 | let msg = msg.map(|x| Message::decode_v1(&x.unwrap()).unwrap()); 281 | assert_eq!( 282 | msg, 283 | Some(Message::Awareness(AwarenessUpdate { 284 | clients: HashMap::from([( 285 | 1, 286 | AwarenessUpdateEntry { 287 | clock: 1, 288 | json: r#"{"key":"value"}"#.to_string(), 289 | }, 290 | )]), 291 | })) 292 | ); 293 | 294 | // check sync state request/response 295 | { 296 | client_sender 297 | .send(Message::Sync(SyncMessage::SyncStep1(StateVector::default())).encode_v1()) 298 | .await?; 299 | let msg = client_receiver.next().await; 300 | let msg = msg.map(|x| Message::decode_v1(&x.unwrap()).unwrap()); 301 | assert_eq!( 302 | msg, 303 | Some(Message::Sync(SyncMessage::SyncStep2(vec![ 304 | 1, 1, 1, 0, 4, 1, 4, 116, 101, 115, 116, 1, 97, 0, 305 | ]))) 306 | ); 307 | } 308 | 309 | Ok(()) 310 | } 311 | } 312 | -------------------------------------------------------------------------------- /src/net/conn.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | use crate::awareness::Awareness; 3 | use crate::sync::{DefaultProtocol, Error, Message, MessageReader, Protocol, SyncMessage}; 4 | use futures_util::sink::SinkExt; 5 | use futures_util::StreamExt; 6 | use std::future::Future; 7 | use std::marker::PhantomData; 8 | use std::pin::Pin; 9 | use std::sync::{Arc, Weak}; 10 | use std::task::{Context, Poll}; 11 | use tokio::spawn; 12 | use tokio::sync::{Mutex, RwLock}; 13 | use tokio::task::JoinHandle; 14 | use yrs::encoding::read::Cursor; 15 | use yrs::updates::decoder::{Decode, DecoderV1}; 16 | use yrs::updates::encoder::{Encode, Encoder, EncoderV1}; 17 | use yrs::Update; 18 | 19 | /// Connection handler over a pair of message streams, which implements a Yjs/Yrs awareness and 20 | /// update exchange protocol. 21 | /// 22 | /// This connection implements Future pattern and can be awaited upon in order for a caller to 23 | /// recognize whether underlying websocket connection has been finished gracefully or abruptly. 24 | #[derive(Debug)] 25 | pub struct Connection { 26 | processing_loop: JoinHandle>, 27 | awareness: Arc>, 28 | inbox: Arc>, 29 | _stream: PhantomData, 30 | } 31 | 32 | impl Connection 33 | where 34 | Sink: SinkExt, Error = E> + Send + Sync + Unpin + 'static, 35 | E: Into + Send + Sync, 36 | { 37 | pub async fn send(&self, msg: Vec) -> Result<(), Error> { 38 | let mut inbox = self.inbox.lock().await; 39 | match inbox.send(msg).await { 40 | Ok(_) => Ok(()), 41 | Err(err) => Err(err.into()), 42 | } 43 | } 44 | 45 | pub async fn close(self) -> Result<(), E> { 46 | let mut inbox = self.inbox.lock().await; 47 | inbox.close().await 48 | } 49 | 50 | pub fn sink(&self) -> Weak> { 51 | Arc::downgrade(&self.inbox) 52 | } 53 | } 54 | 55 | impl Connection 56 | where 57 | Stream: StreamExt, E>> + Send + Sync + Unpin + 'static, 58 | Sink: SinkExt, Error = E> + Send + Sync + Unpin + 'static, 59 | E: Into + Send + Sync, 60 | { 61 | /// Wraps incoming [WebSocket] connection and supplied [Awareness] accessor into a new 62 | /// connection handler capable of exchanging Yrs/Yjs messages. 63 | /// 64 | /// While creation of new [WarpConn] always succeeds, a connection itself can possibly fail 65 | /// while processing incoming input/output. This can be detected by awaiting for returned 66 | /// [WarpConn] and handling the awaited result. 67 | pub fn new(awareness: Arc>, sink: Sink, stream: Stream) -> Self { 68 | Self::with_protocol(awareness, sink, stream, DefaultProtocol) 69 | } 70 | 71 | /// Returns an underlying [Awareness] structure, that contains client state of that connection. 72 | pub fn awareness(&self) -> &Arc> { 73 | &self.awareness 74 | } 75 | 76 | /// Wraps incoming [WebSocket] connection and supplied [Awareness] accessor into a new 77 | /// connection handler capable of exchanging Yrs/Yjs messages. 78 | /// 79 | /// While creation of new [WarpConn] always succeeds, a connection itself can possibly fail 80 | /// while processing incoming input/output. This can be detected by awaiting for returned 81 | /// [WarpConn] and handling the awaited result. 82 | pub fn with_protocol

( 83 | awareness: Arc>, 84 | sink: Sink, 85 | mut stream: Stream, 86 | protocol: P, 87 | ) -> Self 88 | where 89 | P: Protocol + Send + Sync + 'static, 90 | { 91 | let sink = Arc::new(Mutex::new(sink)); 92 | let inbox = sink.clone(); 93 | let loop_sink = Arc::downgrade(&sink); 94 | let loop_awareness = Arc::downgrade(&awareness); 95 | let processing_loop: JoinHandle> = spawn(async move { 96 | // at the beginning send SyncStep1 and AwarenessUpdate 97 | let payload = { 98 | let awareness = loop_awareness.upgrade().unwrap(); 99 | let mut encoder = EncoderV1::new(); 100 | let awareness = awareness.read().await; 101 | protocol.start(&awareness, &mut encoder)?; 102 | encoder.to_vec() 103 | }; 104 | if !payload.is_empty() { 105 | if let Some(sink) = loop_sink.upgrade() { 106 | let mut s = sink.lock().await; 107 | if let Err(e) = s.send(payload).await { 108 | return Err(e.into()); 109 | } 110 | } else { 111 | return Ok(()); // parent ConnHandler has been dropped 112 | } 113 | } 114 | 115 | while let Some(input) = stream.next().await { 116 | match input { 117 | Ok(data) => { 118 | if let Some(mut sink) = loop_sink.upgrade() { 119 | if let Some(awareness) = loop_awareness.upgrade() { 120 | match Self::process(&protocol, &awareness, &mut sink, data).await { 121 | Ok(()) => { /* continue */ } 122 | Err(e) => { 123 | return Err(e); 124 | } 125 | } 126 | } else { 127 | return Ok(()); // parent ConnHandler has been dropped 128 | } 129 | } else { 130 | return Ok(()); // parent ConnHandler has been dropped 131 | } 132 | } 133 | Err(e) => return Err(e.into()), 134 | } 135 | } 136 | 137 | Ok(()) 138 | }); 139 | Connection { 140 | processing_loop, 141 | awareness, 142 | inbox, 143 | _stream: PhantomData::default(), 144 | } 145 | } 146 | 147 | async fn process( 148 | protocol: &P, 149 | awareness: &Arc>, 150 | sink: &mut Arc>, 151 | input: Vec, 152 | ) -> Result<(), Error> { 153 | let mut decoder = DecoderV1::new(Cursor::new(&input)); 154 | let reader = MessageReader::new(&mut decoder); 155 | for r in reader { 156 | let msg = r?; 157 | if let Some(reply) = handle_msg(protocol, &awareness, msg).await? { 158 | let mut sender = sink.lock().await; 159 | if let Err(e) = sender.send(reply.encode_v1()).await { 160 | println!("connection failed to send back the reply"); 161 | return Err(e.into()); 162 | } else { 163 | println!("connection send back the reply"); 164 | } 165 | } 166 | } 167 | Ok(()) 168 | } 169 | } 170 | 171 | impl Unpin for Connection {} 172 | 173 | impl Future for Connection { 174 | type Output = Result<(), Error>; 175 | 176 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 177 | match Pin::new(&mut self.processing_loop).poll(cx) { 178 | Poll::Pending => Poll::Pending, 179 | Poll::Ready(Err(e)) => Poll::Ready(Err(e.into())), 180 | Poll::Ready(Ok(r)) => Poll::Ready(r), 181 | } 182 | } 183 | } 184 | 185 | pub async fn handle_msg( 186 | protocol: &P, 187 | a: &Arc>, 188 | msg: Message, 189 | ) -> Result, Error> { 190 | match msg { 191 | Message::Sync(msg) => match msg { 192 | SyncMessage::SyncStep1(sv) => { 193 | let awareness = a.read().await; 194 | protocol.handle_sync_step1(&awareness, sv) 195 | } 196 | SyncMessage::SyncStep2(update) => { 197 | let mut awareness = a.write().await; 198 | protocol.handle_sync_step2(&mut awareness, Update::decode_v1(&update)?) 199 | } 200 | SyncMessage::Update(update) => { 201 | let mut awareness = a.write().await; 202 | protocol.handle_update(&mut awareness, Update::decode_v1(&update)?) 203 | } 204 | }, 205 | Message::Auth(reason) => { 206 | let awareness = a.read().await; 207 | protocol.handle_auth(&awareness, reason) 208 | } 209 | Message::AwarenessQuery => { 210 | let awareness = a.read().await; 211 | protocol.handle_awareness_query(&awareness) 212 | } 213 | Message::Awareness(update) => { 214 | let mut awareness = a.write().await; 215 | protocol.handle_awareness_update(&mut awareness, update) 216 | } 217 | Message::Custom(tag, data) => { 218 | let mut awareness = a.write().await; 219 | protocol.missing_handle(&mut awareness, tag, data) 220 | } 221 | } 222 | } 223 | 224 | #[cfg(test)] 225 | mod test { 226 | use crate::awareness::Awareness; 227 | use crate::net::conn::Connection; 228 | use crate::net::BroadcastGroup; 229 | use crate::sync::{Error, Message, SyncMessage}; 230 | use bytes::{Bytes, BytesMut}; 231 | use futures_util::SinkExt; 232 | use std::net::SocketAddr; 233 | use std::str::FromStr; 234 | use std::sync::Arc; 235 | use std::time::Duration; 236 | use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; 237 | use tokio::net::{TcpListener, TcpSocket}; 238 | use tokio::sync::{Mutex, Notify, RwLock}; 239 | use tokio::task; 240 | use tokio::task::JoinHandle; 241 | use tokio::time::{sleep, timeout}; 242 | use tokio_util::codec::{Decoder, Encoder, FramedRead, FramedWrite, LengthDelimitedCodec}; 243 | use yrs::updates::encoder::Encode; 244 | use yrs::{Doc, GetString, Text, Transact, UpdateSubscription}; 245 | 246 | #[derive(Debug, Default)] 247 | struct YrsCodec(LengthDelimitedCodec); 248 | 249 | impl Encoder> for YrsCodec { 250 | type Error = Error; 251 | 252 | fn encode(&mut self, item: Vec, dst: &mut BytesMut) -> Result<(), Self::Error> { 253 | self.0.encode(Bytes::from(item), dst)?; 254 | Ok(()) 255 | } 256 | } 257 | 258 | impl Decoder for YrsCodec { 259 | type Item = Vec; 260 | type Error = Error; 261 | 262 | fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 263 | if let Some(bytes) = self.0.decode(src)? { 264 | Ok(Some(bytes.freeze().to_vec())) 265 | } else { 266 | Ok(None) 267 | } 268 | } 269 | } 270 | 271 | type WrappedStream = FramedRead; 272 | type WrappedSink = FramedWrite; 273 | 274 | async fn start_server( 275 | addr: SocketAddr, 276 | bcast: BroadcastGroup, 277 | ) -> Result, Box> { 278 | let server = TcpListener::bind(addr).await?; 279 | Ok(tokio::spawn(async move { 280 | let mut subscribers = Vec::new(); 281 | while let Ok((stream, _)) = server.accept().await { 282 | let (reader, writer) = stream.into_split(); 283 | let stream = WrappedStream::new(reader, YrsCodec::default()); 284 | let sink = WrappedSink::new(writer, YrsCodec::default()); 285 | let sub = bcast.subscribe(Arc::new(Mutex::new(sink)), stream); 286 | subscribers.push(sub); 287 | } 288 | })) 289 | } 290 | 291 | async fn client( 292 | addr: SocketAddr, 293 | doc: Doc, 294 | ) -> Result, Box> { 295 | let stream = TcpSocket::new_v4()?.connect(addr).await?; 296 | let (reader, writer) = stream.into_split(); 297 | let stream: WrappedStream = WrappedStream::new(reader, YrsCodec::default()); 298 | let sink: WrappedSink = WrappedSink::new(writer, YrsCodec::default()); 299 | Ok(Connection::new( 300 | Arc::new(RwLock::new(Awareness::new(doc))), 301 | sink, 302 | stream, 303 | )) 304 | } 305 | 306 | fn create_notifier(doc: &Doc) -> (Arc, UpdateSubscription) { 307 | let n = Arc::new(Notify::new()); 308 | let sub = { 309 | let n = n.clone(); 310 | doc.observe_update_v1(move |_, _| n.notify_waiters()) 311 | .unwrap() 312 | }; 313 | (n, sub) 314 | } 315 | 316 | const TIMEOUT: Duration = Duration::from_secs(5); 317 | 318 | #[tokio::test] 319 | async fn change_introduced_by_server_reaches_subscribed_clients( 320 | ) -> Result<(), Box> { 321 | let server_addr = SocketAddr::from_str("127.0.0.1:6600").unwrap(); 322 | let doc = Doc::with_client_id(1); 323 | let text = doc.get_or_insert_text("test"); 324 | let awareness = Arc::new(RwLock::new(Awareness::new(doc))); 325 | let bcast = BroadcastGroup::new(awareness.clone(), 10).await; 326 | let server = start_server(server_addr.clone(), bcast).await?; 327 | 328 | let doc = Doc::new(); 329 | let (n, sub) = create_notifier(&doc); 330 | let c1 = client(server_addr.clone(), doc).await?; 331 | 332 | { 333 | let lock = awareness.write().await; 334 | text.push(&mut lock.doc().transact_mut(), "abc"); 335 | } 336 | 337 | timeout(TIMEOUT, n.notified()).await?; 338 | 339 | { 340 | let awareness = c1.awareness().read().await; 341 | let doc = awareness.doc(); 342 | let text = doc.get_or_insert_text("test"); 343 | let str = text.get_string(&doc.transact()); 344 | assert_eq!(str, "abc".to_string()); 345 | } 346 | 347 | Ok(()) 348 | } 349 | 350 | #[tokio::test] 351 | async fn subscribed_client_fetches_initial_state() -> Result<(), Box> { 352 | let server_addr = SocketAddr::from_str("127.0.0.1:6601").unwrap(); 353 | let doc = Doc::with_client_id(1); 354 | let text = doc.get_or_insert_text("test"); 355 | 356 | text.push(&mut doc.transact_mut(), "abc"); 357 | 358 | let awareness = Arc::new(RwLock::new(Awareness::new(doc))); 359 | let bcast = BroadcastGroup::new(awareness.clone(), 10).await; 360 | let server = start_server(server_addr.clone(), bcast).await?; 361 | 362 | let doc = Doc::new(); 363 | let (n, sub) = create_notifier(&doc); 364 | let c1 = client(server_addr.clone(), doc).await?; 365 | 366 | timeout(TIMEOUT, n.notified()).await?; 367 | 368 | { 369 | let awareness = c1.awareness().read().await; 370 | let doc = awareness.doc(); 371 | let text = doc.get_or_insert_text("test"); 372 | let str = text.get_string(&doc.transact()); 373 | assert_eq!(str, "abc".to_string()); 374 | } 375 | 376 | Ok(()) 377 | } 378 | 379 | #[tokio::test] 380 | async fn changes_from_one_client_reach_others() -> Result<(), Box> { 381 | let server_addr = SocketAddr::from_str("127.0.0.1:6602").unwrap(); 382 | let doc = Doc::with_client_id(1); 383 | let text = doc.get_or_insert_text("test"); 384 | 385 | let awareness = Arc::new(RwLock::new(Awareness::new(doc))); 386 | let bcast = BroadcastGroup::new(awareness.clone(), 10).await; 387 | let server = start_server(server_addr.clone(), bcast).await?; 388 | 389 | let d1 = Doc::with_client_id(2); 390 | let c1 = client(server_addr.clone(), d1).await?; 391 | // by default changes made by document on the client side are not propagated automatically 392 | let sub11 = { 393 | let sink = c1.sink(); 394 | let a = c1.awareness().write().await; 395 | let doc = a.doc(); 396 | doc.observe_update_v1(move |txn, e| { 397 | let update = e.update.to_owned(); 398 | if let Some(sink) = sink.upgrade() { 399 | task::spawn(async move { 400 | let msg = Message::Sync(SyncMessage::Update(update)).encode_v1(); 401 | let mut sink = sink.lock().await; 402 | sink.send(msg).await.unwrap(); 403 | }); 404 | } 405 | }) 406 | .unwrap() 407 | }; 408 | 409 | let d2 = Doc::with_client_id(3); 410 | let (n2, sub2) = create_notifier(&d2); 411 | let c2 = client(server_addr.clone(), d2).await?; 412 | 413 | { 414 | let a = c1.awareness().write().await; 415 | let doc = a.doc(); 416 | let text = doc.get_or_insert_text("test"); 417 | text.push(&mut doc.transact_mut(), "def"); 418 | } 419 | 420 | timeout(TIMEOUT, n2.notified()).await?; 421 | 422 | { 423 | let awareness = c2.awareness.read().await; 424 | let doc = awareness.doc(); 425 | let text = doc.get_or_insert_text("test"); 426 | let str = text.get_string(&doc.transact()); 427 | assert_eq!(str, "def".to_string()); 428 | } 429 | 430 | Ok(()) 431 | } 432 | 433 | #[tokio::test] 434 | async fn client_failure_doesnt_affect_others() -> Result<(), Box> { 435 | let server_addr = SocketAddr::from_str("127.0.0.1:6603").unwrap(); 436 | let doc = Doc::with_client_id(1); 437 | let text = doc.get_or_insert_text("test"); 438 | 439 | let awareness = Arc::new(RwLock::new(Awareness::new(doc))); 440 | let bcast = BroadcastGroup::new(awareness.clone(), 10).await; 441 | let server = start_server(server_addr.clone(), bcast).await?; 442 | 443 | let d1 = Doc::with_client_id(2); 444 | let c1 = client(server_addr.clone(), d1).await?; 445 | // by default changes made by document on the client side are not propagated automatically 446 | let sub11 = { 447 | let sink = c1.sink(); 448 | let a = c1.awareness().write().await; 449 | let doc = a.doc(); 450 | doc.observe_update_v1(move |txn, e| { 451 | let update = e.update.to_owned(); 452 | if let Some(sink) = sink.upgrade() { 453 | task::spawn(async move { 454 | let msg = Message::Sync(SyncMessage::Update(update)).encode_v1(); 455 | let mut sink = sink.lock().await; 456 | sink.send(msg).await.unwrap(); 457 | }); 458 | } 459 | }) 460 | .unwrap() 461 | }; 462 | 463 | let d2 = Doc::with_client_id(3); 464 | let (n2, sub2) = create_notifier(&d2); 465 | let c2 = client(server_addr.clone(), d2).await?; 466 | 467 | let d3 = Doc::with_client_id(4); 468 | let (n3, sub3) = create_notifier(&d3); 469 | let c3 = client(server_addr.clone(), d3).await?; 470 | 471 | { 472 | let a = c1.awareness().write().await; 473 | let doc = a.doc(); 474 | let text = doc.get_or_insert_text("test"); 475 | text.push(&mut doc.transact_mut(), "abc"); 476 | } 477 | 478 | // on the first try both C2 and C3 should receive the update 479 | //timeout(TIMEOUT, n2.notified()).await.unwrap(); 480 | //timeout(TIMEOUT, n3.notified()).await.unwrap(); 481 | sleep(TIMEOUT).await; 482 | 483 | { 484 | let awareness = c2.awareness.read().await; 485 | let doc = awareness.doc(); 486 | let text = doc.get_or_insert_text("test"); 487 | let str = text.get_string(&doc.transact()); 488 | assert_eq!(str, "abc".to_string()); 489 | } 490 | { 491 | let awareness = c3.awareness.read().await; 492 | let doc = awareness.doc(); 493 | let text = doc.get_or_insert_text("test"); 494 | let str = text.get_string(&doc.transact()); 495 | assert_eq!(str, "abc".to_string()); 496 | } 497 | 498 | // drop client, causing abrupt ending 499 | drop(c3); 500 | drop(n3); 501 | drop(sub3); 502 | // C2 notification subscription has been realized, we need to refresh it 503 | drop(n2); 504 | drop(sub2); 505 | 506 | let (n2, sub2) = { 507 | let a = c2.awareness().write().await; 508 | let doc = a.doc(); 509 | create_notifier(doc) 510 | }; 511 | 512 | { 513 | let a = c1.awareness().write().await; 514 | let doc = a.doc(); 515 | let text = doc.get_or_insert_text("test"); 516 | text.push(&mut doc.transact_mut(), "def"); 517 | } 518 | 519 | timeout(TIMEOUT, n2.notified()).await.unwrap(); 520 | 521 | { 522 | let awareness = c2.awareness.read().await; 523 | let doc = awareness.doc(); 524 | let text = doc.get_or_insert_text("test"); 525 | let str = text.get_string(&doc.transact()); 526 | assert_eq!(str, "abcdef".to_string()); 527 | } 528 | 529 | Ok(()) 530 | } 531 | } 532 | -------------------------------------------------------------------------------- /src/net/mod.rs: -------------------------------------------------------------------------------- 1 | mod broadcast; 2 | mod conn; 3 | 4 | pub type Connection = conn::Connection; 5 | pub type BroadcastGroup = broadcast::BroadcastGroup; 6 | pub type Subscription = broadcast::Subscription; 7 | -------------------------------------------------------------------------------- /src/sync.rs: -------------------------------------------------------------------------------- 1 | use crate::awareness; 2 | use crate::awareness::{Awareness, AwarenessUpdate}; 3 | use thiserror::Error; 4 | use yrs::encoding::read; 5 | use yrs::updates::decoder::{Decode, Decoder}; 6 | use yrs::updates::encoder::{Encode, Encoder}; 7 | use yrs::{ReadTxn, StateVector, Transact, Update}; 8 | 9 | /* 10 | Core Yjs defines two message types: 11 | • YjsSyncStep1: Includes the State Set of the sending client. When received, the client should reply with YjsSyncStep2. 12 | • YjsSyncStep2: Includes all missing structs and the complete delete set. When received, the client is assured that it 13 | received all information from the remote client. 14 | 15 | In a peer-to-peer network, you may want to introduce a SyncDone message type. Both parties should initiate the connection 16 | with SyncStep1. When a client received SyncStep2, it should reply with SyncDone. When the local client received both 17 | SyncStep2 and SyncDone, it is assured that it is synced to the remote client. 18 | 19 | In a client-server model, you want to handle this differently: The client should initiate the connection with SyncStep1. 20 | When the server receives SyncStep1, it should reply with SyncStep2 immediately followed by SyncStep1. The client replies 21 | with SyncStep2 when it receives SyncStep1. Optionally the server may send a SyncDone after it received SyncStep2, so the 22 | client knows that the sync is finished. There are two reasons for this more elaborated sync model: 1. This protocol can 23 | easily be implemented on top of http and websockets. 2. The server shoul only reply to requests, and not initiate them. 24 | Therefore it is necesarry that the client initiates the sync. 25 | 26 | Construction of a message: 27 | [messageType : varUint, message definition..] 28 | 29 | Note: A message does not include information about the room name. This must to be handled by the upper layer protocol! 30 | 31 | stringify[messageType] stringifies a message definition (messageType is already read from the bufffer) 32 | */ 33 | 34 | /// A default implementation of y-sync [Protocol]. 35 | pub struct DefaultProtocol; 36 | 37 | impl Protocol for DefaultProtocol {} 38 | 39 | /// Trait implementing a y-sync protocol. The default implementation can be found in 40 | /// [DefaultProtocol], but its implementation steps can be potentially changed by the user if 41 | /// necessary. 42 | pub trait Protocol { 43 | /// To be called whenever a new connection has been accepted. Returns an encoded list of 44 | /// messages to be send back to initiator. This binary may contain multiple messages inside, 45 | /// stored one after another. 46 | fn start(&self, awareness: &Awareness, encoder: &mut E) -> Result<(), Error> { 47 | let (sv, update) = { 48 | let sv = awareness.doc().transact().state_vector(); 49 | let update = awareness.update()?; 50 | (sv, update) 51 | }; 52 | Message::Sync(SyncMessage::SyncStep1(sv)).encode(encoder); 53 | Message::Awareness(update).encode(encoder); 54 | Ok(()) 55 | } 56 | 57 | /// Y-sync protocol sync-step-1 - given a [StateVector] of a remote side, calculate missing 58 | /// updates. Returns a sync-step-2 message containing a calculated update. 59 | fn handle_sync_step1( 60 | &self, 61 | awareness: &Awareness, 62 | sv: StateVector, 63 | ) -> Result, Error> { 64 | let update = awareness.doc().transact().encode_state_as_update_v1(&sv); 65 | Ok(Some(Message::Sync(SyncMessage::SyncStep2(update)))) 66 | } 67 | 68 | /// Handle reply for a sync-step-1 send from this replica previously. By default just apply 69 | /// an update to current `awareness` document instance. 70 | fn handle_sync_step2( 71 | &self, 72 | awareness: &mut Awareness, 73 | update: Update, 74 | ) -> Result, Error> { 75 | let mut txn = awareness.doc().transact_mut(); 76 | txn.apply_update(update); 77 | Ok(None) 78 | } 79 | 80 | /// Handle continuous update send from the client. By default just apply an update to a current 81 | /// `awareness` document instance. 82 | fn handle_update( 83 | &self, 84 | awareness: &mut Awareness, 85 | update: Update, 86 | ) -> Result, Error> { 87 | self.handle_sync_step2(awareness, update) 88 | } 89 | 90 | /// Handle authorization message. By default if reason for auth denial has been provided, 91 | /// send back [Error::PermissionDenied]. 92 | fn handle_auth( 93 | &self, 94 | _awareness: &Awareness, 95 | deny_reason: Option, 96 | ) -> Result, Error> { 97 | if let Some(reason) = deny_reason { 98 | Err(Error::PermissionDenied { reason }) 99 | } else { 100 | Ok(None) 101 | } 102 | } 103 | 104 | /// Returns an [AwarenessUpdate] which is a serializable representation of a current `awareness` 105 | /// instance. 106 | fn handle_awareness_query(&self, awareness: &Awareness) -> Result, Error> { 107 | let update = awareness.update()?; 108 | Ok(Some(Message::Awareness(update))) 109 | } 110 | 111 | /// Reply to awareness query or just incoming [AwarenessUpdate], where current `awareness` 112 | /// instance is being updated with incoming data. 113 | fn handle_awareness_update( 114 | &self, 115 | awareness: &mut Awareness, 116 | update: AwarenessUpdate, 117 | ) -> Result, Error> { 118 | awareness.apply_update(update)?; 119 | Ok(None) 120 | } 121 | 122 | /// Y-sync protocol enables to extend its own settings with custom handles. These can be 123 | /// implemented here. By default it returns an [Error::Unsupported]. 124 | fn missing_handle( 125 | &self, 126 | _awareness: &mut Awareness, 127 | tag: u8, 128 | _data: Vec, 129 | ) -> Result, Error> { 130 | Err(Error::Unsupported(tag)) 131 | } 132 | } 133 | 134 | /// Tag id for [Message::Sync]. 135 | pub const MSG_SYNC: u8 = 0; 136 | /// Tag id for [Message::Awareness]. 137 | pub const MSG_AWARENESS: u8 = 1; 138 | /// Tag id for [Message::Auth]. 139 | pub const MSG_AUTH: u8 = 2; 140 | /// Tag id for [Message::AwarenessQuery]. 141 | pub const MSG_QUERY_AWARENESS: u8 = 3; 142 | 143 | pub const PERMISSION_DENIED: u8 = 0; 144 | pub const PERMISSION_GRANTED: u8 = 1; 145 | 146 | #[derive(Debug, Eq, PartialEq)] 147 | pub enum Message { 148 | Sync(SyncMessage), 149 | Auth(Option), 150 | AwarenessQuery, 151 | Awareness(AwarenessUpdate), 152 | Custom(u8, Vec), 153 | } 154 | 155 | impl Encode for Message { 156 | fn encode(&self, encoder: &mut E) { 157 | match self { 158 | Message::Sync(msg) => { 159 | encoder.write_var(MSG_SYNC); 160 | msg.encode(encoder); 161 | } 162 | Message::Auth(reason) => { 163 | encoder.write_var(MSG_AUTH); 164 | if let Some(reason) = reason { 165 | encoder.write_var(PERMISSION_DENIED); 166 | encoder.write_string(&reason); 167 | } else { 168 | encoder.write_var(PERMISSION_GRANTED); 169 | } 170 | } 171 | Message::AwarenessQuery => { 172 | encoder.write_var(MSG_QUERY_AWARENESS); 173 | } 174 | Message::Awareness(update) => { 175 | encoder.write_var(MSG_AWARENESS); 176 | encoder.write_buf(&update.encode_v1()) 177 | } 178 | Message::Custom(tag, data) => { 179 | encoder.write_u8(*tag); 180 | encoder.write_buf(&data); 181 | } 182 | } 183 | } 184 | } 185 | 186 | impl Decode for Message { 187 | fn decode(decoder: &mut D) -> Result { 188 | let tag: u8 = decoder.read_var()?; 189 | match tag { 190 | MSG_SYNC => { 191 | let msg = SyncMessage::decode(decoder)?; 192 | Ok(Message::Sync(msg)) 193 | } 194 | MSG_AWARENESS => { 195 | let data = decoder.read_buf()?; 196 | let update = AwarenessUpdate::decode_v1(data)?; 197 | Ok(Message::Awareness(update)) 198 | } 199 | MSG_AUTH => { 200 | let reason = if decoder.read_var::()? == PERMISSION_DENIED { 201 | Some(decoder.read_string()?.to_string()) 202 | } else { 203 | None 204 | }; 205 | Ok(Message::Auth(reason)) 206 | } 207 | MSG_QUERY_AWARENESS => Ok(Message::AwarenessQuery), 208 | tag => { 209 | let data = decoder.read_buf()?; 210 | Ok(Message::Custom(tag, data.to_vec())) 211 | } 212 | } 213 | } 214 | } 215 | 216 | /// Tag id for [SyncMessage::SyncStep1]. 217 | pub const MSG_SYNC_STEP_1: u8 = 0; 218 | /// Tag id for [SyncMessage::SyncStep2]. 219 | pub const MSG_SYNC_STEP_2: u8 = 1; 220 | /// Tag id for [SyncMessage::Update]. 221 | pub const MSG_SYNC_UPDATE: u8 = 2; 222 | 223 | #[derive(Debug, PartialEq, Eq)] 224 | pub enum SyncMessage { 225 | SyncStep1(StateVector), 226 | SyncStep2(Vec), 227 | Update(Vec), 228 | } 229 | 230 | impl Encode for SyncMessage { 231 | fn encode(&self, encoder: &mut E) { 232 | match self { 233 | SyncMessage::SyncStep1(sv) => { 234 | encoder.write_var(MSG_SYNC_STEP_1); 235 | encoder.write_buf(sv.encode_v1()); 236 | } 237 | SyncMessage::SyncStep2(u) => { 238 | encoder.write_var(MSG_SYNC_STEP_2); 239 | encoder.write_buf(u); 240 | } 241 | SyncMessage::Update(u) => { 242 | encoder.write_var(MSG_SYNC_UPDATE); 243 | encoder.write_buf(u); 244 | } 245 | } 246 | } 247 | } 248 | 249 | impl Decode for SyncMessage { 250 | fn decode(decoder: &mut D) -> Result { 251 | let tag: u8 = decoder.read_var()?; 252 | match tag { 253 | MSG_SYNC_STEP_1 => { 254 | let buf = decoder.read_buf()?; 255 | let sv = StateVector::decode_v1(buf)?; 256 | Ok(SyncMessage::SyncStep1(sv)) 257 | } 258 | MSG_SYNC_STEP_2 => { 259 | let buf = decoder.read_buf()?; 260 | Ok(SyncMessage::SyncStep2(buf.into())) 261 | } 262 | MSG_SYNC_UPDATE => { 263 | let buf = decoder.read_buf()?; 264 | Ok(SyncMessage::Update(buf.into())) 265 | } 266 | _ => Err(read::Error::UnexpectedValue), 267 | } 268 | } 269 | } 270 | 271 | /// An error type returned in response from y-sync [Protocol]. 272 | #[derive(Debug, Error)] 273 | pub enum Error { 274 | /// Incoming Y-protocol message couldn't be deserialized. 275 | #[error("failed to deserialize message: {0}")] 276 | DecodingError(#[from] read::Error), 277 | 278 | /// Applying incoming Y-protocol awareness update has failed. 279 | #[error("failed to process awareness update: {0}")] 280 | AwarenessEncoding(#[from] awareness::Error), 281 | 282 | /// An incoming Y-protocol authorization request has been denied. 283 | #[error("permission denied to access: {reason}")] 284 | PermissionDenied { reason: String }, 285 | 286 | /// Thrown whenever an unknown message tag has been sent. 287 | #[error("unsupported message tag identifier: {0}")] 288 | Unsupported(u8), 289 | 290 | /// Thrown in case of I/O errors. 291 | #[error("IO error: {0}")] 292 | IO(#[from] std::io::Error), 293 | 294 | /// Custom dynamic kind of error, usually related to a warp internal error messages. 295 | #[error("internal failure: {0}")] 296 | Other(#[from] Box), 297 | } 298 | 299 | #[cfg(feature = "net")] 300 | impl From for Error { 301 | fn from(value: tokio::task::JoinError) -> Self { 302 | Error::Other(value.into()) 303 | } 304 | } 305 | 306 | /// Since y-sync protocol enables for a multiple messages to be packed into a singe byte payload, 307 | /// [MessageReader] can be used over the decoder to read these messages one by one in iterable 308 | /// fashion. 309 | pub struct MessageReader<'a, D: Decoder>(&'a mut D); 310 | 311 | impl<'a, D: Decoder> MessageReader<'a, D> { 312 | pub fn new(decoder: &'a mut D) -> Self { 313 | MessageReader(decoder) 314 | } 315 | } 316 | 317 | impl<'a, D: Decoder> Iterator for MessageReader<'a, D> { 318 | type Item = Result; 319 | 320 | fn next(&mut self) -> Option { 321 | match Message::decode(self.0) { 322 | Ok(msg) => Some(Ok(msg)), 323 | Err(read::Error::EndOfBuffer(_)) => None, 324 | Err(error) => Some(Err(error)), 325 | } 326 | } 327 | } 328 | 329 | #[cfg(test)] 330 | mod test { 331 | use crate::awareness::Awareness; 332 | use crate::sync::*; 333 | use std::collections::HashMap; 334 | use yrs::encoding::read::Cursor; 335 | use yrs::updates::decoder::{Decode, DecoderV1}; 336 | use yrs::updates::encoder::{Encode, EncoderV1}; 337 | use yrs::{Doc, GetString, ReadTxn, StateVector, Text, Transact}; 338 | 339 | #[test] 340 | fn message_encoding() { 341 | let doc = Doc::new(); 342 | let txt = doc.get_or_insert_text("text"); 343 | txt.push(&mut doc.transact_mut(), "hello world"); 344 | let mut awareness = Awareness::new(doc); 345 | awareness.set_local_state("{\"user\":{\"name\":\"Anonymous 50\",\"color\":\"#30bced\",\"colorLight\":\"#30bced33\"}}"); 346 | 347 | let messages = [ 348 | Message::Sync(SyncMessage::SyncStep1( 349 | awareness.doc().transact().state_vector(), 350 | )), 351 | Message::Sync(SyncMessage::SyncStep2( 352 | awareness 353 | .doc() 354 | .transact() 355 | .encode_state_as_update_v1(&StateVector::default()), 356 | )), 357 | Message::Awareness(awareness.update().unwrap()), 358 | Message::Auth(Some("reason".to_string())), 359 | Message::AwarenessQuery, 360 | ]; 361 | 362 | for msg in messages { 363 | let encoded = msg.encode_v1(); 364 | let decoded = 365 | Message::decode_v1(&encoded).expect(&format!("failed to decode {:?}", msg)); 366 | assert_eq!(decoded, msg); 367 | } 368 | } 369 | 370 | #[test] 371 | fn protocol_init() { 372 | let awareness = Awareness::default(); 373 | let protocol = DefaultProtocol; 374 | let mut encoder = EncoderV1::new(); 375 | protocol.start(&awareness, &mut encoder).unwrap(); 376 | let data = encoder.to_vec(); 377 | let mut decoder = DecoderV1::new(Cursor::new(&data)); 378 | let mut reader = MessageReader::new(&mut decoder); 379 | 380 | assert_eq!( 381 | reader.next().unwrap().unwrap(), 382 | Message::Sync(SyncMessage::SyncStep1(StateVector::default())) 383 | ); 384 | 385 | assert_eq!( 386 | reader.next().unwrap().unwrap(), 387 | Message::Awareness(awareness.update().unwrap()) 388 | ); 389 | 390 | assert!(reader.next().is_none()); 391 | } 392 | 393 | #[test] 394 | fn protocol_sync_steps() { 395 | let protocol = DefaultProtocol; 396 | 397 | let mut a1 = Awareness::new(Doc::with_client_id(1)); 398 | let mut a2 = Awareness::new(Doc::with_client_id(2)); 399 | 400 | let expected = { 401 | let txt = a1.doc_mut().get_or_insert_text("test"); 402 | let mut txn = a1.doc_mut().transact_mut(); 403 | txt.push(&mut txn, "hello"); 404 | txn.encode_state_as_update_v1(&StateVector::default()) 405 | }; 406 | 407 | let result = protocol 408 | .handle_sync_step1(&a1, a2.doc().transact().state_vector()) 409 | .unwrap(); 410 | 411 | assert_eq!( 412 | result, 413 | Some(Message::Sync(SyncMessage::SyncStep2(expected))) 414 | ); 415 | 416 | if let Some(Message::Sync(SyncMessage::SyncStep2(u))) = result { 417 | let result2 = protocol 418 | .handle_sync_step2(&mut a2, Update::decode_v1(&u).unwrap()) 419 | .unwrap(); 420 | 421 | assert!(result2.is_none()); 422 | } 423 | 424 | let txt = a2.doc().transact().get_text("test").unwrap(); 425 | assert_eq!(txt.get_string(&a2.doc().transact()), "hello".to_owned()); 426 | } 427 | 428 | #[test] 429 | fn protocol_sync_step_update() { 430 | let protocol = DefaultProtocol; 431 | 432 | let mut a1 = Awareness::new(Doc::with_client_id(1)); 433 | let mut a2 = Awareness::new(Doc::with_client_id(2)); 434 | 435 | let data = { 436 | let txt = a1.doc_mut().get_or_insert_text("test"); 437 | let mut txn = a1.doc_mut().transact_mut(); 438 | txt.push(&mut txn, "hello"); 439 | txn.encode_update_v1() 440 | }; 441 | 442 | let result = protocol 443 | .handle_update(&mut a2, Update::decode_v1(&data).unwrap()) 444 | .unwrap(); 445 | 446 | assert!(result.is_none()); 447 | 448 | let txt = a2.doc().transact().get_text("test").unwrap(); 449 | assert_eq!(txt.get_string(&a2.doc().transact()), "hello".to_owned()); 450 | } 451 | 452 | #[test] 453 | fn protocol_awareness_sync() { 454 | let protocol = DefaultProtocol; 455 | 456 | let mut a1 = Awareness::new(Doc::with_client_id(1)); 457 | let mut a2 = Awareness::new(Doc::with_client_id(2)); 458 | 459 | a1.set_local_state("{x:3}"); 460 | let result = protocol.handle_awareness_query(&a1).unwrap(); 461 | 462 | assert_eq!(result, Some(Message::Awareness(a1.update().unwrap()))); 463 | 464 | if let Some(Message::Awareness(u)) = result { 465 | let result = protocol.handle_awareness_update(&mut a2, u).unwrap(); 466 | assert!(result.is_none()); 467 | } 468 | 469 | assert_eq!(a2.clients(), &HashMap::from([(1, "{x:3}".to_owned())])); 470 | } 471 | } 472 | --------------------------------------------------------------------------------