From 765cd983fc2cb6e975adf1791e25dac066d2999e Mon Sep 17 00:00:00 2001 From: linonetwo Date: Fri, 3 Feb 2023 10:55:56 +0800 Subject: [PATCH] feat: try use a yrs doc to merge state before store to sqlite, but not working --- client-app/src-tauri/Cargo.lock | 253 ++++++++++++++++++ client-app/src-tauri/Cargo.toml | 2 + client-app/src-tauri/src/commands/document.rs | 73 +++-- client-app/src-tauri/src/state.rs | 10 +- 4 files changed, 317 insertions(+), 21 deletions(-) diff --git a/client-app/src-tauri/Cargo.lock b/client-app/src-tauri/Cargo.lock index cc722cb571..890b67a243 100644 --- a/client-app/src-tauri/Cargo.lock +++ b/client-app/src-tauri/Cargo.lock @@ -15,6 +15,7 @@ dependencies = [ "jwst", "jwst-storage", "lib0", + "moka", "project-root", "schemars", "serde", @@ -22,6 +23,7 @@ dependencies = [ "tauri", "tauri-build", "tokio", + "y-sync", "yrs", ] @@ -117,6 +119,36 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +[[package]] +name = "async-io" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c374dda1ed3e7d8f0d9ba58715f924862c63eae6849c92d3a18e7fbde9e2794" +dependencies = [ + "async-lock", + "autocfg", + "concurrent-queue", + "futures-lite", + "libc", + "log", + "parking", + "polling", + "slab", + "socket2", + "waker-fn", + "windows-sys 0.42.0", +] + +[[package]] +name = "async-lock" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" +dependencies = [ + "event-listener", + "futures-lite", +] + [[package]] name = "async-stream" version = "0.3.3" @@ -372,6 +404,12 @@ dependencies = [ "syn", ] +[[package]] +name = "bytecount" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" + [[package]] name = "bytemuck" version = "1.12.3" @@ -414,6 +452,15 @@ dependencies = [ "system-deps 6.0.3", ] +[[package]] +name = "camino" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77df041dc383319cc661b428b6961a005db4d6808d5e12536931b1ca9556055" +dependencies = [ + "serde", +] + [[package]] name = "cang-jie" version = "0.14.0" @@ -423,6 +470,28 @@ dependencies = [ "tantivy", ] +[[package]] +name = "cargo-platform" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.14", + "serde", + "serde_json", +] + [[package]] name = "cargo_toml" version = "0.13.0" @@ -634,6 +703,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "concurrent-queue" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -1026,6 +1104,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -1225,6 +1312,21 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.25" @@ -2095,6 +2197,15 @@ dependencies = [ "time 0.3.17", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -2203,6 +2314,32 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "moka" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b49a05f67020456541f4f29cbaa812016a266a86ec76f96d3873d459c68fe5e" +dependencies = [ + "async-io", + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "futures-util", + "num_cpus", + "once_cell", + "parking_lot 0.12.1", + "quanta", + "rustc_version 0.4.0", + "scheduled-thread-pool", + "skeptic", + "smallvec", + "tagptr", + "thiserror", + "triomphe", + "uuid 1.2.2", +] + [[package]] name = "murmurhash32" version = "0.2.0" @@ -2572,6 +2709,12 @@ dependencies = [ "system-deps 6.0.3", ] +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.11.2" @@ -2857,6 +3000,20 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "polling" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22122d5ec4f9fe1b3916419b76be1e80bcb93f618d071d2edf841b137b2a2bd6" +dependencies = [ + "autocfg", + "cfg-if", + "libc", + "log", + "wepoll-ffi", + "windows-sys 0.42.0", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2954,6 +3111,33 @@ dependencies = [ "syn", ] +[[package]] +name = "pulldown-cmark" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d9cc634bc78768157b5cbfe988ffcd1dcba95cd2b2f03a88316c08c6d00ed63" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "quanta" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e31331286705f455e56cca62e0e717158474ff02b7936c1fa596d983f4ae27" +dependencies = [ + "crossbeam-utils", + "libc", + "mach", + "once_cell", + "raw-cpuid", + "wasi 0.10.0+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-xml" version = "0.23.1" @@ -3053,6 +3237,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "raw-cpuid" +version = "10.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6823ea29436221176fe662da99998ad3b4db2c7f31e7b6f5fe43adccd6320bb" +dependencies = [ + "bitflags", +] + [[package]] name = "raw-window-handle" version = "0.5.0" @@ -3322,6 +3515,15 @@ dependencies = [ "windows-sys 0.36.1", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf" +dependencies = [ + "parking_lot 0.12.1", +] + [[package]] name = "schemars" version = "0.8.11" @@ -3765,6 +3967,21 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +[[package]] +name = "skeptic" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8" +dependencies = [ + "bytecount", + "cargo_metadata", + "error-chain", + "glob", + "pulldown-cmark", + "tempfile", + "walkdir", +] + [[package]] name = "slab" version = "0.4.7" @@ -4062,6 +4279,12 @@ dependencies = [ "version-compare 0.1.1", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tantivy" version = "0.19.0" @@ -4698,6 +4921,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "triomphe" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1ee9bd9239c339d714d657fac840c6d2a4f9c45f4f9ec7b0975113458be78db" + [[package]] name = "type-map" version = "0.5.0" @@ -4719,6 +4948,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.8" @@ -4859,6 +5097,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "walkdir" version = "2.3.2" @@ -5068,6 +5312,15 @@ dependencies = [ "windows-metadata", ] +[[package]] +name = "wepoll-ffi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +dependencies = [ + "cc", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/client-app/src-tauri/Cargo.toml b/client-app/src-tauri/Cargo.toml index bbd0f3d276..4bd6c9eb5c 100644 --- a/client-app/src-tauri/Cargo.toml +++ b/client-app/src-tauri/Cargo.toml @@ -33,7 +33,9 @@ dotenvy = "0.15.6" tauri = { version = "1.2", features = ["api-all", "devtools"] } tokio = { version = "1.23.0", features = ["rt", "macros"] } lib0 = "0.12.0" +moka = { version = "0.9.6", features = ["future"] } yrs = { path = "../src-OctoBase/libs/vendors/y-crdt/yrs" } +y-sync = { path = "../src-OctoBase/libs/vendors/y-sync" } [features] # by default Tauri runs in production mode diff --git a/client-app/src-tauri/src/commands/document.rs b/client-app/src-tauri/src/commands/document.rs index 60caf07d06..b92130918e 100644 --- a/client-app/src-tauri/src/commands/document.rs +++ b/client-app/src-tauri/src/commands/document.rs @@ -1,13 +1,15 @@ -use std::io::ErrorKind; - use ipc_types::document::{ CreateDocumentParameter, GetDocumentParameter, GetDocumentResponse, YDocumentUpdate, }; +use jwst::encode_update; use jwst::DocStorage; use jwst::Workspace as OctoBaseWorkspace; use lib0::any::Any; -use tokio::fs::File; -use tokio::io::AsyncReadExt; +use std::sync::{Arc, RwLock}; +use y_sync::sync::Message; +use y_sync::sync::MessageReader; +use y_sync::sync::SyncMessage; +use yrs::updates::decoder::DecoderV1; use yrs::{updates::decoder::Decode, Doc, StateVector, Update}; use crate::state::AppState; @@ -30,7 +32,7 @@ pub async fn create_doc<'s>( .0 .lock() .await - .doc_storage + .doc_db .write_doc(parameters.workspace_id.clone(), workspace_doc.doc()) .await { @@ -51,9 +53,12 @@ pub async fn get_doc<'s>( ) -> Result { // TODO: check user permission - if let Ok(all_updates_of_workspace) = &state.0.lock().await.doc_storage.all(¶meters.id).await { + if let Ok(all_updates_of_workspace) = &state.0.lock().await.doc_db.all(¶meters.id).await { Ok(GetDocumentResponse { - updates: all_updates_of_workspace.iter().map(|model| model.blob.clone()).collect::>>(), + updates: all_updates_of_workspace + .iter() + .map(|model| model.blob.clone()) + .collect::>>(), }) } else { Err(format!( @@ -68,17 +73,47 @@ pub async fn update_y_document<'s>( state: tauri::State<'s, AppState>, parameters: YDocumentUpdate, ) -> Result { - if let Some(success) = &state - .0 - .lock() - .await - .doc_storage - .write_update(parameters.id.clone(), ¶meters.update) - .await - .ok() - { - Ok(*success) - } else { - Err(format!("Failed to update yDoc to {}", parameters.id)) + let state = &state.0.lock().await; + let doc_store = &state.doc_store; + let doc_db = &state.doc_db; + let update = Update::decode_v1(¶meters.update).unwrap(); + let mut decoder = DecoderV1::from(¶meters.update[..]); + for msg in MessageReader::new(&mut decoder) { + msg.ok().and_then(|msg| match msg { + Message::Sync(msg) => match msg { + SyncMessage::SyncStep1(sv) => { + Some(()) + } + SyncMessage::SyncStep2(update) => { + Some(()) + } + SyncMessage::Update(update) => { + let mut tx = doc_store.doc().transact(); + tx.apply_update(Update::decode_v1(&update).unwrap()); + tx.commit(); + let merged_update = tx.encode_update_v1(); + Some(()) + } + }, + Message::Auth(reason) => { + Some(()) + } + Message::AwarenessQuery => { + Some(()) + } + Message::Awareness(update) => { + Some(()) + } + Message::Custom(tag, data) => { + Some(()) + } + }); } + let merged_update = doc_store.doc().transact().encode_update_v1(); + doc_db + .insert(¶meters.id.clone(), &merged_update) + .await + .ok(); + + Ok(true) } diff --git a/client-app/src-tauri/src/state.rs b/client-app/src-tauri/src/state.rs index 69876871f9..4d79085dfa 100644 --- a/client-app/src-tauri/src/state.rs +++ b/client-app/src-tauri/src/state.rs @@ -1,11 +1,14 @@ use cloud_database::SqliteDBContext; +use jwst::Workspace; use jwst_storage::{BlobAutoStorage, DocAutoStorage}; use std::{fs, path::Path}; use tauri::api::path::document_dir; use tokio::sync::Mutex; pub struct AppStateRaw { - pub doc_storage: DocAutoStorage, + pub doc_db: DocAutoStorage, + /// yDoc for receiving yjs update and merge them, before serialize update into sqlite + pub doc_store: Workspace, pub blob_storage: BlobAutoStorage, pub metadata_db: SqliteDBContext, } @@ -37,7 +40,10 @@ impl AppStateRaw { fs::create_dir_all(affine_document_path.clone()).unwrap(); Some(Self { - doc_storage: DocAutoStorage::init_pool(&doc_db_env).await.unwrap(), + doc_db: DocAutoStorage::init_pool(&doc_db_env).await.unwrap(), + // with fake id, we only use yDoc inside of it + // TODO: use workspace pool, to handle multiple workspace + doc_store: Workspace::new(""), blob_storage: BlobAutoStorage::init_pool(&blob_db_env).await.unwrap(), metadata_db: SqliteDBContext::new(metadata_db_env).await, })