mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-20 19:46:32 +08:00
feat: init disk remote source
This commit is contained in:
@@ -430,7 +430,9 @@ fn parse_markdown_inner(markdown: &str) -> Result<MarkdownDocument, ParseError>
|
||||
table_handled = true;
|
||||
}
|
||||
Event::Html(html) | Event::InlineHtml(html) => {
|
||||
if let Some(text) = extract_wrapped_html_text(html) {
|
||||
if is_html_comment(html) || is_iframe_end_tag(html) {
|
||||
// Ignore HTML comments and iframe end tags inside table cells.
|
||||
} else if let Some(text) = extract_wrapped_html_text(html) {
|
||||
state.push_text(&text);
|
||||
} else if is_html_line_break(html) {
|
||||
state.push_text("\n");
|
||||
@@ -621,6 +623,9 @@ fn parse_markdown_inner(markdown: &str) -> Result<MarkdownDocument, ParseError>
|
||||
}
|
||||
}
|
||||
Event::Html(html) | Event::InlineHtml(html) => {
|
||||
if is_html_comment(&html) || is_iframe_end_tag(&html) {
|
||||
continue;
|
||||
}
|
||||
if is_ai_editable_comment(&html) {
|
||||
continue;
|
||||
}
|
||||
@@ -773,6 +778,9 @@ fn validate_markdown_inner(markdown: &str) -> Result<(), ParseError> {
|
||||
match event {
|
||||
Event::Start(tag) => ensure_supported_tag(&tag)?,
|
||||
Event::Html(html) | Event::InlineHtml(html) => {
|
||||
if is_html_comment(&html) || is_iframe_end_tag(&html) {
|
||||
continue;
|
||||
}
|
||||
if is_ai_editable_comment(&html) {
|
||||
continue;
|
||||
}
|
||||
@@ -936,6 +944,15 @@ fn is_ai_editable_comment(html: &str) -> bool {
|
||||
body.contains("block_id=") && body.contains("flavour=")
|
||||
}
|
||||
|
||||
fn is_html_comment(html: &str) -> bool {
|
||||
let trimmed = html.trim();
|
||||
trimmed.starts_with("<!--") && trimmed.ends_with("-->")
|
||||
}
|
||||
|
||||
fn is_iframe_end_tag(html: &str) -> bool {
|
||||
parse_html_tag(html).is_some_and(|tag| tag.closing && tag.name == "iframe")
|
||||
}
|
||||
|
||||
fn is_html_line_break(html: &str) -> bool {
|
||||
let trimmed = html.trim();
|
||||
if !trimmed.starts_with('<') || !trimmed.ends_with('>') {
|
||||
@@ -1716,6 +1733,13 @@ mod tests {
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_markdown_allows_html_comment() {
|
||||
let markdown = "# Title\n\n<!-- omit from toc -->\n\nContent.";
|
||||
let result = validate_markdown(markdown);
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_markdown_rejects_html() {
|
||||
let markdown = "# Title\n\n<div>HTML</div>";
|
||||
|
||||
@@ -282,6 +282,9 @@ pub fn parse_doc_to_markdown(
|
||||
0
|
||||
};
|
||||
let ai_block = ai_editable && block_level == 2;
|
||||
let ai_preserve_block = ai_block
|
||||
&& (matches!(flavour.as_str(), "affine:database" | "affine:callout")
|
||||
|| BlockFlavour::from_str(flavour.as_str()).is_none());
|
||||
|
||||
let mut block_markdown = String::new();
|
||||
|
||||
@@ -308,7 +311,9 @@ pub fn parse_doc_to_markdown(
|
||||
};
|
||||
renderer.write_block(&mut block_markdown, &spec, list_depth);
|
||||
} else {
|
||||
return Err(ParseError::ParserError(format!("unsupported_block_flavour:{flavour}")));
|
||||
block_markdown.push_str(&format!(
|
||||
"<!-- unsupported_block_flavour:{flavour} block_id={block_id} -->\n\n"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -317,6 +322,9 @@ pub fn parse_doc_to_markdown(
|
||||
markdown.push_str(&format!("<!-- block_id={block_id} flavour={flavour} -->\n"));
|
||||
}
|
||||
markdown.push_str(&block_markdown);
|
||||
if ai_preserve_block {
|
||||
markdown.push_str(&format!("<!-- block_id={block_id} flavour={flavour} end -->\n"));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(MarkdownResult {
|
||||
@@ -792,4 +800,59 @@ mod tests {
|
||||
assert!(md.contains("|A|B|"));
|
||||
assert!(md.contains("|---|---|"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_doc_to_markdown_skips_unsupported_block_flavour() {
|
||||
let doc_id = "unsupported-doc".to_string();
|
||||
let doc = DocOptions::new().with_guid(doc_id.clone()).build();
|
||||
let mut blocks = doc.get_or_create_map("blocks").unwrap();
|
||||
|
||||
let mut page = doc.create_map().unwrap();
|
||||
page.insert("sys:id".into(), "page").unwrap();
|
||||
page.insert("sys:flavour".into(), "affine:page").unwrap();
|
||||
let mut page_children = doc.create_array().unwrap();
|
||||
page_children.push("note").unwrap();
|
||||
page.insert("sys:children".into(), Value::Array(page_children)).unwrap();
|
||||
let mut page_title = doc.create_text().unwrap();
|
||||
page_title.insert(0, "Page").unwrap();
|
||||
page.insert("prop:title".into(), Value::Text(page_title)).unwrap();
|
||||
blocks.insert("page".into(), Value::Map(page)).unwrap();
|
||||
|
||||
let mut note = doc.create_map().unwrap();
|
||||
note.insert("sys:id".into(), "note").unwrap();
|
||||
note.insert("sys:flavour".into(), "affine:note").unwrap();
|
||||
let mut note_children = doc.create_array().unwrap();
|
||||
note_children.push("latex").unwrap();
|
||||
note_children.push("paragraph").unwrap();
|
||||
note.insert("sys:children".into(), Value::Array(note_children)).unwrap();
|
||||
note.insert("prop:displayMode".into(), "page").unwrap();
|
||||
blocks.insert("note".into(), Value::Map(note)).unwrap();
|
||||
|
||||
let mut unsupported = doc.create_map().unwrap();
|
||||
unsupported.insert("sys:id".into(), "latex").unwrap();
|
||||
unsupported.insert("sys:flavour".into(), "affine:latex").unwrap();
|
||||
unsupported
|
||||
.insert("sys:children".into(), Value::Array(doc.create_array().unwrap()))
|
||||
.unwrap();
|
||||
blocks.insert("latex".into(), Value::Map(unsupported)).unwrap();
|
||||
|
||||
let mut paragraph = doc.create_map().unwrap();
|
||||
paragraph.insert("sys:id".into(), "paragraph").unwrap();
|
||||
paragraph.insert("sys:flavour".into(), "affine:paragraph").unwrap();
|
||||
paragraph
|
||||
.insert("sys:children".into(), Value::Array(doc.create_array().unwrap()))
|
||||
.unwrap();
|
||||
let mut paragraph_text = doc.create_text().unwrap();
|
||||
paragraph_text.insert(0, "After unsupported block").unwrap();
|
||||
paragraph
|
||||
.insert("prop:text".into(), Value::Text(paragraph_text))
|
||||
.unwrap();
|
||||
blocks.insert("paragraph".into(), Value::Map(paragraph)).unwrap();
|
||||
|
||||
let doc_bin = doc.encode_update_v1().unwrap();
|
||||
let result = parse_doc_to_markdown(doc_bin, doc_id, false, None).expect("parse doc");
|
||||
|
||||
assert!(result.markdown.contains("unsupported_block_flavour:affine:latex"));
|
||||
assert!(result.markdown.contains("After unsupported block"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! Converts markdown content into AFFiNE-compatible y-octo document binary
|
||||
//! format.
|
||||
|
||||
use y_octo::DocOptions;
|
||||
use y_octo::{DocOptions, StateVector};
|
||||
|
||||
use super::{
|
||||
super::{
|
||||
@@ -73,7 +73,7 @@ fn build_doc_update(doc_id: &str, title: &str, blocks: &[BlockNode]) -> Result<V
|
||||
note_map.insert(PROP_HIDDEN.to_string(), Any::False)?;
|
||||
note_map.insert(PROP_DISPLAY_MODE.to_string(), Any::String("both".to_string()))?;
|
||||
|
||||
Ok(doc.encode_update_v1()?)
|
||||
Ok(doc.encode_state_as_update_v1(&StateVector::default())?)
|
||||
}
|
||||
|
||||
fn insert_block_trees(doc: &Doc, blocks_map: &mut Map, blocks: &[BlockNode]) -> Result<Vec<String>, ParseError> {
|
||||
|
||||
@@ -8,19 +8,37 @@ use std::collections::HashMap;
|
||||
use super::{
|
||||
super::{
|
||||
block_spec::{TreeNode, count_tree_nodes, text_delta_eq},
|
||||
blocksuite::{collect_child_ids, find_child_id_by_flavour},
|
||||
blocksuite::{collect_child_ids, find_child_id_by_flavour, get_string},
|
||||
markdown::{MAX_BLOCKS, parse_markdown_blocks},
|
||||
schema::{PROP_BACKGROUND, PROP_DISPLAY_MODE, PROP_ELEMENTS, PROP_HIDDEN, PROP_INDEX, PROP_XYWH, SURFACE_FLAVOUR},
|
||||
},
|
||||
builder::{
|
||||
ApplyBlockOptions, BOXED_NATIVE_TYPE, NOTE_BG_DARK, NOTE_BG_LIGHT, apply_block_spec, boxed_empty_map,
|
||||
insert_block_map, insert_block_tree, insert_children, insert_sys_fields, insert_text, note_background_map,
|
||||
text_ops_from_plain,
|
||||
},
|
||||
builder::{ApplyBlockOptions, apply_block_spec, insert_block_tree, insert_children},
|
||||
*,
|
||||
};
|
||||
|
||||
const MAX_LCS_CELLS: usize = 2_000_000;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum NodeSpec {
|
||||
Supported(BlockSpec),
|
||||
/// A block flavour we don't support for markdown diffing/updating (e.g.
|
||||
/// `affine:database`).
|
||||
///
|
||||
/// These nodes are treated as opaque: we preserve them and never modify their
|
||||
/// properties/children.
|
||||
Opaque {
|
||||
flavour: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct StoredNode {
|
||||
id: String,
|
||||
spec: BlockSpec,
|
||||
spec: NodeSpec,
|
||||
children: Vec<StoredNode>,
|
||||
}
|
||||
|
||||
@@ -30,6 +48,20 @@ impl TreeNode for StoredNode {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TargetNode {
|
||||
/// Optional block id marker from exported markdown (AI-editable markers).
|
||||
id_hint: Option<String>,
|
||||
spec: NodeSpec,
|
||||
children: Vec<TargetNode>,
|
||||
}
|
||||
|
||||
impl TreeNode for TargetNode {
|
||||
fn children(&self) -> &[TargetNode] {
|
||||
&self.children
|
||||
}
|
||||
}
|
||||
|
||||
struct DocState {
|
||||
doc: Doc,
|
||||
note_id: String,
|
||||
@@ -59,8 +91,24 @@ enum PatchOp {
|
||||
/// # Returns
|
||||
/// A binary vector representing only the delta (changes) to apply
|
||||
pub fn update_doc(existing_binary: &[u8], new_markdown: &str, doc_id: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let mut new_nodes = parse_markdown_blocks(new_markdown)?;
|
||||
let state = load_doc_state(existing_binary, doc_id)?;
|
||||
let state = match load_doc_state(existing_binary, doc_id) {
|
||||
Ok(state) => state,
|
||||
Err(ParseError::ParserError(msg))
|
||||
if matches!(
|
||||
msg.as_str(),
|
||||
"blocks map is empty" | "page block not found" | "note block not found"
|
||||
) =>
|
||||
{
|
||||
// The existing doc may be a stub/partial document (e.g. created by references)
|
||||
// and doesn't contain the canonical page/note structure yet. In that
|
||||
// case, initialize the doc from the markdown instead of failing hard.
|
||||
let new_nodes = parse_markdown_blocks(new_markdown)?;
|
||||
return init_doc_from_markdown(existing_binary, new_markdown, doc_id, &new_nodes);
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
let mut new_nodes = parse_markdown_targets(new_markdown)?;
|
||||
|
||||
check_limits(&state.blocks, &new_nodes)?;
|
||||
|
||||
@@ -74,6 +122,315 @@ pub fn update_doc(existing_binary: &[u8], new_markdown: &str, doc_id: &str) -> R
|
||||
Ok(state.doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct BlockMarker {
|
||||
id: String,
|
||||
flavour: String,
|
||||
end: bool,
|
||||
}
|
||||
|
||||
fn parse_block_marker_line(line: &str) -> Option<BlockMarker> {
|
||||
let trimmed = line.trim();
|
||||
if !trimmed.starts_with("<!--") || !trimmed.ends_with("-->") {
|
||||
return None;
|
||||
}
|
||||
let body = trimmed.trim_start_matches("<!--").trim_end_matches("-->").trim();
|
||||
if !body.contains("block_id=") || !body.contains("flavour=") {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut id: Option<String> = None;
|
||||
let mut flavour: Option<String> = None;
|
||||
let mut end = false;
|
||||
|
||||
for token in body.split_whitespace() {
|
||||
if token == "end" || token == "type=end" || token == "end=true" {
|
||||
end = true;
|
||||
continue;
|
||||
}
|
||||
if let Some(value) = token.strip_prefix("block_id=") {
|
||||
if !value.is_empty() {
|
||||
id = Some(value.to_string());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if let Some(value) = token.strip_prefix("flavour=") {
|
||||
if !value.is_empty() {
|
||||
flavour = Some(value.to_string());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
Some(BlockMarker {
|
||||
id: id?,
|
||||
flavour: flavour?,
|
||||
end,
|
||||
})
|
||||
}
|
||||
|
||||
fn should_preserve_marker_flavour(flavour: &str) -> bool {
|
||||
matches!(flavour, "affine:database" | "affine:callout")
|
||||
}
|
||||
|
||||
fn parse_markdown_targets(markdown: &str) -> Result<Vec<TargetNode>, ParseError> {
|
||||
// Fast path: no markers, behave like the original implementation.
|
||||
if !markdown.contains("block_id=") || !markdown.contains("flavour=") {
|
||||
let blocks = parse_markdown_blocks(markdown)?;
|
||||
return Ok(blocks.into_iter().map(|b| target_from_block_node(b, None)).collect());
|
||||
}
|
||||
|
||||
// Split the markdown by marker comments. For most blocks, a marker indicates
|
||||
// the start of a block. For preserved blocks (e.g. database), an optional end
|
||||
// marker can be emitted so users can append new content after the preserved
|
||||
// section without needing to add markers manually.
|
||||
let mut segments: Vec<(Option<BlockMarker>, String)> = Vec::new();
|
||||
let mut current_marker: Option<BlockMarker> = None;
|
||||
let mut current_body = String::new();
|
||||
let mut saw_marker = false;
|
||||
|
||||
for line in markdown.lines() {
|
||||
if let Some(marker) = parse_block_marker_line(line) {
|
||||
saw_marker = true;
|
||||
if marker.end {
|
||||
if current_marker.is_some() || !current_body.is_empty() {
|
||||
segments.push((current_marker.take(), std::mem::take(&mut current_body)));
|
||||
}
|
||||
// Close the marker scope; subsequent lines belong to an unmarked segment.
|
||||
current_marker = None;
|
||||
continue;
|
||||
}
|
||||
|
||||
if current_marker.is_some() || !current_body.is_empty() {
|
||||
segments.push((current_marker.take(), std::mem::take(&mut current_body)));
|
||||
}
|
||||
current_marker = Some(marker);
|
||||
continue;
|
||||
}
|
||||
|
||||
current_body.push_str(line);
|
||||
current_body.push('\n');
|
||||
}
|
||||
|
||||
if current_marker.is_some() || !current_body.is_empty() {
|
||||
segments.push((current_marker.take(), current_body));
|
||||
}
|
||||
|
||||
if !saw_marker {
|
||||
let blocks = parse_markdown_blocks(markdown)?;
|
||||
return Ok(blocks.into_iter().map(|b| target_from_block_node(b, None)).collect());
|
||||
}
|
||||
|
||||
let mut out: Vec<TargetNode> = Vec::new();
|
||||
for (marker, body) in segments {
|
||||
if let Some(marker) = marker {
|
||||
let preserve =
|
||||
should_preserve_marker_flavour(&marker.flavour) || BlockFlavour::from_str(&marker.flavour).is_none();
|
||||
if preserve {
|
||||
out.push(TargetNode {
|
||||
id_hint: Some(marker.id),
|
||||
spec: NodeSpec::Opaque {
|
||||
flavour: marker.flavour,
|
||||
},
|
||||
children: Vec::new(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let blocks = parse_markdown_blocks(&body)?;
|
||||
for (idx, block) in blocks.into_iter().enumerate() {
|
||||
let id_hint = if idx == 0 { Some(marker.id.clone()) } else { None };
|
||||
out.push(target_from_block_node(block, id_hint));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let trimmed = body.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let blocks = parse_markdown_blocks(&body)?;
|
||||
for block in blocks {
|
||||
out.push(target_from_block_node(block, None));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn target_from_block_node(node: BlockNode, id_hint: Option<String>) -> TargetNode {
|
||||
TargetNode {
|
||||
id_hint,
|
||||
spec: NodeSpec::Supported(node.spec),
|
||||
children: node
|
||||
.children
|
||||
.into_iter()
|
||||
.map(|child| target_from_block_node(child, None))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn target_node_to_block_node(node: &TargetNode) -> Result<BlockNode, ParseError> {
|
||||
let NodeSpec::Supported(spec) = &node.spec else {
|
||||
return Err(ParseError::ParserError("cannot_insert_opaque_block".into()));
|
||||
};
|
||||
Ok(BlockNode {
|
||||
spec: spec.clone(),
|
||||
children: node
|
||||
.children
|
||||
.iter()
|
||||
.map(target_node_to_block_node)
|
||||
.collect::<Result<Vec<_>, _>>()?,
|
||||
})
|
||||
}
|
||||
|
||||
fn init_doc_from_markdown(
|
||||
existing_binary: &[u8],
|
||||
new_markdown: &str,
|
||||
doc_id: &str,
|
||||
blocks: &[BlockNode],
|
||||
) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = load_doc(existing_binary, Some(doc_id))?;
|
||||
let state_before = doc.get_state_vector();
|
||||
let mut blocks_map = doc.get_or_create_map("blocks")?;
|
||||
|
||||
let title = derive_title_from_markdown(new_markdown).unwrap_or_else(|| "Untitled".to_string());
|
||||
// Prefer reusing an existing page block if the doc already has one (but is
|
||||
// missing surface/note). This avoids creating multiple page roots when
|
||||
// recovering from partial documents.
|
||||
if !blocks_map.is_empty() {
|
||||
let index = build_block_index(&blocks_map);
|
||||
if let Some(page_id) = find_block_id_by_flavour(&index.block_pool, PAGE_FLAVOUR) {
|
||||
insert_page_children(&doc, &mut blocks_map, &page_id, &title, blocks)?;
|
||||
return Ok(doc.encode_state_as_update_v1(&state_before)?);
|
||||
}
|
||||
}
|
||||
|
||||
insert_page_doc(&doc, &mut blocks_map, &title, blocks)?;
|
||||
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
fn derive_title_from_markdown(markdown: &str) -> Option<String> {
|
||||
for line in markdown.lines() {
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Some(rest) = trimmed.strip_prefix("# ") {
|
||||
let title = rest.trim();
|
||||
if !title.is_empty() {
|
||||
return Some(title.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn insert_page_doc(doc: &Doc, blocks_map: &mut Map, title: &str, blocks: &[BlockNode]) -> Result<(), ParseError> {
|
||||
let page_id = nanoid::nanoid!();
|
||||
let surface_id = nanoid::nanoid!();
|
||||
let note_id = nanoid::nanoid!();
|
||||
|
||||
// Insert root blocks first to establish stable IDs.
|
||||
let mut page_map = insert_block_map(doc, blocks_map, &page_id)?;
|
||||
let mut surface_map = insert_block_map(doc, blocks_map, &surface_id)?;
|
||||
let mut note_map = insert_block_map(doc, blocks_map, ¬e_id)?;
|
||||
|
||||
// Create content blocks under note.
|
||||
let content_ids = insert_block_trees(doc, blocks_map, blocks)?;
|
||||
|
||||
// Page block.
|
||||
insert_sys_fields(&mut page_map, &page_id, PAGE_FLAVOUR)?;
|
||||
insert_children(doc, &mut page_map, &[surface_id.clone(), note_id.clone()])?;
|
||||
insert_text(doc, &mut page_map, PROP_TITLE, &text_ops_from_plain(title))?;
|
||||
|
||||
// Surface block.
|
||||
insert_sys_fields(&mut surface_map, &surface_id, SURFACE_FLAVOUR)?;
|
||||
insert_children(doc, &mut surface_map, &[])?;
|
||||
let mut boxed = boxed_empty_map(doc)?;
|
||||
surface_map.insert(PROP_ELEMENTS.to_string(), Value::Map(boxed.clone()))?;
|
||||
boxed.insert("type".to_string(), Any::String(BOXED_NATIVE_TYPE.to_string()))?;
|
||||
let value = doc.create_map()?;
|
||||
boxed.insert("value".to_string(), Value::Map(value))?;
|
||||
|
||||
// Note block.
|
||||
insert_sys_fields(&mut note_map, ¬e_id, NOTE_FLAVOUR)?;
|
||||
insert_children(doc, &mut note_map, &content_ids)?;
|
||||
let mut background = note_background_map(doc)?;
|
||||
note_map.insert(PROP_BACKGROUND.to_string(), Value::Map(background.clone()))?;
|
||||
background.insert("light".to_string(), Any::String(NOTE_BG_LIGHT.to_string()))?;
|
||||
background.insert("dark".to_string(), Any::String(NOTE_BG_DARK.to_string()))?;
|
||||
note_map.insert(PROP_XYWH.to_string(), Any::String("[0,0,800,95]".to_string()))?;
|
||||
note_map.insert(PROP_INDEX.to_string(), Any::String("a0".to_string()))?;
|
||||
note_map.insert(PROP_HIDDEN.to_string(), Any::False)?;
|
||||
note_map.insert(PROP_DISPLAY_MODE.to_string(), Any::String("both".to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn insert_page_children(
|
||||
doc: &Doc,
|
||||
blocks_map: &mut Map,
|
||||
page_id: &str,
|
||||
title: &str,
|
||||
blocks: &[BlockNode],
|
||||
) -> Result<(), ParseError> {
|
||||
let surface_id = nanoid::nanoid!();
|
||||
let note_id = nanoid::nanoid!();
|
||||
|
||||
// Insert root blocks first to establish stable IDs.
|
||||
let mut surface_map = insert_block_map(doc, blocks_map, &surface_id)?;
|
||||
let mut note_map = insert_block_map(doc, blocks_map, ¬e_id)?;
|
||||
|
||||
// Create content blocks under note.
|
||||
let content_ids = insert_block_trees(doc, blocks_map, blocks)?;
|
||||
|
||||
let Some(mut page_map) = blocks_map.get(page_id).and_then(|v| v.to_map()) else {
|
||||
return Err(ParseError::ParserError("page block not found".into()));
|
||||
};
|
||||
|
||||
// Page block.
|
||||
insert_sys_fields(&mut page_map, page_id, PAGE_FLAVOUR)?;
|
||||
insert_children(doc, &mut page_map, &[surface_id.clone(), note_id.clone()])?;
|
||||
if page_map.get(PROP_TITLE).is_none() {
|
||||
insert_text(doc, &mut page_map, PROP_TITLE, &text_ops_from_plain(title))?;
|
||||
}
|
||||
|
||||
// Surface block.
|
||||
insert_sys_fields(&mut surface_map, &surface_id, SURFACE_FLAVOUR)?;
|
||||
insert_children(doc, &mut surface_map, &[])?;
|
||||
let mut boxed = boxed_empty_map(doc)?;
|
||||
surface_map.insert(PROP_ELEMENTS.to_string(), Value::Map(boxed.clone()))?;
|
||||
boxed.insert("type".to_string(), Any::String(BOXED_NATIVE_TYPE.to_string()))?;
|
||||
let value = doc.create_map()?;
|
||||
boxed.insert("value".to_string(), Value::Map(value))?;
|
||||
|
||||
// Note block.
|
||||
insert_sys_fields(&mut note_map, ¬e_id, NOTE_FLAVOUR)?;
|
||||
insert_children(doc, &mut note_map, &content_ids)?;
|
||||
let mut background = note_background_map(doc)?;
|
||||
note_map.insert(PROP_BACKGROUND.to_string(), Value::Map(background.clone()))?;
|
||||
background.insert("light".to_string(), Any::String(NOTE_BG_LIGHT.to_string()))?;
|
||||
background.insert("dark".to_string(), Any::String(NOTE_BG_DARK.to_string()))?;
|
||||
note_map.insert(PROP_XYWH.to_string(), Any::String("[0,0,800,95]".to_string()))?;
|
||||
note_map.insert(PROP_INDEX.to_string(), Any::String("a0".to_string()))?;
|
||||
note_map.insert(PROP_HIDDEN.to_string(), Any::False)?;
|
||||
note_map.insert(PROP_DISPLAY_MODE.to_string(), Any::String("both".to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn insert_block_trees(doc: &Doc, blocks_map: &mut Map, blocks: &[BlockNode]) -> Result<Vec<String>, ParseError> {
|
||||
let mut ids = Vec::with_capacity(blocks.len());
|
||||
for block in blocks {
|
||||
let id = insert_block_tree(doc, blocks_map, block)?;
|
||||
ids.push(id);
|
||||
}
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
fn load_doc_state(binary: &[u8], doc_id: &str) -> Result<DocState, ParseError> {
|
||||
let doc = load_doc(binary, Some(doc_id))?;
|
||||
|
||||
@@ -110,14 +467,31 @@ fn load_doc_state(binary: &[u8], doc_id: &str) -> Result<DocState, ParseError> {
|
||||
}
|
||||
|
||||
fn build_stored_tree(block_id: &str, block: &Map, pool: &HashMap<String, Map>) -> Result<StoredNode, ParseError> {
|
||||
let spec = BlockSpec::from_block_map(block)?;
|
||||
|
||||
let child_ids = collect_child_ids(block);
|
||||
let flavour = get_string(block, "sys:flavour").unwrap_or_default();
|
||||
|
||||
let spec = match BlockSpec::from_block_map(block) {
|
||||
Ok(spec) => spec,
|
||||
Err(ParseError::ParserError(msg)) if msg.starts_with("unsupported block flavour:") => {
|
||||
return Ok(StoredNode {
|
||||
id: block_id.to_string(),
|
||||
spec: NodeSpec::Opaque { flavour },
|
||||
children: Vec::new(),
|
||||
});
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
// Only list/callout are supported as containers for markdown diffing.
|
||||
// For any other block with children, treat as opaque so we never corrupt it.
|
||||
if !child_ids.is_empty() && !matches!(spec.flavour, BlockFlavour::List | BlockFlavour::Callout) {
|
||||
return Err(ParseError::ParserError(format!(
|
||||
"unsupported children on block: {block_id}"
|
||||
)));
|
||||
return Ok(StoredNode {
|
||||
id: block_id.to_string(),
|
||||
spec: NodeSpec::Opaque { flavour },
|
||||
children: Vec::new(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut children = Vec::new();
|
||||
for child_id in child_ids {
|
||||
let child_block = pool
|
||||
@@ -128,7 +502,7 @@ fn build_stored_tree(block_id: &str, block: &Map, pool: &HashMap<String, Map>) -
|
||||
|
||||
Ok(StoredNode {
|
||||
id: block_id.to_string(),
|
||||
spec,
|
||||
spec: NodeSpec::Supported(spec),
|
||||
children,
|
||||
})
|
||||
}
|
||||
@@ -137,7 +511,7 @@ fn sync_nodes(
|
||||
doc: &Doc,
|
||||
blocks_map: &mut Map,
|
||||
current: &[StoredNode],
|
||||
target: &mut [BlockNode],
|
||||
target: &mut [TargetNode],
|
||||
) -> Result<Vec<String>, ParseError> {
|
||||
let ops = diff_blocks(current, target);
|
||||
let mut new_children = Vec::new();
|
||||
@@ -148,29 +522,47 @@ fn sync_nodes(
|
||||
PatchOp::Keep(old_idx, new_idx) => {
|
||||
let old_node = ¤t[old_idx];
|
||||
let new_node = &target[new_idx];
|
||||
update_block_props(doc, blocks_map, old_node, &new_node.spec, true)?;
|
||||
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
|
||||
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
|
||||
if let (NodeSpec::Supported(old_spec), NodeSpec::Supported(new_spec)) = (&old_node.spec, &new_node.spec) {
|
||||
update_block_props(doc, blocks_map, &old_node.id, old_spec, new_spec, true)?;
|
||||
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
|
||||
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
|
||||
} else {
|
||||
// Preserve opaque blocks (and any mismatched marker blocks) as-is.
|
||||
// Don't touch their properties or children ordering.
|
||||
}
|
||||
new_children.push(old_node.id.clone());
|
||||
}
|
||||
PatchOp::Update(old_idx, new_idx) => {
|
||||
let old_node = ¤t[old_idx];
|
||||
let new_node = &target[new_idx];
|
||||
update_block_props(doc, blocks_map, old_node, &new_node.spec, false)?;
|
||||
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
|
||||
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
|
||||
if let (NodeSpec::Supported(old_spec), NodeSpec::Supported(new_spec)) = (&old_node.spec, &new_node.spec) {
|
||||
update_block_props(doc, blocks_map, &old_node.id, old_spec, new_spec, false)?;
|
||||
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
|
||||
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
|
||||
} else {
|
||||
// Opaque blocks are never updated from markdown.
|
||||
}
|
||||
new_children.push(old_node.id.clone());
|
||||
}
|
||||
PatchOp::Insert(new_idx) => {
|
||||
let new_id = insert_block_tree(doc, blocks_map, &target[new_idx])?;
|
||||
new_children.push(new_id);
|
||||
if let Ok(node) = target_node_to_block_node(&target[new_idx]) {
|
||||
let new_id = insert_block_tree(doc, blocks_map, &node)?;
|
||||
new_children.push(new_id);
|
||||
}
|
||||
}
|
||||
PatchOp::Delete(old_idx) => {
|
||||
let node = ¤t[old_idx];
|
||||
if node.spec.flavour == BlockFlavour::Callout {
|
||||
new_children.push(node.id.clone());
|
||||
} else {
|
||||
collect_tree_ids(node, &mut to_remove);
|
||||
match &node.spec {
|
||||
NodeSpec::Opaque { .. } => {
|
||||
// Never delete opaque blocks when syncing from markdown. They might contain
|
||||
// rich data that can't be represented in markdown, so keeping them
|
||||
// avoids data loss.
|
||||
new_children.push(node.id.clone());
|
||||
}
|
||||
NodeSpec::Supported(spec) if spec.flavour == BlockFlavour::Callout => {
|
||||
new_children.push(node.id.clone());
|
||||
}
|
||||
NodeSpec::Supported(_) => collect_tree_ids(node, &mut to_remove),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -183,7 +575,7 @@ fn sync_nodes(
|
||||
Ok(new_children)
|
||||
}
|
||||
|
||||
fn diff_blocks(current: &[StoredNode], target: &[BlockNode]) -> Vec<PatchOp> {
|
||||
fn diff_blocks(current: &[StoredNode], target: &[TargetNode]) -> Vec<PatchOp> {
|
||||
let old_len = current.len();
|
||||
let new_len = target.len();
|
||||
|
||||
@@ -198,10 +590,10 @@ fn diff_blocks(current: &[StoredNode], target: &[BlockNode]) -> Vec<PatchOp> {
|
||||
|
||||
for i in 1..=old_len {
|
||||
for j in 1..=new_len {
|
||||
let old_spec = ¤t[i - 1].spec;
|
||||
let new_spec = &target[j - 1].spec;
|
||||
let old_node = ¤t[i - 1];
|
||||
let new_node = &target[j - 1];
|
||||
|
||||
if old_spec.is_exact(new_spec) {
|
||||
if nodes_align(old_node, new_node) {
|
||||
lcs[i][j] = lcs[i - 1][j - 1] + 1;
|
||||
} else {
|
||||
lcs[i][j] = std::cmp::max(lcs[i - 1][j], lcs[i][j - 1]);
|
||||
@@ -215,14 +607,18 @@ fn diff_blocks(current: &[StoredNode], target: &[BlockNode]) -> Vec<PatchOp> {
|
||||
|
||||
while i > 0 || j > 0 {
|
||||
if i > 0 && j > 0 {
|
||||
let old_spec = ¤t[i - 1].spec;
|
||||
let new_spec = &target[j - 1].spec;
|
||||
let old_node = ¤t[i - 1];
|
||||
let new_node = &target[j - 1];
|
||||
|
||||
if old_spec.is_exact(new_spec) {
|
||||
ops.push(PatchOp::Keep(i - 1, j - 1));
|
||||
if nodes_align(old_node, new_node) {
|
||||
if nodes_should_update(old_node, new_node) {
|
||||
ops.push(PatchOp::Update(i - 1, j - 1));
|
||||
} else {
|
||||
ops.push(PatchOp::Keep(i - 1, j - 1));
|
||||
}
|
||||
i -= 1;
|
||||
j -= 1;
|
||||
} else if old_spec.is_similar(new_spec)
|
||||
} else if nodes_similar(old_node, new_node)
|
||||
&& lcs[i - 1][j - 1] >= lcs[i - 1][j]
|
||||
&& lcs[i - 1][j - 1] >= lcs[i][j - 1]
|
||||
{
|
||||
@@ -249,15 +645,60 @@ fn diff_blocks(current: &[StoredNode], target: &[BlockNode]) -> Vec<PatchOp> {
|
||||
ops
|
||||
}
|
||||
|
||||
fn nodes_align(old_node: &StoredNode, new_node: &TargetNode) -> bool {
|
||||
if marker_matches(old_node, new_node) {
|
||||
return true;
|
||||
}
|
||||
match (&old_node.spec, &new_node.spec) {
|
||||
(NodeSpec::Supported(old_spec), NodeSpec::Supported(new_spec)) => old_spec.is_exact(new_spec),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn nodes_should_update(old_node: &StoredNode, new_node: &TargetNode) -> bool {
|
||||
if marker_matches(old_node, new_node) {
|
||||
return match (&old_node.spec, &new_node.spec) {
|
||||
(NodeSpec::Supported(old_spec), NodeSpec::Supported(new_spec)) => !old_spec.is_exact(new_spec),
|
||||
_ => false,
|
||||
};
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn nodes_similar(old_node: &StoredNode, new_node: &TargetNode) -> bool {
|
||||
match (&old_node.spec, &new_node.spec) {
|
||||
(NodeSpec::Supported(old_spec), NodeSpec::Supported(new_spec)) => old_spec.is_similar(new_spec),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn marker_matches(old_node: &StoredNode, new_node: &TargetNode) -> bool {
|
||||
let Some(id) = new_node.id_hint.as_deref() else {
|
||||
return false;
|
||||
};
|
||||
if id != old_node.id.as_str() {
|
||||
return false;
|
||||
}
|
||||
node_flavour_str(&old_node.spec) == node_flavour_str(&new_node.spec)
|
||||
}
|
||||
|
||||
fn node_flavour_str(spec: &NodeSpec) -> &str {
|
||||
match spec {
|
||||
NodeSpec::Supported(spec) => spec.flavour.as_str(),
|
||||
NodeSpec::Opaque { flavour } => flavour.as_str(),
|
||||
}
|
||||
}
|
||||
|
||||
fn update_block_props(
|
||||
doc: &Doc,
|
||||
blocks_map: &mut Map,
|
||||
node: &StoredNode,
|
||||
node_id: &str,
|
||||
current: &BlockSpec,
|
||||
target: &BlockSpec,
|
||||
preserve_text: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
let Some(mut block) = blocks_map.get(&node.id).and_then(|v| v.to_map()) else {
|
||||
return Err(ParseError::ParserError(format!("Block {} not found", node.id)));
|
||||
let Some(mut block) = blocks_map.get(node_id).and_then(|v| v.to_map()) else {
|
||||
return Err(ParseError::ParserError(format!("Block {} not found", node_id)));
|
||||
};
|
||||
|
||||
let preserve = match target.flavour {
|
||||
@@ -266,7 +707,7 @@ fn update_block_props(
|
||||
| BlockFlavour::Bookmark
|
||||
| BlockFlavour::EmbedYoutube
|
||||
| BlockFlavour::EmbedIframe => preserve_text,
|
||||
_ => preserve_text || text_delta_eq(&node.spec.text, &target.text),
|
||||
_ => preserve_text || text_delta_eq(¤t.text, &target.text),
|
||||
};
|
||||
|
||||
apply_block_spec(
|
||||
@@ -302,7 +743,7 @@ fn collect_tree_ids(node: &StoredNode, output: &mut Vec<String>) {
|
||||
}
|
||||
}
|
||||
|
||||
fn check_limits(current: &[StoredNode], target: &[BlockNode]) -> Result<(), ParseError> {
|
||||
fn check_limits(current: &[StoredNode], target: &[TargetNode]) -> Result<(), ParseError> {
|
||||
let current_count = count_tree_nodes(current);
|
||||
let target_count = count_tree_nodes(target);
|
||||
|
||||
@@ -319,7 +760,7 @@ fn check_limits(current: &[StoredNode], target: &[BlockNode]) -> Result<(), Pars
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::{Any, DocOptions, TextDeltaOp, TextInsert};
|
||||
use y_octo::{Any, DocOptions, StateVector, TextDeltaOp, TextInsert};
|
||||
|
||||
use super::{super::builder::text_ops_from_plain, *};
|
||||
use crate::doc_parser::{
|
||||
@@ -647,6 +1088,233 @@ mod tests {
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_fallback_when_blocks_empty() {
|
||||
let doc_id = "stub-empty-blocks";
|
||||
let markdown = "# From Markdown\n\nHello from markdown.";
|
||||
|
||||
// Build a valid ydoc update that results in an empty `blocks` map.
|
||||
// NOTE: yjs/y-octo may encode a completely empty doc as `[0,0]`, which we treat
|
||||
// as empty/invalid. We intentionally insert + remove a temp key so the
|
||||
// update is non-empty while the final map is empty.
|
||||
let doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
let mut blocks = doc.get_or_create_map("blocks").expect("create blocks map");
|
||||
blocks
|
||||
.insert("tmp".to_string(), Any::String("1".to_string()))
|
||||
.expect("insert temp");
|
||||
blocks.remove("tmp");
|
||||
let stub_bin = doc
|
||||
.encode_state_as_update_v1(&StateVector::default())
|
||||
.expect("encode stub update");
|
||||
assert!(
|
||||
!stub_bin.is_empty() && stub_bin.as_slice() != [0, 0],
|
||||
"stub update should not be empty update"
|
||||
);
|
||||
|
||||
let delta = update_doc(&stub_bin, markdown, doc_id).expect("fallback delta");
|
||||
assert!(!delta.is_empty(), "delta should contain changes");
|
||||
|
||||
let mut updated = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
updated
|
||||
.apply_update_from_binary_v1(&stub_bin)
|
||||
.expect("apply stub update");
|
||||
updated
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("apply fallback delta");
|
||||
|
||||
let blocks_map = updated.get_map("blocks").expect("blocks map exists");
|
||||
|
||||
let mut page: Option<Map> = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
|
||||
{
|
||||
page = Some(block_map);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let page = page.expect("page block created");
|
||||
assert_eq!(
|
||||
get_string(&page, "prop:title").as_deref(),
|
||||
Some("From Markdown"),
|
||||
"page title should be derived from markdown H1"
|
||||
);
|
||||
|
||||
let index = build_block_index(&blocks_map);
|
||||
let note_id = find_child_id_by_flavour(&page, &index.block_pool, NOTE_FLAVOUR).expect("note child exists");
|
||||
|
||||
let note = index.block_pool.get(¬e_id).expect("note block exists").clone();
|
||||
assert!(
|
||||
!collect_child_ids(¬e).is_empty(),
|
||||
"note should contain imported content blocks"
|
||||
);
|
||||
|
||||
let full_bin = updated
|
||||
.encode_state_as_update_v1(&StateVector::default())
|
||||
.expect("encode full doc");
|
||||
let md = parse_doc_to_markdown(full_bin, doc_id.to_string(), false, None).expect("render markdown");
|
||||
assert!(md.markdown.contains("Hello from markdown."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_fallback_when_page_missing() {
|
||||
let doc_id = "stub-page-missing";
|
||||
let markdown = "# Title\n\nUpdated content.";
|
||||
|
||||
// Build a stub doc that has some blocks, but no `affine:page` root.
|
||||
let doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
let mut blocks_map = doc.get_or_create_map("blocks").expect("create blocks map");
|
||||
let para_id = "para-1";
|
||||
let mut para = insert_block_map(&doc, &mut blocks_map, para_id).expect("insert para");
|
||||
insert_sys_fields(&mut para, para_id, "affine:paragraph").expect("sys fields");
|
||||
insert_children(&doc, &mut para, &[]).expect("children");
|
||||
|
||||
let stub_bin = doc
|
||||
.encode_state_as_update_v1(&StateVector::default())
|
||||
.expect("encode stub update");
|
||||
assert!(!stub_bin.is_empty(), "stub update should not be empty");
|
||||
|
||||
let delta = update_doc(&stub_bin, markdown, doc_id).expect("fallback delta");
|
||||
assert!(!delta.is_empty(), "delta should contain changes");
|
||||
|
||||
let mut updated = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
updated
|
||||
.apply_update_from_binary_v1(&stub_bin)
|
||||
.expect("apply stub update");
|
||||
updated
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("apply fallback delta");
|
||||
|
||||
let blocks_map = updated.get_map("blocks").expect("blocks map exists");
|
||||
let index = build_block_index(&blocks_map);
|
||||
let page_id = find_block_id_by_flavour(&index.block_pool, PAGE_FLAVOUR).expect("page block exists");
|
||||
let page = index.block_pool.get(&page_id).expect("page map exists").clone();
|
||||
|
||||
let note_id = find_child_id_by_flavour(&page, &index.block_pool, NOTE_FLAVOUR).expect("note child exists");
|
||||
let note = index.block_pool.get(¬e_id).expect("note block exists").clone();
|
||||
assert!(
|
||||
!collect_child_ids(¬e).is_empty(),
|
||||
"note should contain imported content blocks"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_fallback_when_note_missing() {
|
||||
let doc_id = "stub-note-missing";
|
||||
let markdown = "# Title\n\nUpdated content.";
|
||||
|
||||
// Build a stub doc that has an `affine:page` block but doesn't contain a note
|
||||
// child.
|
||||
let doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
let mut blocks_map = doc.get_or_create_map("blocks").expect("create blocks map");
|
||||
let page_id = "page-1";
|
||||
let mut page = insert_block_map(&doc, &mut blocks_map, page_id).expect("insert page");
|
||||
insert_sys_fields(&mut page, page_id, PAGE_FLAVOUR).expect("sys fields");
|
||||
insert_children(&doc, &mut page, &[]).expect("children");
|
||||
|
||||
let stub_bin = doc
|
||||
.encode_state_as_update_v1(&StateVector::default())
|
||||
.expect("encode stub update");
|
||||
assert!(!stub_bin.is_empty(), "stub update should not be empty");
|
||||
|
||||
let delta = update_doc(&stub_bin, markdown, doc_id).expect("fallback delta");
|
||||
assert!(!delta.is_empty(), "delta should contain changes");
|
||||
|
||||
let mut updated = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
updated
|
||||
.apply_update_from_binary_v1(&stub_bin)
|
||||
.expect("apply stub update");
|
||||
updated
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("apply fallback delta");
|
||||
|
||||
let blocks_map = updated.get_map("blocks").expect("blocks map exists");
|
||||
let index = build_block_index(&blocks_map);
|
||||
let page_id = find_block_id_by_flavour(&index.block_pool, PAGE_FLAVOUR).expect("page block exists");
|
||||
let page = index.block_pool.get(&page_id).expect("page map exists").clone();
|
||||
|
||||
let note_id = find_child_id_by_flavour(&page, &index.block_pool, NOTE_FLAVOUR).expect("note child exists");
|
||||
let note = index.block_pool.get(¬e_id).expect("note block exists").clone();
|
||||
assert!(
|
||||
!collect_child_ids(¬e).is_empty(),
|
||||
"note should contain imported content blocks"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_preserves_opaque_blocks_when_unsupported_block_flavour() {
|
||||
let doc_id = "unsupported-flavour-replace";
|
||||
|
||||
// Build a doc with canonical page/note structure, but add an unsupported block
|
||||
// flavour under note. This simulates real-world docs that contain blocks we
|
||||
// don't support for structural diffing.
|
||||
let doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
let mut blocks_map = doc.get_or_create_map("blocks").expect("create blocks map");
|
||||
|
||||
let page_id = "page-1";
|
||||
let surface_id = "surface-1";
|
||||
let note_id = "note-1";
|
||||
let db_id = "db-1";
|
||||
|
||||
let mut page = insert_block_map(&doc, &mut blocks_map, page_id).expect("insert page");
|
||||
let mut surface = insert_block_map(&doc, &mut blocks_map, surface_id).expect("insert surface");
|
||||
let mut note = insert_block_map(&doc, &mut blocks_map, note_id).expect("insert note");
|
||||
let mut db = insert_block_map(&doc, &mut blocks_map, db_id).expect("insert db");
|
||||
|
||||
insert_sys_fields(&mut page, page_id, PAGE_FLAVOUR).expect("page sys fields");
|
||||
insert_children(&doc, &mut page, &[surface_id.to_string(), note_id.to_string()]).expect("page children");
|
||||
insert_text(&doc, &mut page, PROP_TITLE, &text_ops_from_plain("Title")).expect("page title");
|
||||
|
||||
insert_sys_fields(&mut surface, surface_id, SURFACE_FLAVOUR).expect("surface sys fields");
|
||||
insert_children(&doc, &mut surface, &[]).expect("surface children");
|
||||
let mut boxed = boxed_empty_map(&doc).expect("boxed map");
|
||||
surface
|
||||
.insert(PROP_ELEMENTS.to_string(), Value::Map(boxed.clone()))
|
||||
.expect("surface elements");
|
||||
boxed
|
||||
.insert("type".to_string(), Any::String(BOXED_NATIVE_TYPE.to_string()))
|
||||
.expect("boxed type");
|
||||
let value = doc.create_map().expect("boxed value map");
|
||||
boxed
|
||||
.insert("value".to_string(), Value::Map(value))
|
||||
.expect("boxed value");
|
||||
|
||||
insert_sys_fields(&mut note, note_id, NOTE_FLAVOUR).expect("note sys fields");
|
||||
insert_children(&doc, &mut note, &[db_id.to_string()]).expect("note children");
|
||||
|
||||
// Unsupported flavour.
|
||||
insert_sys_fields(&mut db, db_id, "affine:database").expect("db sys fields");
|
||||
insert_children(&doc, &mut db, &[]).expect("db children");
|
||||
|
||||
let initial_bin = doc
|
||||
.encode_state_as_update_v1(&StateVector::default())
|
||||
.expect("encode initial");
|
||||
|
||||
// Updating should succeed and preserve the opaque block rather than deleting
|
||||
// it.
|
||||
let updated_md = "# New Title\n\nHello.";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta");
|
||||
assert!(!delta.is_empty(), "delta should contain changes");
|
||||
|
||||
let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("apply initial");
|
||||
updated_doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let blocks_map = updated_doc.get_map("blocks").expect("blocks map");
|
||||
assert!(
|
||||
blocks_map.get(db_id).is_some(),
|
||||
"opaque block should be preserved when syncing from markdown"
|
||||
);
|
||||
|
||||
let md = parse_doc_to_markdown(updated_doc.encode_update_v1().unwrap(), doc_id.to_string(), false, None)
|
||||
.expect("render markdown")
|
||||
.markdown;
|
||||
assert!(md.contains("Hello."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_markdown_too_large() {
|
||||
let initial_md = "# Title\n\nContent.";
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
"./broadcast-channel": "./src/impls/broadcast-channel/index.ts",
|
||||
"./idb/v1": "./src/impls/idb/v1/index.ts",
|
||||
"./cloud": "./src/impls/cloud/index.ts",
|
||||
"./disk": "./src/impls/disk/index.ts",
|
||||
"./sqlite": "./src/impls/sqlite/index.ts",
|
||||
"./sqlite/v1": "./src/impls/sqlite/v1/index.ts",
|
||||
"./sync": "./src/sync/index.ts",
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { AutoReconnectConnection } from '../../connection';
|
||||
import type { DocClock, DocUpdate } from '../../storage';
|
||||
import { type SpaceType, universalId } from '../../utils/universal-id';
|
||||
|
||||
export interface DiskSessionOptions {
|
||||
workspaceId: string;
|
||||
syncFolder: string;
|
||||
}
|
||||
|
||||
export type DiskSyncEvent =
|
||||
| { type: 'ready' }
|
||||
| {
|
||||
type: 'doc-update';
|
||||
update: {
|
||||
docId: string;
|
||||
bin: Uint8Array;
|
||||
timestamp: Date;
|
||||
editor?: string;
|
||||
};
|
||||
origin?: string;
|
||||
}
|
||||
| { type: 'doc-delete'; docId: string; timestamp: Date }
|
||||
| { type: 'error'; message: string };
|
||||
|
||||
export interface DiskSyncApis {
|
||||
startSession: (
|
||||
sessionId: string,
|
||||
options: DiskSessionOptions
|
||||
) => Promise<void>;
|
||||
stopSession: (sessionId: string) => Promise<void>;
|
||||
applyLocalUpdate: (
|
||||
sessionId: string,
|
||||
update: DocUpdate,
|
||||
origin?: string
|
||||
) => Promise<DocClock>;
|
||||
subscribeEvents: (
|
||||
sessionId: string,
|
||||
callback: (event: DiskSyncEvent) => void
|
||||
) => () => void;
|
||||
}
|
||||
|
||||
interface DiskSyncOptions {
|
||||
readonly flavour: string;
|
||||
readonly type: SpaceType;
|
||||
readonly id: string;
|
||||
readonly syncFolder: string;
|
||||
}
|
||||
|
||||
interface DiskSyncApisWrapper {
|
||||
startSession: (options: DiskSessionOptions) => Promise<void>;
|
||||
stopSession: () => Promise<void>;
|
||||
applyLocalUpdate: (update: DocUpdate, origin?: string) => Promise<DocClock>;
|
||||
subscribeEvents: (callback: (event: DiskSyncEvent) => void) => () => void;
|
||||
}
|
||||
|
||||
let apis: DiskSyncApis | null = null;
|
||||
|
||||
export function bindDiskSyncApis(a: DiskSyncApis) {
|
||||
apis = a;
|
||||
}
|
||||
|
||||
export class DiskSyncConnection extends AutoReconnectConnection<{
|
||||
unsubscribe: () => void;
|
||||
}> {
|
||||
readonly apis: DiskSyncApisWrapper;
|
||||
readonly sessionId: string;
|
||||
|
||||
readonly flavour = this.options.flavour;
|
||||
readonly type = this.options.type;
|
||||
readonly id = this.options.id;
|
||||
|
||||
constructor(
|
||||
private readonly options: DiskSyncOptions,
|
||||
private readonly onEvent: (event: DiskSyncEvent) => void
|
||||
) {
|
||||
super();
|
||||
if (!apis) {
|
||||
throw new Error('Not in native context.');
|
||||
}
|
||||
this.sessionId = universalId({
|
||||
peer: this.flavour,
|
||||
type: this.type,
|
||||
id: this.id,
|
||||
});
|
||||
this.apis = this.wrapApis(apis);
|
||||
}
|
||||
|
||||
override get shareId(): string {
|
||||
return `disk:${this.sessionId}:${this.options.syncFolder}`;
|
||||
}
|
||||
|
||||
private wrapApis(originalApis: DiskSyncApis): DiskSyncApisWrapper {
|
||||
const sessionId = this.sessionId;
|
||||
return new Proxy(
|
||||
{},
|
||||
{
|
||||
get: (_target, key: keyof DiskSyncApisWrapper) => {
|
||||
const method = originalApis[key];
|
||||
return (...args: unknown[]) => {
|
||||
// oxlint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return (method as any)(sessionId, ...args);
|
||||
};
|
||||
},
|
||||
}
|
||||
) as DiskSyncApisWrapper;
|
||||
}
|
||||
|
||||
override async doConnect() {
|
||||
await this.apis.startSession({
|
||||
workspaceId: this.id,
|
||||
syncFolder: this.options.syncFolder,
|
||||
});
|
||||
const unsubscribe = this.apis.subscribeEvents(this.onEvent);
|
||||
return { unsubscribe };
|
||||
}
|
||||
|
||||
override doDisconnect(conn: { unsubscribe: () => void }) {
|
||||
try {
|
||||
conn.unsubscribe();
|
||||
} catch (error) {
|
||||
console.error('DiskSyncConnection unsubscribe failed', error);
|
||||
}
|
||||
this.apis.stopSession().catch(error => {
|
||||
console.error('DiskSyncConnection stopSession failed', error);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const PROJECT_ROOT = path.resolve(__dirname, '../../../../../../');
|
||||
|
||||
const JS_BOUNDARY_FILES = [
|
||||
path.join(PROJECT_ROOT, 'packages/common/nbstore/src/impls/disk/doc.ts'),
|
||||
path.join(
|
||||
PROJECT_ROOT,
|
||||
'packages/frontend/apps/electron/src/helper/disk-sync/handlers.ts'
|
||||
),
|
||||
];
|
||||
|
||||
const FORBIDDEN_PATTERNS = [
|
||||
/frontmatter/i,
|
||||
/gray-matter/i,
|
||||
/MarkdownAdapter/,
|
||||
/markdownToSnapshot/,
|
||||
/fromMarkdown/,
|
||||
/toMarkdown/,
|
||||
];
|
||||
|
||||
describe('disk boundary', () => {
|
||||
it('keeps markdown/frontmatter parsing out of JS adapter layer', () => {
|
||||
for (const file of JS_BOUNDARY_FILES) {
|
||||
const content = fs.readFileSync(file, 'utf-8');
|
||||
|
||||
for (const pattern of FORBIDDEN_PATTERNS) {
|
||||
expect(content).not.toMatch(pattern);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps JS layer focused on session orchestration APIs', () => {
|
||||
const adapter = fs.readFileSync(JS_BOUNDARY_FILES[0], 'utf-8');
|
||||
expect(adapter).toMatch(/applyLocalUpdate/);
|
||||
|
||||
const helper = fs.readFileSync(JS_BOUNDARY_FILES[1], 'utf-8');
|
||||
expect(helper).toMatch(/startSession/);
|
||||
expect(helper).toMatch(/stopSession/);
|
||||
expect(helper).toMatch(/applyLocalUpdate/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,451 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
applyUpdate,
|
||||
Array as YArray,
|
||||
Doc as YDoc,
|
||||
encodeStateAsUpdate,
|
||||
Map as YMap,
|
||||
} from 'yjs';
|
||||
|
||||
import { universalId } from '../../utils/universal-id';
|
||||
import { bindDiskSyncApis, type DiskSyncApis, type DiskSyncEvent } from './api';
|
||||
import { DiskDocStorage } from './doc';
|
||||
|
||||
function createUpdate(text: string): Uint8Array {
|
||||
const doc = new YDoc();
|
||||
doc.getText('content').insert(0, text);
|
||||
return encodeStateAsUpdate(doc);
|
||||
}
|
||||
|
||||
function createMapUpdate(entries: Record<string, string>): Uint8Array {
|
||||
const doc = new YDoc();
|
||||
const map = doc.getMap('test');
|
||||
for (const [key, value] of Object.entries(entries)) {
|
||||
map.set(key, value);
|
||||
}
|
||||
return encodeStateAsUpdate(doc);
|
||||
}
|
||||
|
||||
function createRootMetaUpdate(docIds: string[]): Uint8Array {
|
||||
const doc = new YDoc();
|
||||
const meta = doc.getMap('meta');
|
||||
const pages = new YArray<YMap<unknown>>();
|
||||
for (const docId of docIds) {
|
||||
const page = new YMap<unknown>();
|
||||
page.set('id', docId);
|
||||
pages.push([page]);
|
||||
}
|
||||
meta.set('pages', pages);
|
||||
return encodeStateAsUpdate(doc);
|
||||
}
|
||||
|
||||
describe('DiskDocStorage', () => {
|
||||
const sessionId = universalId({
|
||||
peer: 'local',
|
||||
type: 'workspace',
|
||||
id: 'workspace-test',
|
||||
});
|
||||
const listeners = new Map<string, Set<(event: DiskSyncEvent) => void>>();
|
||||
|
||||
const startSession = vi.fn(
|
||||
async (_sessionId: string, _options: { workspaceId: string }) => {}
|
||||
);
|
||||
const stopSession = vi.fn(async (_sessionId: string) => {});
|
||||
const applyLocalUpdate = vi.fn(
|
||||
async (_sessionId: string, update: { docId: string }) => {
|
||||
return {
|
||||
docId: update.docId,
|
||||
timestamp: new Date('2026-01-02T00:00:00.000Z'),
|
||||
};
|
||||
}
|
||||
);
|
||||
const subscribeEvents = vi.fn(
|
||||
(currentSessionId: string, callback: (event: DiskSyncEvent) => void) => {
|
||||
let set = listeners.get(currentSessionId);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
listeners.set(currentSessionId, set);
|
||||
}
|
||||
set.add(callback);
|
||||
return () => {
|
||||
set?.delete(callback);
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
const apis: DiskSyncApis = {
|
||||
startSession,
|
||||
stopSession,
|
||||
applyLocalUpdate,
|
||||
subscribeEvents,
|
||||
};
|
||||
|
||||
function emit(event: DiskSyncEvent) {
|
||||
const callbacks = listeners.get(sessionId);
|
||||
for (const callback of callbacks ?? []) {
|
||||
callback(event);
|
||||
}
|
||||
}
|
||||
|
||||
function createStorage() {
|
||||
return new DiskDocStorage({
|
||||
flavour: 'local',
|
||||
type: 'workspace',
|
||||
id: 'workspace-test',
|
||||
syncFolder: '/tmp/sync',
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
bindDiskSyncApis(apis);
|
||||
listeners.clear();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
listeners.clear();
|
||||
});
|
||||
|
||||
it('starts and stops disk session with connection lifecycle', async () => {
|
||||
const storage = createStorage();
|
||||
storage.connection.connect();
|
||||
await storage.connection.waitForConnected();
|
||||
|
||||
expect(startSession).toHaveBeenCalledWith(sessionId, {
|
||||
workspaceId: 'workspace-test',
|
||||
syncFolder: '/tmp/sync',
|
||||
});
|
||||
|
||||
storage.connection.disconnect();
|
||||
await vi.waitFor(() => {
|
||||
expect(stopSession).toHaveBeenCalledWith(sessionId);
|
||||
});
|
||||
});
|
||||
|
||||
it('forwards local updates and emits doc update events', async () => {
|
||||
const storage = createStorage();
|
||||
storage.connection.connect();
|
||||
await storage.connection.waitForConnected();
|
||||
|
||||
const seen: Array<{ docId: string; origin?: string }> = [];
|
||||
const unsubscribe = storage.subscribeDocUpdate((update, origin) => {
|
||||
seen.push({ docId: update.docId, origin });
|
||||
});
|
||||
|
||||
const bin = createUpdate('local');
|
||||
await storage.pushDocUpdate({ docId: 'doc-local', bin }, 'origin:local');
|
||||
|
||||
expect(applyLocalUpdate).toHaveBeenCalledWith(
|
||||
sessionId,
|
||||
expect.objectContaining({
|
||||
docId: 'doc-local',
|
||||
}),
|
||||
'origin:local'
|
||||
);
|
||||
expect(seen).toEqual([{ docId: 'doc-local', origin: 'origin:local' }]);
|
||||
|
||||
const snapshot = await storage.getDoc('doc-local');
|
||||
expect(snapshot?.docId).toBe('doc-local');
|
||||
expect(snapshot?.timestamp.toISOString()).toBe('2026-01-02T00:00:00.000Z');
|
||||
|
||||
unsubscribe();
|
||||
storage.connection.disconnect();
|
||||
});
|
||||
|
||||
it('applies remote events into local snapshots and handles delete events', async () => {
|
||||
const storage = createStorage();
|
||||
storage.connection.connect();
|
||||
await storage.connection.waitForConnected();
|
||||
|
||||
emit({
|
||||
type: 'doc-update',
|
||||
update: {
|
||||
docId: 'doc-remote',
|
||||
bin: createUpdate('remote'),
|
||||
timestamp: new Date('2026-01-03T00:00:00.000Z'),
|
||||
},
|
||||
});
|
||||
|
||||
await vi.waitFor(async () => {
|
||||
const snapshot = await storage.getDoc('doc-remote');
|
||||
expect(snapshot?.docId).toBe('doc-remote');
|
||||
});
|
||||
|
||||
const timestamps = await storage.getDocTimestamps();
|
||||
expect(timestamps['doc-remote']?.toISOString()).toBe(
|
||||
'2026-01-03T00:00:00.000Z'
|
||||
);
|
||||
|
||||
emit({
|
||||
type: 'doc-delete',
|
||||
docId: 'doc-remote',
|
||||
timestamp: new Date('2026-01-03T00:00:01.000Z'),
|
||||
});
|
||||
|
||||
await vi.waitFor(async () => {
|
||||
expect(await storage.getDoc('doc-remote')).toBeNull();
|
||||
});
|
||||
|
||||
storage.connection.disconnect();
|
||||
});
|
||||
|
||||
it('serializes concurrent remote doc-update merges for the same doc', async () => {
|
||||
const storage = createStorage();
|
||||
storage.connection.connect();
|
||||
await storage.connection.waitForConnected();
|
||||
|
||||
const originalMergeUpdates = (
|
||||
storage as unknown as {
|
||||
mergeUpdates: (updates: Uint8Array[]) => Promise<Uint8Array>;
|
||||
}
|
||||
).mergeUpdates.bind(storage);
|
||||
|
||||
let mergeCall = 0;
|
||||
vi.spyOn(
|
||||
storage as unknown as {
|
||||
mergeUpdates: (updates: Uint8Array[]) => Promise<Uint8Array>;
|
||||
},
|
||||
'mergeUpdates'
|
||||
).mockImplementation(async updates => {
|
||||
mergeCall += 1;
|
||||
// Force two in-flight merge operations to overlap and complete out-of-order.
|
||||
if (mergeCall === 1) {
|
||||
await new Promise(resolve => setTimeout(resolve, 20));
|
||||
}
|
||||
return originalMergeUpdates(updates);
|
||||
});
|
||||
|
||||
emit({
|
||||
type: 'doc-update',
|
||||
update: {
|
||||
docId: 'doc-race',
|
||||
bin: createMapUpdate({ first: '1' }),
|
||||
timestamp: new Date('2026-01-03T00:00:00.000Z'),
|
||||
},
|
||||
});
|
||||
emit({
|
||||
type: 'doc-update',
|
||||
update: {
|
||||
docId: 'doc-race',
|
||||
bin: createMapUpdate({ second: '2' }),
|
||||
timestamp: new Date('2026-01-03T00:00:00.001Z'),
|
||||
},
|
||||
});
|
||||
|
||||
await vi.waitFor(async () => {
|
||||
const snapshot = await storage.getDoc('doc-race');
|
||||
expect(snapshot).not.toBeNull();
|
||||
expect(snapshot!.timestamp.toISOString()).toBe(
|
||||
'2026-01-03T00:00:00.001Z'
|
||||
);
|
||||
|
||||
const doc = new YDoc();
|
||||
applyUpdate(doc, snapshot!.bin);
|
||||
expect(doc.getMap('test').toJSON()).toEqual({
|
||||
first: '1',
|
||||
second: '2',
|
||||
});
|
||||
});
|
||||
|
||||
storage.connection.disconnect();
|
||||
});
|
||||
|
||||
it('does not block follow-up updates when snapshot merge fails once', async () => {
|
||||
const storage = createStorage();
|
||||
storage.connection.connect();
|
||||
await storage.connection.waitForConnected();
|
||||
|
||||
const originalMergeUpdates = (
|
||||
storage as unknown as {
|
||||
mergeUpdates: (updates: Uint8Array[]) => Promise<Uint8Array>;
|
||||
}
|
||||
).mergeUpdates.bind(storage);
|
||||
|
||||
let mergeCall = 0;
|
||||
vi.spyOn(
|
||||
storage as unknown as {
|
||||
mergeUpdates: (updates: Uint8Array[]) => Promise<Uint8Array>;
|
||||
},
|
||||
'mergeUpdates'
|
||||
).mockImplementation(async updates => {
|
||||
mergeCall += 1;
|
||||
if (mergeCall === 1) {
|
||||
throw new Error('merge failed once');
|
||||
}
|
||||
return originalMergeUpdates(updates);
|
||||
});
|
||||
|
||||
await expect(
|
||||
storage.pushDocUpdate({
|
||||
docId: 'doc-merge-fallback',
|
||||
bin: createMapUpdate({ a: '1' }),
|
||||
})
|
||||
).resolves.toEqual({
|
||||
docId: 'doc-merge-fallback',
|
||||
timestamp: new Date('2026-01-02T00:00:00.000Z'),
|
||||
});
|
||||
|
||||
// This update triggers the mocked merge failure, but should still resolve.
|
||||
await expect(
|
||||
storage.pushDocUpdate({
|
||||
docId: 'doc-merge-fallback',
|
||||
bin: createMapUpdate({ b: '2' }),
|
||||
})
|
||||
).resolves.toEqual({
|
||||
docId: 'doc-merge-fallback',
|
||||
timestamp: new Date('2026-01-02T00:00:00.000Z'),
|
||||
});
|
||||
|
||||
// Follow-up update should continue to work without requiring reconnect/reload.
|
||||
await expect(
|
||||
storage.pushDocUpdate({
|
||||
docId: 'doc-merge-fallback',
|
||||
bin: createMapUpdate({ c: '3' }),
|
||||
})
|
||||
).resolves.toEqual({
|
||||
docId: 'doc-merge-fallback',
|
||||
timestamp: new Date('2026-01-02T00:00:00.000Z'),
|
||||
});
|
||||
|
||||
const snapshot = await storage.getDoc('doc-merge-fallback');
|
||||
expect(snapshot).not.toBeNull();
|
||||
const doc = new YDoc();
|
||||
applyUpdate(doc, snapshot!.bin);
|
||||
const data = doc.getMap('test').toJSON();
|
||||
expect(data).toMatchObject({
|
||||
b: '2',
|
||||
c: '3',
|
||||
});
|
||||
|
||||
storage.connection.disconnect();
|
||||
});
|
||||
|
||||
it('accepts remote doc-update bins as number[] (from native binding)', async () => {
|
||||
const storage = createStorage();
|
||||
storage.connection.connect();
|
||||
await storage.connection.waitForConnected();
|
||||
|
||||
const original = createUpdate('remote-array');
|
||||
const bin = Array.from(original) as unknown as Uint8Array;
|
||||
|
||||
emit({
|
||||
type: 'doc-update',
|
||||
update: {
|
||||
docId: 'doc-remote-array',
|
||||
bin,
|
||||
timestamp: new Date('2026-01-03T00:00:00.000Z'),
|
||||
},
|
||||
});
|
||||
|
||||
await vi.waitFor(async () => {
|
||||
const snapshot = await storage.getDoc('doc-remote-array');
|
||||
expect(snapshot).not.toBeNull();
|
||||
|
||||
const doc = new YDoc();
|
||||
applyUpdate(doc, snapshot!.bin);
|
||||
expect(doc.getText('content').toString()).toBe('remote-array');
|
||||
});
|
||||
|
||||
storage.connection.disconnect();
|
||||
});
|
||||
|
||||
it('throws when applyLocalUpdate returns invalid timestamp', async () => {
|
||||
applyLocalUpdate.mockResolvedValueOnce({
|
||||
docId: 'doc-invalid-clock',
|
||||
timestamp: new Date('invalid'),
|
||||
});
|
||||
|
||||
const storage = createStorage();
|
||||
storage.connection.connect();
|
||||
await storage.connection.waitForConnected();
|
||||
|
||||
await expect(
|
||||
storage.pushDocUpdate({
|
||||
docId: 'doc-invalid-clock',
|
||||
bin: createUpdate('invalid'),
|
||||
})
|
||||
).rejects.toThrow('[disk] invalid timestamp');
|
||||
|
||||
storage.connection.disconnect();
|
||||
});
|
||||
|
||||
it('skips remote doc-update with invalid timestamp', async () => {
|
||||
const storage = createStorage();
|
||||
storage.connection.connect();
|
||||
await storage.connection.waitForConnected();
|
||||
|
||||
emit({
|
||||
type: 'doc-update',
|
||||
update: {
|
||||
docId: 'doc-invalid-remote-clock',
|
||||
bin: createUpdate('remote-invalid'),
|
||||
timestamp: new Date('invalid') as unknown as Date,
|
||||
},
|
||||
});
|
||||
|
||||
await vi.waitFor(async () => {
|
||||
expect(await storage.getDoc('doc-invalid-remote-clock')).toBeNull();
|
||||
});
|
||||
|
||||
storage.connection.disconnect();
|
||||
});
|
||||
|
||||
it('discovers doc ids from root meta and emits connect-driving updates once', async () => {
|
||||
const storage = createStorage();
|
||||
storage.connection.connect();
|
||||
await storage.connection.waitForConnected();
|
||||
|
||||
const seen: Array<{ docId: string; origin?: string; size: number }> = [];
|
||||
const unsubscribe = storage.subscribeDocUpdate((update, origin) => {
|
||||
seen.push({
|
||||
docId: update.docId,
|
||||
origin,
|
||||
size: update.bin.byteLength,
|
||||
});
|
||||
});
|
||||
|
||||
const rootUpdate = createRootMetaUpdate(['doc-a', 'doc-b']);
|
||||
|
||||
await storage.pushDocUpdate(
|
||||
{
|
||||
docId: 'workspace-test',
|
||||
bin: rootUpdate,
|
||||
},
|
||||
'origin:root'
|
||||
);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
const discovered = seen.filter(
|
||||
item => item.origin === 'disk:root-meta-discovery'
|
||||
);
|
||||
expect(discovered).toHaveLength(2);
|
||||
});
|
||||
|
||||
const discoveredDocIds = seen
|
||||
.filter(item => item.origin === 'disk:root-meta-discovery')
|
||||
.map(item => item.docId)
|
||||
.sort();
|
||||
expect(discoveredDocIds).toEqual(['doc-a', 'doc-b']);
|
||||
expect(
|
||||
seen
|
||||
.filter(item => item.origin === 'disk:root-meta-discovery')
|
||||
.every(item => item.size === 0)
|
||||
).toBe(true);
|
||||
|
||||
await storage.pushDocUpdate(
|
||||
{
|
||||
docId: 'workspace-test',
|
||||
bin: rootUpdate,
|
||||
},
|
||||
'origin:root'
|
||||
);
|
||||
|
||||
const discoveryCountAfterSecondPush = seen.filter(
|
||||
item => item.origin === 'disk:root-meta-discovery'
|
||||
).length;
|
||||
expect(discoveryCountAfterSecondPush).toBe(2);
|
||||
|
||||
unsubscribe();
|
||||
storage.connection.disconnect();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,291 @@
|
||||
import { applyUpdate, Doc as YDoc } from 'yjs';
|
||||
|
||||
import {
|
||||
type DocClock,
|
||||
type DocClocks,
|
||||
type DocRecord,
|
||||
DocStorageBase,
|
||||
type DocUpdate,
|
||||
} from '../../storage';
|
||||
import { type SpaceType } from '../../utils/universal-id';
|
||||
import { DiskSyncConnection, type DiskSyncEvent } from './api';
|
||||
|
||||
export interface DiskDocStorageOptions {
|
||||
readonly flavour: string;
|
||||
readonly type: SpaceType;
|
||||
readonly id: string;
|
||||
readonly syncFolder: string;
|
||||
}
|
||||
|
||||
export class DiskDocStorage extends DocStorageBase<DiskDocStorageOptions> {
|
||||
static readonly identifier = 'DiskDocStorage';
|
||||
|
||||
readonly connection: DiskSyncConnection;
|
||||
|
||||
private readonly snapshots = new Map<string, DocRecord>();
|
||||
private readonly pendingUpdates = new Map<string, DocRecord[]>();
|
||||
private readonly discoveredRootDocs = new Set<string>();
|
||||
|
||||
constructor(options: DiskDocStorageOptions) {
|
||||
super(options);
|
||||
this.connection = new DiskSyncConnection(options, this.handleDiskEvent);
|
||||
}
|
||||
|
||||
override async pushDocUpdate(update: DocUpdate, origin?: string) {
|
||||
const { timestamp } = await this.connection.apis.applyLocalUpdate(
|
||||
update,
|
||||
origin
|
||||
);
|
||||
const clock = normalizeDate(timestamp);
|
||||
const next: DocRecord = {
|
||||
docId: update.docId,
|
||||
bin: update.bin,
|
||||
timestamp: clock,
|
||||
editor: update.editor,
|
||||
};
|
||||
await this.applySnapshotUpdate(next, origin);
|
||||
return { docId: update.docId, timestamp: clock };
|
||||
}
|
||||
|
||||
override async getDocTimestamp(docId: string): Promise<DocClock | null> {
|
||||
const snapshot = this.snapshots.get(docId);
|
||||
if (!snapshot) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
docId,
|
||||
timestamp: snapshot.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
override async getDocTimestamps(after?: Date): Promise<DocClocks> {
|
||||
const timestamps: DocClocks = {};
|
||||
for (const [docId, snapshot] of this.snapshots.entries()) {
|
||||
if (after && snapshot.timestamp.getTime() <= after.getTime()) {
|
||||
continue;
|
||||
}
|
||||
timestamps[docId] = snapshot.timestamp;
|
||||
}
|
||||
return timestamps;
|
||||
}
|
||||
|
||||
override async deleteDoc(docId: string): Promise<void> {
|
||||
this.snapshots.delete(docId);
|
||||
this.pendingUpdates.delete(docId);
|
||||
}
|
||||
|
||||
protected override async getDocSnapshot(docId: string) {
|
||||
return this.snapshots.get(docId) ?? null;
|
||||
}
|
||||
|
||||
protected override async setDocSnapshot(
|
||||
snapshot: DocRecord
|
||||
): Promise<boolean> {
|
||||
const existing = this.snapshots.get(snapshot.docId);
|
||||
if (
|
||||
existing &&
|
||||
existing.timestamp.getTime() > snapshot.timestamp.getTime()
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
this.snapshots.set(snapshot.docId, snapshot);
|
||||
return true;
|
||||
}
|
||||
|
||||
protected override async getDocUpdates(docId: string): Promise<DocRecord[]> {
|
||||
return this.pendingUpdates.get(docId) ?? [];
|
||||
}
|
||||
|
||||
protected override async markUpdatesMerged(
|
||||
docId: string,
|
||||
updates: DocRecord[]
|
||||
): Promise<number> {
|
||||
if (updates.length) {
|
||||
this.pendingUpdates.delete(docId);
|
||||
}
|
||||
return updates.length;
|
||||
}
|
||||
|
||||
private readonly handleDiskEvent = (event: DiskSyncEvent) => {
|
||||
switch (event.type) {
|
||||
case 'doc-update': {
|
||||
let timestamp: Date;
|
||||
try {
|
||||
timestamp = normalizeDate(event.update.timestamp);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[disk] invalid doc-update timestamp, skip event',
|
||||
error
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let bin: Uint8Array;
|
||||
try {
|
||||
bin = normalizeBin(event.update.bin);
|
||||
} catch (error) {
|
||||
console.warn('[disk] invalid doc-update bin, skip event', error);
|
||||
return;
|
||||
}
|
||||
|
||||
const update: DocRecord = {
|
||||
docId: event.update.docId,
|
||||
bin,
|
||||
timestamp,
|
||||
editor: event.update.editor,
|
||||
};
|
||||
void this.applySnapshotUpdate(update, event.origin).catch(error => {
|
||||
console.warn(
|
||||
'[disk] failed to apply remote doc-update, skip event',
|
||||
error
|
||||
);
|
||||
});
|
||||
return;
|
||||
}
|
||||
case 'doc-delete': {
|
||||
this.snapshots.delete(event.docId);
|
||||
this.pendingUpdates.delete(event.docId);
|
||||
return;
|
||||
}
|
||||
case 'error': {
|
||||
console.warn('[disk] session error', event.message);
|
||||
return;
|
||||
}
|
||||
default: {
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
private async applySnapshotUpdate(update: DocRecord, origin?: string) {
|
||||
await using _lock = await this.lockDocForUpdate(update.docId);
|
||||
try {
|
||||
await this.mergeIntoSnapshot(update);
|
||||
} catch (error) {
|
||||
// Snapshot cache is best-effort. A merge failure must not block upstream sync
|
||||
// forever (it can otherwise require a full app reload to recover).
|
||||
console.warn(
|
||||
'[disk] snapshot merge failed, reset in-memory snapshot cache',
|
||||
error
|
||||
);
|
||||
this.snapshots.set(update.docId, update);
|
||||
}
|
||||
this.emit('update', update, origin);
|
||||
if (update.docId === this.spaceId) {
|
||||
this.emitRootMetaDiscoveryUpdates();
|
||||
}
|
||||
}
|
||||
|
||||
private async mergeIntoSnapshot(update: DocRecord) {
|
||||
const current = this.snapshots.get(update.docId);
|
||||
if (!current) {
|
||||
this.snapshots.set(update.docId, update);
|
||||
return;
|
||||
}
|
||||
|
||||
const merged = await this.mergeUpdates([current.bin, update.bin]);
|
||||
this.snapshots.set(update.docId, {
|
||||
...update,
|
||||
bin: merged,
|
||||
timestamp:
|
||||
current.timestamp.getTime() > update.timestamp.getTime()
|
||||
? current.timestamp
|
||||
: update.timestamp,
|
||||
editor: update.editor ?? current.editor,
|
||||
});
|
||||
}
|
||||
|
||||
private emitRootMetaDiscoveryUpdates() {
|
||||
const rootSnapshot = this.snapshots.get(this.spaceId);
|
||||
if (!rootSnapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
const docIds = extractRootMetaDocIds(rootSnapshot.bin);
|
||||
// These discovery events are only meant to "introduce" doc ids to the sync
|
||||
// peer, so it can connect/pull/push them. They should NOT be treated as a
|
||||
// remote clock; otherwise switching sync folders (remote empty) can be
|
||||
// incorrectly seen as "remote newer than local" and skip the initial push.
|
||||
const discoveryTimestamp = new Date(0);
|
||||
for (const docId of docIds) {
|
||||
if (docId === this.spaceId || this.discoveredRootDocs.has(docId)) {
|
||||
continue;
|
||||
}
|
||||
this.discoveredRootDocs.add(docId);
|
||||
this.emit(
|
||||
'update',
|
||||
{
|
||||
docId,
|
||||
bin: new Uint8Array(),
|
||||
timestamp: discoveryTimestamp,
|
||||
},
|
||||
'disk:root-meta-discovery'
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeDate(date: Date | string | number): Date {
|
||||
const normalized = date instanceof Date ? date : new Date(date);
|
||||
if (Number.isNaN(normalized.getTime())) {
|
||||
throw new Error(`[disk] invalid timestamp: ${String(date)}`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function extractRootMetaDocIds(rootBin: Uint8Array): string[] {
|
||||
const doc = new YDoc();
|
||||
try {
|
||||
applyUpdate(doc, rootBin);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
const meta = doc.getMap<unknown>('meta');
|
||||
const pages = meta.get('pages');
|
||||
const pagesJson =
|
||||
typeof pages === 'object' &&
|
||||
pages !== null &&
|
||||
'toJSON' in pages &&
|
||||
typeof pages.toJSON === 'function'
|
||||
? pages.toJSON()
|
||||
: pages;
|
||||
|
||||
if (!Array.isArray(pagesJson)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const docIds: string[] = [];
|
||||
for (const page of pagesJson) {
|
||||
if (!page || typeof page !== 'object') {
|
||||
continue;
|
||||
}
|
||||
const id = (page as { id?: unknown }).id;
|
||||
if (typeof id === 'string' && id.length > 0) {
|
||||
docIds.push(id);
|
||||
}
|
||||
}
|
||||
return docIds;
|
||||
}
|
||||
|
||||
function normalizeBin(bin: unknown): Uint8Array {
|
||||
// Native NAPI binding may send `number[]` for `Vec<u8>` fields.
|
||||
if (bin instanceof Uint8Array) {
|
||||
return bin;
|
||||
}
|
||||
if (Array.isArray(bin)) {
|
||||
return Uint8Array.from(bin);
|
||||
}
|
||||
// Some transports may serialize Buffer as `{ type: 'Buffer', data: number[] }`.
|
||||
if (
|
||||
bin &&
|
||||
typeof bin === 'object' &&
|
||||
'data' in bin &&
|
||||
Array.isArray((bin as { data?: unknown }).data)
|
||||
) {
|
||||
return Uint8Array.from((bin as { data: number[] }).data);
|
||||
}
|
||||
throw new Error(
|
||||
`[disk] invalid update bin type: ${Object.prototype.toString.call(bin)}`
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import type { StorageConstructor } from '..';
|
||||
import { DiskDocStorage } from './doc';
|
||||
|
||||
export * from './api';
|
||||
export * from './doc';
|
||||
|
||||
export const diskStorages = [DiskDocStorage] satisfies StorageConstructor[];
|
||||
@@ -0,0 +1,378 @@
|
||||
import 'fake-indexeddb/auto';
|
||||
|
||||
import { expect, test, vi } from 'vitest';
|
||||
import { Doc as YDoc, encodeStateAsUpdate } from 'yjs';
|
||||
|
||||
import { expectYjsEqual } from '../../__tests__/utils';
|
||||
import { SpaceStorage } from '../../storage';
|
||||
import { Sync } from '../../sync';
|
||||
import { universalId } from '../../utils/universal-id';
|
||||
import { IndexedDBDocStorage, IndexedDBDocSyncStorage } from '../idb';
|
||||
import { bindDiskSyncApis, type DiskSyncApis, type DiskSyncEvent } from './api';
|
||||
import { DiskDocStorage } from './doc';
|
||||
|
||||
test('sync local <-> disk remote updates through DocSyncPeer', async () => {
|
||||
const workspaceId = 'ws-disk-integration';
|
||||
const sessionId = universalId({
|
||||
peer: 'local',
|
||||
type: 'workspace',
|
||||
id: workspaceId,
|
||||
});
|
||||
|
||||
const listeners = new Map<string, Set<(event: DiskSyncEvent) => void>>();
|
||||
const remoteDocs = new Map<string, { timestamp: Date; bin: Uint8Array }>();
|
||||
|
||||
const apis: DiskSyncApis = {
|
||||
startSession: async currentSessionId => {
|
||||
if (!listeners.has(currentSessionId)) {
|
||||
listeners.set(currentSessionId, new Set());
|
||||
}
|
||||
},
|
||||
stopSession: async currentSessionId => {
|
||||
listeners.delete(currentSessionId);
|
||||
},
|
||||
applyLocalUpdate: async (currentSessionId, update) => {
|
||||
const timestamp = new Date();
|
||||
remoteDocs.set(update.docId, { timestamp, bin: update.bin });
|
||||
for (const callback of listeners.get(currentSessionId) ?? []) {
|
||||
callback({
|
||||
type: 'doc-update',
|
||||
update: {
|
||||
docId: update.docId,
|
||||
bin: update.bin,
|
||||
timestamp,
|
||||
},
|
||||
origin: 'sync:disk-mock',
|
||||
});
|
||||
}
|
||||
return {
|
||||
docId: update.docId,
|
||||
timestamp,
|
||||
};
|
||||
},
|
||||
subscribeEvents: (currentSessionId, callback) => {
|
||||
let set = listeners.get(currentSessionId);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
listeners.set(currentSessionId, set);
|
||||
}
|
||||
set.add(callback);
|
||||
return () => {
|
||||
set?.delete(callback);
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
bindDiskSyncApis(apis);
|
||||
|
||||
const localDoc = new IndexedDBDocStorage({
|
||||
id: workspaceId,
|
||||
flavour: 'local',
|
||||
type: 'workspace',
|
||||
});
|
||||
const localDocSync = new IndexedDBDocSyncStorage({
|
||||
id: workspaceId,
|
||||
flavour: 'local',
|
||||
type: 'workspace',
|
||||
});
|
||||
const remoteDoc = new DiskDocStorage({
|
||||
id: workspaceId,
|
||||
flavour: 'local',
|
||||
type: 'workspace',
|
||||
syncFolder: '/tmp/disk-sync',
|
||||
});
|
||||
|
||||
const local = new SpaceStorage({
|
||||
doc: localDoc,
|
||||
docSync: localDocSync,
|
||||
});
|
||||
const remote = new SpaceStorage({
|
||||
doc: remoteDoc,
|
||||
});
|
||||
|
||||
local.connect();
|
||||
remote.connect();
|
||||
await local.waitForConnected();
|
||||
await remote.waitForConnected();
|
||||
|
||||
const sync = new Sync({
|
||||
local,
|
||||
remotes: {
|
||||
disk: remote,
|
||||
},
|
||||
});
|
||||
sync.start();
|
||||
|
||||
const localSource = new YDoc();
|
||||
localSource.getMap('test').set('origin', 'local');
|
||||
await localDoc.pushDocUpdate({
|
||||
docId: 'doc-local',
|
||||
bin: encodeStateAsUpdate(localSource),
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(remoteDocs.has('doc-local')).toBe(true);
|
||||
});
|
||||
|
||||
const remoteSource = new YDoc();
|
||||
remoteSource.getMap('test').set('origin', 'remote');
|
||||
remoteSource.getMap('test').set('synced', 'yes');
|
||||
const remoteUpdate = encodeStateAsUpdate(remoteSource);
|
||||
const remoteTimestamp = new Date('2026-01-05T00:00:00.000Z');
|
||||
for (const callback of listeners.get(sessionId) ?? []) {
|
||||
callback({
|
||||
type: 'doc-update',
|
||||
update: {
|
||||
docId: 'doc-remote',
|
||||
bin: remoteUpdate,
|
||||
timestamp: remoteTimestamp,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
await vi.waitFor(async () => {
|
||||
const doc = await localDoc.getDoc('doc-remote');
|
||||
expect(doc).not.toBeNull();
|
||||
expectYjsEqual(doc!.bin, {
|
||||
test: {
|
||||
origin: 'remote',
|
||||
synced: 'yes',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
sync.stop();
|
||||
// Intentionally keep IndexedDB connections open in tests. Disconnecting can
|
||||
// abort in-flight IDB transactions in fake-indexeddb and surface as unhandled
|
||||
// rejections, which makes Vitest fail the run.
|
||||
remote.disconnect();
|
||||
});
|
||||
|
||||
test('forces initial push when disk has stale pushed clocks but remote is empty', async () => {
|
||||
const workspaceId = 'ws-disk-stale-push';
|
||||
|
||||
const listeners = new Map<string, Set<(event: DiskSyncEvent) => void>>();
|
||||
const remoteDocs = new Map<string, { timestamp: Date; bin: Uint8Array }>();
|
||||
|
||||
const apis: DiskSyncApis = {
|
||||
startSession: async currentSessionId => {
|
||||
if (!listeners.has(currentSessionId)) {
|
||||
listeners.set(currentSessionId, new Set());
|
||||
}
|
||||
},
|
||||
stopSession: async currentSessionId => {
|
||||
listeners.delete(currentSessionId);
|
||||
},
|
||||
applyLocalUpdate: async (currentSessionId, update) => {
|
||||
const timestamp = new Date();
|
||||
remoteDocs.set(update.docId, { timestamp, bin: update.bin });
|
||||
for (const callback of listeners.get(currentSessionId) ?? []) {
|
||||
callback({
|
||||
type: 'doc-update',
|
||||
update: {
|
||||
docId: update.docId,
|
||||
bin: update.bin,
|
||||
timestamp,
|
||||
},
|
||||
origin: 'sync:disk-mock',
|
||||
});
|
||||
}
|
||||
return {
|
||||
docId: update.docId,
|
||||
timestamp,
|
||||
};
|
||||
},
|
||||
subscribeEvents: (currentSessionId, callback) => {
|
||||
let set = listeners.get(currentSessionId);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
listeners.set(currentSessionId, set);
|
||||
}
|
||||
set.add(callback);
|
||||
return () => {
|
||||
set?.delete(callback);
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
bindDiskSyncApis(apis);
|
||||
|
||||
const localDoc = new IndexedDBDocStorage({
|
||||
id: workspaceId,
|
||||
flavour: 'local',
|
||||
type: 'workspace',
|
||||
});
|
||||
const localDocSync = new IndexedDBDocSyncStorage({
|
||||
id: workspaceId,
|
||||
flavour: 'local',
|
||||
type: 'workspace',
|
||||
});
|
||||
const remoteDoc = new DiskDocStorage({
|
||||
id: workspaceId,
|
||||
flavour: 'local',
|
||||
type: 'workspace',
|
||||
syncFolder: '/tmp/disk-sync-stale',
|
||||
});
|
||||
|
||||
const local = new SpaceStorage({
|
||||
doc: localDoc,
|
||||
docSync: localDocSync,
|
||||
});
|
||||
const remote = new SpaceStorage({
|
||||
doc: remoteDoc,
|
||||
});
|
||||
|
||||
local.connect();
|
||||
remote.connect();
|
||||
await local.waitForConnected();
|
||||
await remote.waitForConnected();
|
||||
|
||||
const source = new YDoc();
|
||||
source.getMap('test').set('value', 'local');
|
||||
await localDoc.pushDocUpdate({
|
||||
docId: 'doc-local-stale',
|
||||
bin: encodeStateAsUpdate(source),
|
||||
});
|
||||
|
||||
await localDocSync.setPeerPushedClock('disk', {
|
||||
docId: 'doc-local-stale',
|
||||
timestamp: new Date('2099-01-01T00:00:00.000Z'),
|
||||
});
|
||||
|
||||
const sync = new Sync({
|
||||
local,
|
||||
remotes: {
|
||||
disk: remote,
|
||||
},
|
||||
});
|
||||
sync.start();
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(remoteDocs.has('doc-local-stale')).toBe(true);
|
||||
});
|
||||
|
||||
sync.stop();
|
||||
remote.disconnect();
|
||||
});
|
||||
|
||||
test('root-meta discovery must not block pushing page docs when switching disk folders', async () => {
|
||||
const workspaceId = 'ws-disk-discovery-nonblocking';
|
||||
|
||||
const pageDocId = 'page-doc-1';
|
||||
|
||||
const listeners = new Map<string, Set<(event: DiskSyncEvent) => void>>();
|
||||
const remoteDocs = new Map<string, { timestamp: Date; bin: Uint8Array }>();
|
||||
|
||||
const apis: DiskSyncApis = {
|
||||
startSession: async currentSessionId => {
|
||||
if (!listeners.has(currentSessionId)) {
|
||||
listeners.set(currentSessionId, new Set());
|
||||
}
|
||||
},
|
||||
stopSession: async currentSessionId => {
|
||||
listeners.delete(currentSessionId);
|
||||
},
|
||||
applyLocalUpdate: async (currentSessionId, update) => {
|
||||
const timestamp = new Date();
|
||||
remoteDocs.set(update.docId, { timestamp, bin: update.bin });
|
||||
for (const callback of listeners.get(currentSessionId) ?? []) {
|
||||
callback({
|
||||
type: 'doc-update',
|
||||
update: {
|
||||
docId: update.docId,
|
||||
bin: update.bin,
|
||||
timestamp,
|
||||
},
|
||||
origin: 'sync:disk-mock',
|
||||
});
|
||||
}
|
||||
return {
|
||||
docId: update.docId,
|
||||
timestamp,
|
||||
};
|
||||
},
|
||||
subscribeEvents: (currentSessionId, callback) => {
|
||||
let set = listeners.get(currentSessionId);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
listeners.set(currentSessionId, set);
|
||||
}
|
||||
set.add(callback);
|
||||
return () => {
|
||||
set?.delete(callback);
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
bindDiskSyncApis(apis);
|
||||
|
||||
const localDoc = new IndexedDBDocStorage({
|
||||
id: workspaceId,
|
||||
flavour: 'local',
|
||||
type: 'workspace',
|
||||
});
|
||||
const localDocSync = new IndexedDBDocSyncStorage({
|
||||
id: workspaceId,
|
||||
flavour: 'local',
|
||||
type: 'workspace',
|
||||
});
|
||||
const remoteDoc = new DiskDocStorage({
|
||||
id: workspaceId,
|
||||
flavour: 'local',
|
||||
type: 'workspace',
|
||||
syncFolder: '/tmp/disk-sync-discovery',
|
||||
});
|
||||
|
||||
const local = new SpaceStorage({
|
||||
doc: localDoc,
|
||||
docSync: localDocSync,
|
||||
});
|
||||
const remote = new SpaceStorage({
|
||||
doc: remoteDoc,
|
||||
});
|
||||
|
||||
local.connect();
|
||||
remote.connect();
|
||||
await local.waitForConnected();
|
||||
await remote.waitForConnected();
|
||||
|
||||
// Seed local root meta so disk can discover the page doc id from it.
|
||||
const root = new YDoc();
|
||||
const meta = root.getMap('meta');
|
||||
meta.set('pages', [{ id: pageDocId }]);
|
||||
await localDoc.pushDocUpdate({
|
||||
docId: workspaceId,
|
||||
bin: encodeStateAsUpdate(root),
|
||||
});
|
||||
|
||||
// Seed the page doc itself.
|
||||
const page = new YDoc();
|
||||
page.getMap('test').set('value', 'local');
|
||||
const { timestamp: pageClock } = await localDoc.pushDocUpdate({
|
||||
docId: pageDocId,
|
||||
bin: encodeStateAsUpdate(page),
|
||||
});
|
||||
|
||||
// Simulate "already pushed" clocks from a previous disk folder.
|
||||
await localDocSync.setPeerPushedClock('disk', {
|
||||
docId: pageDocId,
|
||||
timestamp: pageClock,
|
||||
});
|
||||
|
||||
const sync = new Sync({
|
||||
local,
|
||||
remotes: {
|
||||
disk: remote,
|
||||
},
|
||||
});
|
||||
// Match workspace engine behavior: sync root doc first.
|
||||
sync.doc.addPriority(workspaceId, 100);
|
||||
sync.start();
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(remoteDocs.has(pageDocId)).toBe(true);
|
||||
});
|
||||
|
||||
sync.stop();
|
||||
remote.disconnect();
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { Storage } from '../storage';
|
||||
import type { broadcastChannelStorages } from './broadcast-channel';
|
||||
import type { cloudStorages } from './cloud';
|
||||
import type { diskStorages } from './disk';
|
||||
import type { idbStorages } from './idb';
|
||||
import type { idbV1Storages } from './idb/v1';
|
||||
import type { sqliteStorages } from './sqlite';
|
||||
@@ -15,6 +16,7 @@ type Storages =
|
||||
| typeof cloudStorages
|
||||
| typeof idbV1Storages
|
||||
| typeof idbStorages
|
||||
| typeof diskStorages
|
||||
| typeof sqliteStorages
|
||||
| typeof sqliteV1Storages
|
||||
| typeof broadcastChannelStorages;
|
||||
|
||||
@@ -241,13 +241,16 @@ export class DocSyncPeer {
|
||||
(await this.syncMetadata.getPeerPushedClock(this.peerId, docId))
|
||||
?.timestamp ?? null;
|
||||
const clock = await this.local.getDocTimestamp(docId);
|
||||
const remoteClock = this.status.remoteClocks.get(docId) ?? null;
|
||||
|
||||
throwIfAborted(signal);
|
||||
if (
|
||||
!this.remote.isReadonly &&
|
||||
clock &&
|
||||
(pushedClock === null ||
|
||||
pushedClock.getTime() < clock.timestamp.getTime())
|
||||
pushedClock.getTime() < clock.timestamp.getTime() ||
|
||||
remoteClock === null ||
|
||||
remoteClock.getTime() < clock.timestamp.getTime())
|
||||
) {
|
||||
await this.jobs.pullAndPush(docId, signal);
|
||||
} else {
|
||||
@@ -255,7 +258,6 @@ export class DocSyncPeer {
|
||||
const pulled =
|
||||
(await this.syncMetadata.getPeerPulledRemoteClock(this.peerId, docId))
|
||||
?.timestamp ?? null;
|
||||
const remoteClock = this.status.remoteClocks.get(docId);
|
||||
if (
|
||||
remoteClock &&
|
||||
(pulled === null || pulled.getTime() < remoteClock.getTime())
|
||||
@@ -676,10 +678,12 @@ export class DocSyncPeer {
|
||||
this.actions.addDoc(docId);
|
||||
}
|
||||
|
||||
const forceFullRemoteClockRefresh = this.peerId === 'disk';
|
||||
|
||||
// get cached clocks from metadata
|
||||
const cachedClocks = await this.syncMetadata.getPeerRemoteClocks(
|
||||
this.peerId
|
||||
);
|
||||
const cachedClocks = forceFullRemoteClockRefresh
|
||||
? {}
|
||||
: await this.syncMetadata.getPeerRemoteClocks(this.peerId);
|
||||
this.status.remoteClocks.clear();
|
||||
throwIfAborted(signal);
|
||||
for (const [id, v] of Object.entries(cachedClocks)) {
|
||||
@@ -687,9 +691,14 @@ export class DocSyncPeer {
|
||||
}
|
||||
this.statusUpdatedSubject$.next(true);
|
||||
|
||||
// get new clocks from server
|
||||
const maxClockValue = this.status.remoteClocks.max;
|
||||
// get clocks from server
|
||||
const maxClockValue = forceFullRemoteClockRefresh
|
||||
? undefined
|
||||
: this.status.remoteClocks.max;
|
||||
const newClocks = await this.remote.getDocTimestamps(maxClockValue);
|
||||
if (forceFullRemoteClockRefresh) {
|
||||
this.status.remoteClocks.clear();
|
||||
}
|
||||
for (const [id, v] of Object.entries(newClocks)) {
|
||||
this.status.remoteClocks.set(id, v);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { OpConsumer } from '@toeverything/infra/op';
|
||||
import { isEqual } from 'lodash-es';
|
||||
import { Observable } from 'rxjs';
|
||||
|
||||
import { type StorageConstructor } from '../impls';
|
||||
@@ -13,8 +14,9 @@ import type { StoreInitOptions, WorkerManagerOps, WorkerOps } from './ops';
|
||||
export type { WorkerManagerOps };
|
||||
|
||||
class StoreConsumer {
|
||||
private readonly storages: PeerStorageOptions<SpaceStorage>;
|
||||
private readonly sync: Sync;
|
||||
private storages: PeerStorageOptions<SpaceStorage> | null = null;
|
||||
private sync: Sync | null = null;
|
||||
private initOptions: StoreInitOptions;
|
||||
|
||||
get ensureLocal() {
|
||||
if (!this.storages) {
|
||||
@@ -70,20 +72,29 @@ class StoreConsumer {
|
||||
private readonly availableStorageImplementations: StorageConstructor[],
|
||||
init: StoreInitOptions
|
||||
) {
|
||||
this.initOptions = init;
|
||||
this.initWithOptions(init);
|
||||
}
|
||||
|
||||
private createStorage(opt: any): any {
|
||||
if (opt === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const Storage = this.availableStorageImplementations.find(
|
||||
impl => impl.identifier === opt.name
|
||||
);
|
||||
if (!Storage) {
|
||||
throw new Error(`Storage implementation ${opt.name} not found`);
|
||||
}
|
||||
return new Storage(opt.opts as any);
|
||||
}
|
||||
|
||||
private initWithOptions(init: StoreInitOptions) {
|
||||
this.storages = {
|
||||
local: new SpaceStorage(
|
||||
Object.fromEntries(
|
||||
Object.entries(init.local).map(([type, opt]) => {
|
||||
if (opt === undefined) {
|
||||
return [type, undefined];
|
||||
}
|
||||
const Storage = this.availableStorageImplementations.find(
|
||||
impl => impl.identifier === opt.name
|
||||
);
|
||||
if (!Storage) {
|
||||
throw new Error(`Storage implementation ${opt.name} not found`);
|
||||
}
|
||||
return [type, new Storage(opt.opts as any)];
|
||||
return [type, this.createStorage(opt)];
|
||||
})
|
||||
)
|
||||
),
|
||||
@@ -94,18 +105,7 @@ class StoreConsumer {
|
||||
new SpaceStorage(
|
||||
Object.fromEntries(
|
||||
Object.entries(opts).map(([type, opt]) => {
|
||||
if (opt === undefined) {
|
||||
return [type, undefined];
|
||||
}
|
||||
const Storage = this.availableStorageImplementations.find(
|
||||
impl => impl.identifier === opt.name
|
||||
);
|
||||
if (!Storage) {
|
||||
throw new Error(
|
||||
`Storage implementation ${opt.name} not found`
|
||||
);
|
||||
}
|
||||
return [type, new Storage(opt.opts as any)];
|
||||
return [type, this.createStorage(opt)];
|
||||
})
|
||||
)
|
||||
),
|
||||
@@ -125,6 +125,69 @@ class StoreConsumer {
|
||||
this.registerHandlers(consumer);
|
||||
}
|
||||
|
||||
async reconfigure(init: StoreInitOptions) {
|
||||
if (isEqual(this.initOptions, init)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// If local storage config changes, fall back to full teardown/rebuild.
|
||||
// (Remote-only changes are expected, like enabling folder sync.)
|
||||
if (
|
||||
!this.storages ||
|
||||
!this.sync ||
|
||||
!isEqual(this.initOptions.local, init.local)
|
||||
) {
|
||||
await this.destroy();
|
||||
this.initOptions = init;
|
||||
this.initWithOptions(init);
|
||||
return;
|
||||
}
|
||||
|
||||
// Remote-only change: rebuild sync graph and remote storages in-place so
|
||||
// existing OpConsumers keep working.
|
||||
const prevInit = this.initOptions;
|
||||
const storages = this.storages;
|
||||
|
||||
this.sync.stop();
|
||||
|
||||
// Destroy removed or changed remote peers.
|
||||
for (const [peerId, prevPeerOpts] of Object.entries(prevInit.remotes)) {
|
||||
const nextPeerOpts = init.remotes[peerId];
|
||||
const changed = !nextPeerOpts || !isEqual(prevPeerOpts, nextPeerOpts);
|
||||
if (!changed) {
|
||||
continue;
|
||||
}
|
||||
const remote = storages.remotes[peerId];
|
||||
if (remote) {
|
||||
delete storages.remotes[peerId];
|
||||
remote.disconnect();
|
||||
await remote.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
// Create added or changed remote peers.
|
||||
for (const [peerId, nextPeerOpts] of Object.entries(init.remotes)) {
|
||||
const prevPeerOpts = prevInit.remotes[peerId];
|
||||
const changed = !prevPeerOpts || !isEqual(prevPeerOpts, nextPeerOpts);
|
||||
if (!changed) {
|
||||
continue;
|
||||
}
|
||||
const remote = new SpaceStorage(
|
||||
Object.fromEntries(
|
||||
Object.entries(nextPeerOpts).map(([type, opt]) => {
|
||||
return [type, this.createStorage(opt)];
|
||||
})
|
||||
)
|
||||
);
|
||||
storages.remotes[peerId] = remote;
|
||||
remote.connect();
|
||||
}
|
||||
|
||||
this.sync = new Sync(storages);
|
||||
this.sync.start();
|
||||
this.initOptions = init;
|
||||
}
|
||||
|
||||
async destroy() {
|
||||
this.sync?.stop();
|
||||
this.storages?.local.disconnect();
|
||||
@@ -133,6 +196,9 @@ class StoreConsumer {
|
||||
remote.disconnect();
|
||||
await remote.destroy();
|
||||
}
|
||||
|
||||
this.sync = null;
|
||||
this.storages = null;
|
||||
}
|
||||
|
||||
private readonly ENABLE_BATTERY_SAVE_MODE_DELAY = 1000;
|
||||
@@ -337,7 +403,12 @@ export class StoreManagerConsumer {
|
||||
private readonly storeDisposers = new Map<string, () => void>();
|
||||
private readonly storePool = new Map<
|
||||
string,
|
||||
{ store: StoreConsumer; refCount: number }
|
||||
{
|
||||
store: StoreConsumer;
|
||||
refCount: number;
|
||||
options: StoreInitOptions;
|
||||
reconfiguring?: Promise<void>;
|
||||
}
|
||||
>();
|
||||
private readonly telemetry = new TelemetryManager();
|
||||
|
||||
@@ -360,7 +431,22 @@ export class StoreManagerConsumer {
|
||||
this.availableStorageImplementations,
|
||||
options
|
||||
);
|
||||
storeRef = { store, refCount: 0 };
|
||||
storeRef = { store, refCount: 0, options };
|
||||
} else if (!isEqual(storeRef.options, options)) {
|
||||
const currentStoreRef = storeRef;
|
||||
// Options can change across renderer reloads (or when features like
|
||||
// folder sync are enabled). Reconfigure the shared store in-place
|
||||
// so existing consumers keep working with the latest remotes.
|
||||
currentStoreRef.reconfiguring = (
|
||||
currentStoreRef.reconfiguring ?? Promise.resolve()
|
||||
)
|
||||
.then(async () => {
|
||||
await currentStoreRef.store.reconfigure(options);
|
||||
currentStoreRef.options = options;
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('failed to reconfigure store', key, error);
|
||||
});
|
||||
}
|
||||
storeRef.refCount++;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user