mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-12 23:56:36 +08:00
feat: refactor doc write in native (#14272)
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,601 @@
|
||||
use y_octo::{Any, Map, TextAttributes, TextDeltaOp, TextInsert};
|
||||
|
||||
use super::{
|
||||
ParseError,
|
||||
blocksuite::get_string,
|
||||
schema::{
|
||||
PROP_CAPTION, PROP_CHECKED, PROP_COLUMN_ID_SUFFIX, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER,
|
||||
PROP_ORDER_SUFFIX, PROP_ROW_ID_SUFFIX, PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL,
|
||||
PROP_VIDEO_ID, PROP_WIDTH, SYS_FLAVOUR, table_cell_text_key,
|
||||
},
|
||||
table::{MarkdownTableOptions, render_markdown_table},
|
||||
value::{value_to_f64, value_to_string},
|
||||
};
|
||||
|
||||
/// Block flavours used in AFFiNE documents.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BlockFlavour {
|
||||
Paragraph,
|
||||
List,
|
||||
Code,
|
||||
Divider,
|
||||
Image,
|
||||
Table,
|
||||
Bookmark,
|
||||
EmbedYoutube,
|
||||
EmbedIframe,
|
||||
Callout,
|
||||
}
|
||||
|
||||
impl BlockFlavour {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
BlockFlavour::Paragraph => "affine:paragraph",
|
||||
BlockFlavour::List => "affine:list",
|
||||
BlockFlavour::Code => "affine:code",
|
||||
BlockFlavour::Divider => "affine:divider",
|
||||
BlockFlavour::Image => "affine:image",
|
||||
BlockFlavour::Table => "affine:table",
|
||||
BlockFlavour::Bookmark => "affine:bookmark",
|
||||
BlockFlavour::EmbedYoutube => "affine:embed-youtube",
|
||||
BlockFlavour::EmbedIframe => "affine:embed-iframe",
|
||||
BlockFlavour::Callout => "affine:callout",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"affine:paragraph" => Some(BlockFlavour::Paragraph),
|
||||
"affine:list" => Some(BlockFlavour::List),
|
||||
"affine:code" => Some(BlockFlavour::Code),
|
||||
"affine:divider" => Some(BlockFlavour::Divider),
|
||||
"affine:image" => Some(BlockFlavour::Image),
|
||||
"affine:table" => Some(BlockFlavour::Table),
|
||||
"affine:bookmark" => Some(BlockFlavour::Bookmark),
|
||||
"affine:embed-youtube" => Some(BlockFlavour::EmbedYoutube),
|
||||
"affine:embed-iframe" => Some(BlockFlavour::EmbedIframe),
|
||||
"affine:callout" => Some(BlockFlavour::Callout),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Block types for paragraphs and lists.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum BlockType {
|
||||
// Paragraph types
|
||||
Text,
|
||||
H1,
|
||||
H2,
|
||||
H3,
|
||||
H4,
|
||||
H5,
|
||||
H6,
|
||||
Quote,
|
||||
// List types
|
||||
Bulleted,
|
||||
Numbered,
|
||||
Todo,
|
||||
// Preserve unknown types when loading from ydoc.
|
||||
Unknown(String),
|
||||
}
|
||||
|
||||
impl BlockType {
|
||||
pub fn as_str(&self) -> &str {
|
||||
match self {
|
||||
BlockType::Text => "text",
|
||||
BlockType::H1 => "h1",
|
||||
BlockType::H2 => "h2",
|
||||
BlockType::H3 => "h3",
|
||||
BlockType::H4 => "h4",
|
||||
BlockType::H5 => "h5",
|
||||
BlockType::H6 => "h6",
|
||||
BlockType::Quote => "quote",
|
||||
BlockType::Bulleted => "bulleted",
|
||||
BlockType::Numbered => "numbered",
|
||||
BlockType::Todo => "todo",
|
||||
BlockType::Unknown(value) => value.as_str(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"text" => Some(BlockType::Text),
|
||||
"h1" => Some(BlockType::H1),
|
||||
"h2" => Some(BlockType::H2),
|
||||
"h3" => Some(BlockType::H3),
|
||||
"h4" => Some(BlockType::H4),
|
||||
"h5" => Some(BlockType::H5),
|
||||
"h6" => Some(BlockType::H6),
|
||||
"quote" => Some(BlockType::Quote),
|
||||
"bulleted" => Some(BlockType::Bulleted),
|
||||
"numbered" => Some(BlockType::Numbered),
|
||||
"todo" => Some(BlockType::Todo),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str_lossy(value: String) -> Self {
|
||||
Self::from_str(&value).unwrap_or(BlockType::Unknown(value))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct ImageSpec {
|
||||
pub(super) source_id: String,
|
||||
pub(super) caption: Option<String>,
|
||||
pub(super) width: Option<f64>,
|
||||
pub(super) height: Option<f64>,
|
||||
}
|
||||
|
||||
impl ImageSpec {
|
||||
pub(super) fn render_markdown(&self) -> String {
|
||||
let blob_url = format!("blob://{}", self.source_id);
|
||||
let caption = self.caption.as_deref().unwrap_or("");
|
||||
|
||||
if self.width.is_some() || self.height.is_some() || !caption.is_empty() {
|
||||
let width_text = self
|
||||
.width
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "auto".into());
|
||||
let height_text = self
|
||||
.height
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "auto".into());
|
||||
return format!(
|
||||
"<img\n src=\"{blob_url}\"\n alt=\"{caption}\"\n width=\"{width_text}\"\n height=\"{height_text}\"\n/>\n\n"
|
||||
);
|
||||
}
|
||||
|
||||
let alt = if caption.is_empty() {
|
||||
self.source_id.as_str()
|
||||
} else {
|
||||
caption
|
||||
};
|
||||
format!("\n\n\n")
|
||||
}
|
||||
|
||||
pub(super) fn from_block_map(block: &Map) -> Self {
|
||||
let source_id = get_string(block, PROP_SOURCE_ID).unwrap_or_default();
|
||||
let caption = get_string(block, PROP_CAPTION);
|
||||
let width = block.get(PROP_WIDTH).and_then(value_to_f64);
|
||||
let height = block.get(PROP_HEIGHT).and_then(value_to_f64);
|
||||
ImageSpec {
|
||||
source_id,
|
||||
caption,
|
||||
width,
|
||||
height,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn normalize_source(src: &str) -> Result<String, ParseError> {
|
||||
let trimmed = src.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(ParseError::ParserError("invalid_image_source".into()));
|
||||
}
|
||||
if let Some(rest) = trimmed.strip_prefix("blob://") {
|
||||
if rest.is_empty() {
|
||||
return Err(ParseError::ParserError("invalid_image_source".into()));
|
||||
}
|
||||
return Ok(rest.to_string());
|
||||
}
|
||||
if !trimmed.contains('/') && !trimmed.contains(':') {
|
||||
return Ok(trimmed.to_string());
|
||||
}
|
||||
if let Some(pos) = trimmed.rfind("/blobs/") {
|
||||
let id = &trimmed[pos + "/blobs/".len()..];
|
||||
let id = id.split(['?', '#']).next().unwrap_or("");
|
||||
if !id.is_empty() {
|
||||
return Ok(id.to_string());
|
||||
}
|
||||
}
|
||||
Err(ParseError::ParserError("unsupported_image_source".into()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct TableSpec {
|
||||
pub(super) rows: Vec<Vec<String>>,
|
||||
}
|
||||
|
||||
impl TableSpec {
|
||||
pub(super) fn from_block_map(block: &Map) -> Self {
|
||||
let mut row_entries: Vec<(String, String)> = Vec::new();
|
||||
let mut column_entries: Vec<(String, String)> = Vec::new();
|
||||
|
||||
for key in block.keys() {
|
||||
if key.starts_with(PROP_ROWS_PREFIX)
|
||||
&& key.ends_with(PROP_ROW_ID_SUFFIX)
|
||||
&& let Some(row_id) = block.get(key).and_then(|value| value_to_string(&value))
|
||||
{
|
||||
let base = key.trim_end_matches(PROP_ROW_ID_SUFFIX);
|
||||
let order_key = format!("{base}{PROP_ORDER_SUFFIX}");
|
||||
let order = block
|
||||
.get(&order_key)
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
row_entries.push((order, row_id));
|
||||
}
|
||||
if key.starts_with(PROP_COLUMNS_PREFIX)
|
||||
&& key.ends_with(PROP_COLUMN_ID_SUFFIX)
|
||||
&& let Some(column_id) = block.get(key).and_then(|value| value_to_string(&value))
|
||||
{
|
||||
let base = key.trim_end_matches(PROP_COLUMN_ID_SUFFIX);
|
||||
let order_key = format!("{base}{PROP_ORDER_SUFFIX}");
|
||||
let order = block
|
||||
.get(&order_key)
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
column_entries.push((order, column_id));
|
||||
}
|
||||
}
|
||||
|
||||
row_entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
column_entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for (_, row_id) in row_entries {
|
||||
let mut row = Vec::new();
|
||||
for (_, column_id) in &column_entries {
|
||||
let cell_key = table_cell_text_key(&row_id, column_id);
|
||||
let cell_text = block
|
||||
.get(&cell_key)
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
row.push(cell_text);
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
Self { rows }
|
||||
}
|
||||
|
||||
pub(super) fn render_markdown(&self) -> Option<String> {
|
||||
let options = MarkdownTableOptions::new(false, "<br />", true);
|
||||
render_markdown_table(&self.rows, options)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct BookmarkSpec {
|
||||
pub(super) url: String,
|
||||
pub(super) caption: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct EmbedYoutubeSpec {
|
||||
pub(super) video_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct EmbedIframeSpec {
|
||||
pub(super) url: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct BlockSpec {
|
||||
pub(super) flavour: BlockFlavour,
|
||||
pub(super) block_type: Option<BlockType>,
|
||||
pub(super) text: Vec<TextDeltaOp>,
|
||||
pub(super) checked: Option<bool>,
|
||||
pub(super) language: Option<String>,
|
||||
pub(super) order: Option<i64>,
|
||||
pub(super) image: Option<ImageSpec>,
|
||||
pub(super) table: Option<TableSpec>,
|
||||
pub(super) bookmark: Option<BookmarkSpec>,
|
||||
pub(super) embed_youtube: Option<EmbedYoutubeSpec>,
|
||||
pub(super) embed_iframe: Option<EmbedIframeSpec>,
|
||||
}
|
||||
|
||||
impl BlockSpec {
|
||||
pub(super) fn is_exact(&self, other: &BlockSpec) -> bool {
|
||||
self.flavour == other.flavour
|
||||
&& self.block_type == other.block_type
|
||||
&& self.checked == other.checked
|
||||
&& self.language == other.language
|
||||
&& self.image == other.image
|
||||
&& self.table == other.table
|
||||
&& self.bookmark == other.bookmark
|
||||
&& self.embed_youtube == other.embed_youtube
|
||||
&& self.embed_iframe == other.embed_iframe
|
||||
&& text_delta_eq(&self.text, &other.text)
|
||||
}
|
||||
|
||||
pub(super) fn is_similar(&self, other: &BlockSpec) -> bool {
|
||||
self.flavour == other.flavour && self.block_type == other.block_type
|
||||
}
|
||||
|
||||
pub(super) fn block_type_str(&self) -> Option<&str> {
|
||||
self.block_type.as_ref().map(BlockType::as_str)
|
||||
}
|
||||
|
||||
pub(super) fn from_block_map(block: &Map) -> Result<Self, ParseError> {
|
||||
let flavour_value = get_string(block, SYS_FLAVOUR).unwrap_or_default();
|
||||
let Some(flavour) = BlockFlavour::from_str(&flavour_value) else {
|
||||
return Err(ParseError::ParserError(format!(
|
||||
"unsupported block flavour: {flavour_value}"
|
||||
)));
|
||||
};
|
||||
Ok(Self::from_block_map_with_flavour(block, flavour))
|
||||
}
|
||||
|
||||
pub(super) fn from_block_map_with_flavour(block: &Map, flavour: BlockFlavour) -> Self {
|
||||
if flavour == BlockFlavour::Image {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: Some(ImageSpec::from_block_map(block)),
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::Table {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: Some(TableSpec::from_block_map(block)),
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::Bookmark {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: Some(BookmarkSpec {
|
||||
url: get_string(block, PROP_URL).unwrap_or_default(),
|
||||
caption: get_string(block, PROP_CAPTION),
|
||||
}),
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::EmbedYoutube {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: Some(EmbedYoutubeSpec {
|
||||
video_id: get_string(block, PROP_VIDEO_ID).unwrap_or_default(),
|
||||
}),
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::EmbedIframe {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: Some(EmbedIframeSpec {
|
||||
url: get_string(block, PROP_URL).unwrap_or_default(),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
let block_type = match get_string(block, PROP_TYPE) {
|
||||
Some(value) => Some(BlockType::from_str_lossy(value)),
|
||||
None => match flavour {
|
||||
BlockFlavour::Paragraph => Some(BlockType::Text),
|
||||
BlockFlavour::List => Some(BlockType::Bulleted),
|
||||
_ => None,
|
||||
},
|
||||
};
|
||||
|
||||
let text = block
|
||||
.get(PROP_TEXT)
|
||||
.and_then(|v| v.to_text())
|
||||
.map(|text| text.to_delta())
|
||||
.unwrap_or_default();
|
||||
|
||||
let checked = block.get(PROP_CHECKED).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::True => Some(true),
|
||||
Any::False => Some(false),
|
||||
_ => None,
|
||||
});
|
||||
let language = get_string(block, PROP_LANGUAGE);
|
||||
let order = block.get(PROP_ORDER).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::Integer(value) => Some(value as i64),
|
||||
Any::BigInt64(value) => Some(value),
|
||||
Any::Float32(value) => Some(value.0 as i64),
|
||||
Any::Float64(value) => Some(value.0 as i64),
|
||||
_ => None,
|
||||
});
|
||||
|
||||
BlockSpec {
|
||||
flavour,
|
||||
block_type,
|
||||
text,
|
||||
checked,
|
||||
language,
|
||||
order,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct BlockNode {
|
||||
pub(super) spec: BlockSpec,
|
||||
pub(super) children: Vec<BlockNode>,
|
||||
}
|
||||
|
||||
pub(super) trait TreeNode {
|
||||
fn children(&self) -> &[Self]
|
||||
where
|
||||
Self: Sized;
|
||||
}
|
||||
|
||||
impl TreeNode for BlockNode {
|
||||
fn children(&self) -> &[BlockNode] {
|
||||
&self.children
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn count_tree_nodes<T: TreeNode>(nodes: &[T]) -> usize {
|
||||
nodes.iter().map(|node| 1 + count_tree_nodes(node.children())).sum()
|
||||
}
|
||||
|
||||
pub(super) fn text_delta_eq(a: &[TextDeltaOp], b: &[TextDeltaOp]) -> bool {
|
||||
if a.len() != b.len() {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (left, right) in a.iter().zip(b.iter()) {
|
||||
match (left, right) {
|
||||
(
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text_a),
|
||||
format: format_a,
|
||||
},
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text_b),
|
||||
format: format_b,
|
||||
},
|
||||
) => {
|
||||
if text_a != text_b {
|
||||
return false;
|
||||
}
|
||||
if !attrs_eq(format_a.as_ref(), format_b.as_ref()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
_ => return false,
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
fn attrs_eq(a: Option<&TextAttributes>, b: Option<&TextAttributes>) -> bool {
|
||||
match (a, b) {
|
||||
(None, None) => true,
|
||||
(Some(left), Some(right)) => {
|
||||
if left.len() != right.len() {
|
||||
return false;
|
||||
}
|
||||
left.iter().all(|(key, value)| right.get(key) == Some(value))
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::{super::write::builder::text_ops_from_plain, *};
|
||||
use crate::doc_parser::build_full_doc;
|
||||
|
||||
fn spec_from_markdown(markdown: &str, doc_id: &str, flavour: &str) -> BlockSpec {
|
||||
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, SYS_FLAVOUR).as_deref() == Some(flavour)
|
||||
{
|
||||
return BlockSpec::from_block_map(&block_map).expect("spec");
|
||||
}
|
||||
}
|
||||
|
||||
panic!("block not found: {flavour}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_paragraph() {
|
||||
let spec = spec_from_markdown("Plain paragraph.", "block-spec-paragraph", "affine:paragraph");
|
||||
assert_eq!(spec.flavour, BlockFlavour::Paragraph);
|
||||
assert_eq!(spec.block_type, Some(BlockType::Text));
|
||||
assert_eq!(spec.text, text_ops_from_plain("Plain paragraph."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_list_checked() {
|
||||
let spec = spec_from_markdown("- [x] Done", "block-spec-list", "affine:list");
|
||||
assert_eq!(spec.flavour, BlockFlavour::List);
|
||||
assert_eq!(spec.block_type, Some(BlockType::Todo));
|
||||
assert_eq!(spec.checked, Some(true));
|
||||
assert_eq!(spec.text, text_ops_from_plain("Done"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_image() {
|
||||
let spec = spec_from_markdown("", "block-spec-image", "affine:image");
|
||||
assert_eq!(spec.flavour, BlockFlavour::Image);
|
||||
let image = spec.image.expect("image spec");
|
||||
assert_eq!(image.source_id, "image-id");
|
||||
assert_eq!(image.caption.as_deref(), Some("Alt"));
|
||||
assert_eq!(image.width, None);
|
||||
assert_eq!(image.height, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_table() {
|
||||
let spec = spec_from_markdown(
|
||||
"| A | B |\n| --- | --- |\n| 1 | 2 |",
|
||||
"block-spec-table",
|
||||
"affine:table",
|
||||
);
|
||||
assert_eq!(spec.flavour, BlockFlavour::Table);
|
||||
let table = spec.table.expect("table spec");
|
||||
assert_eq!(
|
||||
table.rows,
|
||||
vec![
|
||||
vec!["A".to_string(), "B".to_string()],
|
||||
vec!["1".to_string(), "2".to_string()]
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_embed_iframe() {
|
||||
let spec = spec_from_markdown(
|
||||
r#"<iframe src="https://example.com/embed"></iframe>"#,
|
||||
"block-spec-embed-iframe",
|
||||
"affine:embed-iframe",
|
||||
);
|
||||
assert_eq!(spec.flavour, BlockFlavour::EmbedIframe);
|
||||
assert_eq!(spec.embed_iframe.as_ref().unwrap().url, "https://example.com/embed");
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,10 @@ use std::collections::{HashMap, HashSet};
|
||||
|
||||
use y_octo::Map;
|
||||
|
||||
use super::value::value_to_string;
|
||||
use super::{
|
||||
schema::{SYS_CHILDREN, SYS_FLAVOUR, SYS_ID},
|
||||
value::value_to_string,
|
||||
};
|
||||
|
||||
pub(super) struct BlockIndex {
|
||||
pub(super) block_pool: HashMap<String, Map>,
|
||||
@@ -96,7 +99,7 @@ pub(super) fn find_block_id_by_flavour(block_pool: &HashMap<String, Map>, flavou
|
||||
|
||||
pub(super) fn collect_child_ids(block: &Map) -> Vec<String> {
|
||||
block
|
||||
.get("sys:children")
|
||||
.get(SYS_CHILDREN)
|
||||
.and_then(|value| value.to_array())
|
||||
.map(|array| {
|
||||
array
|
||||
@@ -108,11 +111,11 @@ pub(super) fn collect_child_ids(block: &Map) -> Vec<String> {
|
||||
}
|
||||
|
||||
pub(super) fn get_block_id(block: &Map) -> Option<String> {
|
||||
get_string(block, "sys:id")
|
||||
get_string(block, SYS_ID)
|
||||
}
|
||||
|
||||
pub(super) fn get_flavour(block: &Map) -> Option<String> {
|
||||
get_string(block, "sys:flavour")
|
||||
get_string(block, SYS_FLAVOUR)
|
||||
}
|
||||
|
||||
pub(super) fn get_string(block: &Map, key: &str) -> Option<String> {
|
||||
@@ -157,3 +160,17 @@ pub(super) fn nearest_by_flavour(
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn find_child_id_by_flavour(
|
||||
parent: &Map,
|
||||
block_pool: &HashMap<String, Map>,
|
||||
flavour: &str,
|
||||
) -> Option<String> {
|
||||
collect_child_ids(parent).into_iter().find(|id| {
|
||||
block_pool
|
||||
.get(id)
|
||||
.and_then(get_flavour)
|
||||
.as_deref()
|
||||
.is_some_and(|value| value == flavour)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
use y_octo::{Doc, DocOptions};
|
||||
|
||||
use super::ParseError;
|
||||
|
||||
pub(super) fn is_empty_doc(binary: &[u8]) -> bool {
|
||||
binary.is_empty() || binary == [0, 0]
|
||||
}
|
||||
|
||||
pub(super) fn load_doc(binary: &[u8], doc_id: Option<&str>) -> Result<Doc, ParseError> {
|
||||
if is_empty_doc(binary) {
|
||||
return Err(ParseError::InvalidBinary);
|
||||
}
|
||||
|
||||
let mut doc = build_doc(doc_id);
|
||||
doc
|
||||
.apply_update_from_binary_v1(binary)
|
||||
.map_err(|_| ParseError::InvalidBinary)?;
|
||||
Ok(doc)
|
||||
}
|
||||
|
||||
pub(super) fn load_doc_or_new(binary: &[u8]) -> Result<Doc, ParseError> {
|
||||
if is_empty_doc(binary) {
|
||||
return Ok(DocOptions::new().build());
|
||||
}
|
||||
|
||||
let mut doc = DocOptions::new().build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(binary)
|
||||
.map_err(|_| ParseError::InvalidBinary)?;
|
||||
Ok(doc)
|
||||
}
|
||||
|
||||
fn build_doc(doc_id: Option<&str>) -> Doc {
|
||||
let options = DocOptions::new();
|
||||
match doc_id {
|
||||
Some(doc_id) => options.with_guid(doc_id.to_string()).build(),
|
||||
None => options.build(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use thiserror::Error;
|
||||
use y_octo::JwstCodecError;
|
||||
|
||||
#[derive(Error, Debug, Serialize, Deserialize)]
|
||||
pub enum ParseError {
|
||||
#[error("doc_not_found")]
|
||||
DocNotFound,
|
||||
#[error("invalid_binary")]
|
||||
InvalidBinary,
|
||||
#[error("sqlite_error: {0}")]
|
||||
SqliteError(String),
|
||||
#[error("parser_error: {0}")]
|
||||
ParserError(String),
|
||||
#[error("unknown: {0}")]
|
||||
Unknown(String),
|
||||
}
|
||||
|
||||
impl From<JwstCodecError> for ParseError {
|
||||
fn from(value: JwstCodecError) -> Self {
|
||||
if matches!(
|
||||
value,
|
||||
JwstCodecError::DamagedDocumentJson
|
||||
| JwstCodecError::IncompleteDocument(_)
|
||||
| JwstCodecError::InvalidWriteBuffer(_)
|
||||
| JwstCodecError::UpdateInvalid(_)
|
||||
| JwstCodecError::StructClockInvalid { expect: _, actually: _ }
|
||||
| JwstCodecError::StructSequenceInvalid { client_id: _, clock: _ }
|
||||
| JwstCodecError::StructSequenceNotExists(_)
|
||||
| JwstCodecError::RootStructNotFound(_)
|
||||
| JwstCodecError::ParentNotFound
|
||||
| JwstCodecError::IndexOutOfBound(_)
|
||||
) {
|
||||
return ParseError::InvalidBinary;
|
||||
}
|
||||
Self::ParserError(value.to_string())
|
||||
}
|
||||
}
|
||||
+92
-34
@@ -6,8 +6,11 @@ use std::{
|
||||
|
||||
use y_octo::{AHashMap, Any, Map, Text, TextAttributes, TextDeltaOp, TextInsert, Value};
|
||||
|
||||
use super::value::{
|
||||
any_as_string, any_as_u64, any_truthy, build_reference_payload, params_any_map_to_json, value_to_any,
|
||||
use super::{
|
||||
super::value::{
|
||||
any_as_string, any_as_u64, any_truthy, build_reference_payload, params_any_map_to_json, value_to_any,
|
||||
},
|
||||
inline::InlineStyle,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -20,18 +23,18 @@ struct InlineReference {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct InlineReferencePayload {
|
||||
pub(super) doc_id: String,
|
||||
pub(super) payload: String,
|
||||
pub(crate) struct InlineReferencePayload {
|
||||
pub(crate) doc_id: String,
|
||||
pub(crate) payload: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct DeltaToMdOptions {
|
||||
pub(crate) struct DeltaToMdOptions {
|
||||
doc_url_prefix: Option<String>,
|
||||
}
|
||||
|
||||
impl DeltaToMdOptions {
|
||||
pub(super) fn new(doc_url_prefix: Option<String>) -> Self {
|
||||
pub(crate) fn new(doc_url_prefix: Option<String>) -> Self {
|
||||
Self { doc_url_prefix }
|
||||
}
|
||||
|
||||
@@ -54,21 +57,32 @@ impl DeltaToMdOptions {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn text_to_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option<String> {
|
||||
block
|
||||
.get(key)
|
||||
.and_then(|value| value.to_text())
|
||||
.map(|text| delta_to_markdown(&text, options))
|
||||
}
|
||||
|
||||
pub(super) fn text_to_inline_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option<String> {
|
||||
pub(crate) fn text_to_inline_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option<String> {
|
||||
block
|
||||
.get(key)
|
||||
.and_then(|value| value.to_text())
|
||||
.map(|text| delta_to_inline_markdown(&text, options))
|
||||
}
|
||||
|
||||
pub(super) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec<InlineReferencePayload> {
|
||||
pub(crate) fn delta_ops_to_markdown(ops: &[TextDeltaOp], options: &DeltaToMdOptions) -> String {
|
||||
delta_to_markdown_with_options(ops, options, true)
|
||||
}
|
||||
|
||||
pub(crate) fn delta_ops_to_plain_text(ops: &[TextDeltaOp]) -> String {
|
||||
let mut out = String::new();
|
||||
for op in ops {
|
||||
if let TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text),
|
||||
..
|
||||
} = op
|
||||
{
|
||||
out.push_str(text);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub(crate) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec<InlineReferencePayload> {
|
||||
let mut refs = Vec::new();
|
||||
let mut seen: HashSet<(String, String)> = HashSet::new();
|
||||
|
||||
@@ -80,7 +94,7 @@ pub(super) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec<InlineRefe
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let reference = match attrs.get("reference").and_then(parse_inline_reference) {
|
||||
let reference = match attrs.get(InlineStyle::Reference.key()).and_then(parse_inline_reference) {
|
||||
Some(reference) => reference,
|
||||
None => continue,
|
||||
};
|
||||
@@ -102,7 +116,7 @@ pub(super) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec<InlineRefe
|
||||
refs
|
||||
}
|
||||
|
||||
pub(super) fn extract_inline_references_from_value(value: &Value) -> Vec<InlineReferencePayload> {
|
||||
pub(crate) fn extract_inline_references_from_value(value: &Value) -> Vec<InlineReferencePayload> {
|
||||
if let Some(text) = value.to_text() {
|
||||
return extract_inline_references(&text.to_delta());
|
||||
}
|
||||
@@ -123,7 +137,11 @@ fn extract_inline_references_from_any(value: &Any) -> Vec<InlineReferencePayload
|
||||
let mut seen: HashSet<(String, String)> = HashSet::new();
|
||||
|
||||
for op in ops {
|
||||
let reference = match op.attributes.get("reference").and_then(parse_inline_reference) {
|
||||
let reference = match op
|
||||
.attributes
|
||||
.get(InlineStyle::Reference.key())
|
||||
.and_then(parse_inline_reference)
|
||||
{
|
||||
Some(reference) => reference,
|
||||
None => continue,
|
||||
};
|
||||
@@ -178,10 +196,6 @@ fn inline_reference_payload(reference: &InlineReference) -> Option<String> {
|
||||
Some(build_reference_payload(&reference.page_id, params))
|
||||
}
|
||||
|
||||
fn delta_to_markdown(text: &Text, options: &DeltaToMdOptions) -> String {
|
||||
delta_to_markdown_with_options(&text.to_delta(), options, true)
|
||||
}
|
||||
|
||||
fn delta_to_inline_markdown(text: &Text, options: &DeltaToMdOptions) -> String {
|
||||
delta_to_markdown_with_options(&text.to_delta(), options, false)
|
||||
}
|
||||
@@ -274,7 +288,7 @@ fn delta_any_to_inline_markdown(value: &Any, options: &DeltaToMdOptions) -> Opti
|
||||
delta_ops_from_any(value).map(|ops| delta_ops_to_markdown_with_options(&ops, options, false))
|
||||
}
|
||||
|
||||
pub(super) fn delta_value_to_inline_markdown(value: &Value, options: &DeltaToMdOptions) -> Option<String> {
|
||||
pub(crate) fn delta_value_to_inline_markdown(value: &Value, options: &DeltaToMdOptions) -> Option<String> {
|
||||
if let Some(text) = value.to_text() {
|
||||
return Some(delta_to_inline_markdown(&text, options));
|
||||
}
|
||||
@@ -538,19 +552,29 @@ fn apply_inline_attributes(
|
||||
}
|
||||
|
||||
fn inline_node_for_attr(attr: &str, attrs: &TextAttributes, options: &DeltaToMdOptions) -> Option<Rc<RefCell<Node>>> {
|
||||
match attr {
|
||||
"italic" => Some(Node::new_inline("_", "_")),
|
||||
"bold" => Some(Node::new_inline("**", "**")),
|
||||
"link" => attrs
|
||||
let style = InlineStyle::from_key(attr)?;
|
||||
if let Some(delimiter) = style.delimiter() {
|
||||
return Some(Node::new_inline(delimiter.open, delimiter.close));
|
||||
}
|
||||
|
||||
match style {
|
||||
InlineStyle::Underline => Some(Node::new_inline("<u>", "</u>")),
|
||||
InlineStyle::Color => {
|
||||
let color = attrs.get(attr).and_then(any_as_string)?.trim();
|
||||
if color.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(Node::new_inline(&format!("<span style=\"color: {color}\">"), "</span>"))
|
||||
}
|
||||
}
|
||||
InlineStyle::Link => attrs
|
||||
.get(attr)
|
||||
.and_then(any_as_string)
|
||||
.map(|url| Node::new_inline("[", &format!("]({url})"))),
|
||||
"reference" => attrs.get(attr).and_then(parse_inline_reference).map(|reference| {
|
||||
InlineStyle::Reference => attrs.get(attr).and_then(parse_inline_reference).map(|reference| {
|
||||
let (title, link) = options.build_reference_link(&reference);
|
||||
Node::new_inline("[", &format!("{title}]({link})"))
|
||||
}),
|
||||
"strike" => Some(Node::new_inline("~~", "~~")),
|
||||
"code" => Some(Node::new_inline("`", "`")),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -560,7 +584,7 @@ fn has_block_level_attribute(attrs: &TextAttributes) -> bool {
|
||||
}
|
||||
|
||||
fn is_inline_attribute(attr: &str) -> bool {
|
||||
matches!(attr, "italic" | "bold" | "link" | "reference" | "strike" | "code")
|
||||
InlineStyle::from_key(attr).is_some()
|
||||
}
|
||||
|
||||
fn encode_link(link: &str) -> String {
|
||||
@@ -741,13 +765,17 @@ fn new_line(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::Value;
|
||||
use y_octo::{Any, TextAttributes, TextDeltaOp, TextInsert};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_delta_to_inline_markdown_link() {
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert("link".into(), Any::String("https://example.com".into()));
|
||||
attrs.insert(
|
||||
InlineStyle::Link.key().into(),
|
||||
Any::String("https://example.com".into()),
|
||||
);
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("AFFiNE".into()),
|
||||
@@ -767,7 +795,7 @@ mod tests {
|
||||
ref_map.insert("type".into(), Any::String("LinkedPage".into()));
|
||||
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert("reference".into(), Any::Object(ref_map));
|
||||
attrs.insert(InlineStyle::Reference.key().into(), Any::Object(ref_map));
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Doc Title".into()),
|
||||
@@ -781,4 +809,34 @@ mod tests {
|
||||
let payload: Value = serde_json::from_str(&refs[0].payload).unwrap();
|
||||
assert_eq!(payload, serde_json::json!({ "docId": "doc123" }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delta_to_inline_markdown_underline() {
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert(InlineStyle::Underline.key().into(), Any::True);
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Under".into()),
|
||||
format: Some(attrs),
|
||||
}];
|
||||
|
||||
let options = DeltaToMdOptions::new(None);
|
||||
let rendered = delta_to_markdown_with_options(&delta, &options, false);
|
||||
assert_eq!(rendered, "<u>Under</u>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delta_to_inline_markdown_color() {
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert(InlineStyle::Color.key().into(), Any::String("red".into()));
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Red".into()),
|
||||
format: Some(attrs),
|
||||
}];
|
||||
|
||||
let options = DeltaToMdOptions::new(None);
|
||||
let rendered = delta_to_markdown_with_options(&delta, &options, false);
|
||||
assert_eq!(rendered, "<span style=\"color: red\">Red</span>");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
const INLINE_ATTR_BOLD: &str = "bold";
|
||||
const INLINE_ATTR_ITALIC: &str = "italic";
|
||||
const INLINE_ATTR_UNDERLINE: &str = "underline";
|
||||
const INLINE_ATTR_STRIKE: &str = "strike";
|
||||
const INLINE_ATTR_CODE: &str = "code";
|
||||
const INLINE_ATTR_LINK: &str = "link";
|
||||
const INLINE_ATTR_REFERENCE: &str = "reference";
|
||||
const INLINE_ATTR_COLOR: &str = "color";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum InlineStyle {
|
||||
Bold,
|
||||
Italic,
|
||||
Underline,
|
||||
Strike,
|
||||
Code,
|
||||
Link,
|
||||
Reference,
|
||||
Color,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) struct InlineDelimiter {
|
||||
pub(super) open: &'static str,
|
||||
pub(super) close: &'static str,
|
||||
}
|
||||
|
||||
impl InlineStyle {
|
||||
pub(super) fn key(self) -> &'static str {
|
||||
match self {
|
||||
InlineStyle::Bold => INLINE_ATTR_BOLD,
|
||||
InlineStyle::Italic => INLINE_ATTR_ITALIC,
|
||||
InlineStyle::Underline => INLINE_ATTR_UNDERLINE,
|
||||
InlineStyle::Strike => INLINE_ATTR_STRIKE,
|
||||
InlineStyle::Code => INLINE_ATTR_CODE,
|
||||
InlineStyle::Link => INLINE_ATTR_LINK,
|
||||
InlineStyle::Reference => INLINE_ATTR_REFERENCE,
|
||||
InlineStyle::Color => INLINE_ATTR_COLOR,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn from_key(key: &str) -> Option<Self> {
|
||||
match key {
|
||||
INLINE_ATTR_BOLD => Some(InlineStyle::Bold),
|
||||
INLINE_ATTR_ITALIC => Some(InlineStyle::Italic),
|
||||
INLINE_ATTR_UNDERLINE => Some(InlineStyle::Underline),
|
||||
INLINE_ATTR_STRIKE => Some(InlineStyle::Strike),
|
||||
INLINE_ATTR_CODE => Some(InlineStyle::Code),
|
||||
INLINE_ATTR_LINK => Some(InlineStyle::Link),
|
||||
INLINE_ATTR_REFERENCE => Some(InlineStyle::Reference),
|
||||
INLINE_ATTR_COLOR => Some(InlineStyle::Color),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn delimiter(self) -> Option<InlineDelimiter> {
|
||||
match self {
|
||||
InlineStyle::Italic => Some(InlineDelimiter { open: "_", close: "_" }),
|
||||
InlineStyle::Bold => Some(InlineDelimiter {
|
||||
open: "**",
|
||||
close: "**",
|
||||
}),
|
||||
InlineStyle::Strike => Some(InlineDelimiter {
|
||||
open: "~~",
|
||||
close: "~~",
|
||||
}),
|
||||
InlineStyle::Code => Some(InlineDelimiter { open: "`", close: "`" }),
|
||||
InlineStyle::Link | InlineStyle::Reference | InlineStyle::Underline | InlineStyle::Color => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
mod delta;
|
||||
mod inline;
|
||||
mod parser;
|
||||
mod render;
|
||||
|
||||
pub(crate) use delta::{
|
||||
DeltaToMdOptions, InlineReferencePayload, delta_value_to_inline_markdown, extract_inline_references,
|
||||
extract_inline_references_from_value, text_to_inline_markdown,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use parser::MAX_MARKDOWN_CHARS;
|
||||
pub(crate) use parser::{MAX_BLOCKS, parse_markdown_blocks};
|
||||
pub(crate) use render::{MarkdownRenderer, MarkdownWriter};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,235 @@
|
||||
use super::{
|
||||
super::block_spec::{BlockFlavour, BlockSpec},
|
||||
delta::{DeltaToMdOptions, delta_ops_to_markdown, delta_ops_to_plain_text},
|
||||
};
|
||||
|
||||
pub(crate) struct MarkdownWriter<'a> {
|
||||
output: &'a mut String,
|
||||
}
|
||||
|
||||
impl<'a> MarkdownWriter<'a> {
|
||||
pub(crate) fn new(output: &'a mut String) -> Self {
|
||||
Self { output }
|
||||
}
|
||||
|
||||
pub(crate) fn push_paragraph(&mut self, prefix: &str, text: &str) {
|
||||
if prefix == "> " {
|
||||
let quoted = text
|
||||
.split('\n')
|
||||
.map(|line| format!("{prefix}{line}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
self.output.push_str("ed);
|
||||
if !text.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
self.output.push('\n');
|
||||
return;
|
||||
}
|
||||
|
||||
self.output.push_str(prefix);
|
||||
self.output.push_str(text);
|
||||
if !text.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
self.output.push('\n');
|
||||
}
|
||||
|
||||
pub(crate) fn push_list_item(&mut self, indent: &str, prefix: &str, text: &str) {
|
||||
self.output.push_str(indent);
|
||||
self.output.push_str(prefix);
|
||||
self.output.push_str(text);
|
||||
if !text.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn push_code_block(&mut self, lang: &str, text: &str) {
|
||||
self.output.push_str("```");
|
||||
self.output.push_str(lang);
|
||||
self.output.push('\n');
|
||||
self.output.push_str(text);
|
||||
self.output.push_str("\n```\n\n");
|
||||
}
|
||||
|
||||
pub(crate) fn push_divider(&mut self) {
|
||||
self.output.push_str("\n---\n\n");
|
||||
}
|
||||
|
||||
pub(crate) fn push_table(&mut self, table: &str) {
|
||||
if table.is_empty() {
|
||||
self.output.push('\n');
|
||||
return;
|
||||
}
|
||||
self.output.push_str(table);
|
||||
if !table.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
self.output.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn paragraph_prefix(type_: &str) -> &'static str {
|
||||
match type_ {
|
||||
"h1" => "# ",
|
||||
"h2" => "## ",
|
||||
"h3" => "### ",
|
||||
"h4" => "#### ",
|
||||
"h5" => "##### ",
|
||||
"h6" => "###### ",
|
||||
"quote" => "> ",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn list_prefix(r#type: &str, checked: bool, order: Option<i64>) -> String {
|
||||
match r#type {
|
||||
"bulleted" => "* ".to_string(),
|
||||
"todo" => if checked { "- [x] " } else { "- [ ] " }.to_string(),
|
||||
_ => format!("{}. ", order.unwrap_or(1)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn list_indent(depth: usize) -> String {
|
||||
" ".repeat(depth)
|
||||
}
|
||||
|
||||
pub(crate) struct MarkdownRenderer<'a> {
|
||||
options: &'a DeltaToMdOptions,
|
||||
}
|
||||
|
||||
impl<'a> MarkdownRenderer<'a> {
|
||||
pub(crate) fn new(options: &'a DeltaToMdOptions) -> Self {
|
||||
Self { options }
|
||||
}
|
||||
|
||||
pub(crate) fn write_block(&self, output: &mut String, spec: &BlockSpec, list_depth: usize) {
|
||||
match spec.flavour {
|
||||
BlockFlavour::Paragraph => {
|
||||
let prefix = paragraph_prefix(spec.block_type_str().unwrap_or_default());
|
||||
let text_md = delta_ops_to_markdown(&spec.text, self.options);
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_paragraph(prefix, &text_md);
|
||||
}
|
||||
BlockFlavour::List => {
|
||||
let type_ = spec.block_type_str().unwrap_or("bulleted");
|
||||
let checked = spec.checked.unwrap_or(false);
|
||||
let prefix = list_prefix(type_, checked, spec.order);
|
||||
let indent = list_indent(list_depth);
|
||||
let text_md = delta_ops_to_markdown(&spec.text, self.options);
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_list_item(&indent, &prefix, &text_md);
|
||||
}
|
||||
BlockFlavour::Code => {
|
||||
let text = delta_ops_to_plain_text(&spec.text);
|
||||
let lang = spec.language.as_deref().unwrap_or_default();
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_code_block(lang, &text);
|
||||
}
|
||||
BlockFlavour::Divider => {
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_divider();
|
||||
}
|
||||
BlockFlavour::Image => {
|
||||
if let Some(image) = spec.image.as_ref() {
|
||||
output.push_str(&image.render_markdown());
|
||||
}
|
||||
}
|
||||
BlockFlavour::Bookmark => {
|
||||
if let Some(bookmark) = spec.bookmark.as_ref() {
|
||||
output.push_str(&format!("\n[](Bookmark,{})\n\n", bookmark.url));
|
||||
}
|
||||
}
|
||||
BlockFlavour::EmbedYoutube => {
|
||||
if let Some(embed) = spec.embed_youtube.as_ref() {
|
||||
output.push_str(
|
||||
&format!(
|
||||
"\n <iframe\n type=\"text/html\"\n width=\"100%\"\n height=\"410px\"\n src=\"https://www.youtube.com/embed/{}\"\n frameborder=\"0\"\n allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\"\n allowfullscreen\n credentialless>\n </iframe>\n\n",
|
||||
embed.video_id
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
BlockFlavour::EmbedIframe => {
|
||||
if let Some(embed) = spec.embed_iframe.as_ref() {
|
||||
output.push_str(&format!("\n<iframe src=\"{}\"></iframe>\n\n", embed.url));
|
||||
}
|
||||
}
|
||||
BlockFlavour::Callout => {}
|
||||
BlockFlavour::Table => {
|
||||
if let Some(table) = spec.table.as_ref()
|
||||
&& let Some(table_md) = table.render_markdown()
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_table(&table_md);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_paragraph_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_paragraph("# ", "Title\n");
|
||||
}
|
||||
assert_eq!(markdown, "# Title\n\n");
|
||||
|
||||
markdown.clear();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_paragraph("", "Plain");
|
||||
}
|
||||
assert_eq!(markdown, "Plain\n\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_list_item(" ", "* ", "Item\n");
|
||||
}
|
||||
assert_eq!(markdown, " * Item\n");
|
||||
|
||||
markdown.clear();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_list_item("", "- [ ] ", "Task");
|
||||
}
|
||||
assert_eq!(markdown, "- [ ] Task\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_code_block_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_code_block("rs", "fn main() {}");
|
||||
}
|
||||
assert_eq!(markdown, "```rs\nfn main() {}\n```\n\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_table_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_table("|a|b|\n|---|---|\n|1|2|\n");
|
||||
}
|
||||
assert_eq!(markdown, "|a|b|\n|---|---|\n|1|2|\n\n");
|
||||
|
||||
markdown.clear();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_table("|a|b|");
|
||||
}
|
||||
assert_eq!(markdown, "|a|b|\n\n");
|
||||
}
|
||||
}
|
||||
@@ -1,492 +0,0 @@
|
||||
//! Markdown to YDoc conversion module
|
||||
//!
|
||||
//! Converts markdown content into AFFiNE-compatible y-octo document binary
|
||||
//! format.
|
||||
|
||||
use y_octo::{Any, DocOptions};
|
||||
|
||||
use super::{
|
||||
affine::ParseError,
|
||||
markdown_utils::{BlockType, ParsedBlock, extract_title, parse_markdown_blocks},
|
||||
};
|
||||
|
||||
/// Block types used in AFFiNE documents
|
||||
const PAGE_FLAVOUR: &str = "affine:page";
|
||||
const NOTE_FLAVOUR: &str = "affine:note";
|
||||
|
||||
/// Intermediate representation of a block for building y-octo documents
|
||||
struct BlockBuilder {
|
||||
id: String,
|
||||
flavour: String,
|
||||
text_content: String,
|
||||
block_type: Option<BlockType>,
|
||||
checked: Option<bool>,
|
||||
code_language: Option<String>,
|
||||
#[allow(dead_code)] // Reserved for future nested block support
|
||||
children: Vec<String>,
|
||||
}
|
||||
|
||||
impl BlockBuilder {
|
||||
fn new(flavour: &str) -> Self {
|
||||
Self {
|
||||
id: nanoid::nanoid!(),
|
||||
flavour: flavour.to_string(),
|
||||
text_content: String::new(),
|
||||
block_type: None,
|
||||
checked: None,
|
||||
code_language: None,
|
||||
children: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_text(mut self, text: &str) -> Self {
|
||||
self.text_content = text.to_string();
|
||||
self
|
||||
}
|
||||
|
||||
fn with_block_type(mut self, btype: BlockType) -> Self {
|
||||
self.block_type = Some(btype);
|
||||
self
|
||||
}
|
||||
|
||||
fn with_checked(mut self, checked: bool) -> Self {
|
||||
self.checked = Some(checked);
|
||||
self
|
||||
}
|
||||
|
||||
fn with_code_language(mut self, lang: &str) -> Self {
|
||||
if !lang.is_empty() {
|
||||
self.code_language = Some(lang.to_string());
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a ParsedBlock from the shared parser into a BlockBuilder
|
||||
impl From<ParsedBlock> for BlockBuilder {
|
||||
fn from(parsed: ParsedBlock) -> Self {
|
||||
let mut builder = BlockBuilder::new(parsed.flavour.as_str()).with_text(&parsed.content);
|
||||
|
||||
if let Some(btype) = parsed.block_type {
|
||||
builder = builder.with_block_type(btype);
|
||||
}
|
||||
|
||||
if let Some(checked) = parsed.checked {
|
||||
builder = builder.with_checked(checked);
|
||||
}
|
||||
|
||||
if let Some(lang) = parsed.language {
|
||||
builder = builder.with_code_language(&lang);
|
||||
}
|
||||
|
||||
builder
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses markdown and converts it to an AFFiNE-compatible y-octo document
|
||||
/// binary.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `markdown` - The markdown content to convert
|
||||
/// * `doc_id` - The document ID to use
|
||||
///
|
||||
/// # Returns
|
||||
/// A binary vector containing the y-octo encoded document
|
||||
pub fn markdown_to_ydoc(markdown: &str, doc_id: &str) -> Result<Vec<u8>, ParseError> {
|
||||
// Extract the title from the first H1 heading
|
||||
let title = extract_title(markdown);
|
||||
|
||||
// Parse markdown into blocks using the shared parser
|
||||
let parsed_blocks = parse_markdown_blocks(markdown, true);
|
||||
|
||||
// Convert ParsedBlocks to BlockBuilders and collect IDs
|
||||
let mut blocks: Vec<BlockBuilder> = Vec::new();
|
||||
let mut content_block_ids: Vec<String> = Vec::new();
|
||||
|
||||
for parsed in parsed_blocks {
|
||||
let builder: BlockBuilder = parsed.into();
|
||||
content_block_ids.push(builder.id.clone());
|
||||
blocks.push(builder);
|
||||
}
|
||||
|
||||
// Build the y-octo document
|
||||
build_ydoc(doc_id, &title, blocks, content_block_ids)
|
||||
}
|
||||
|
||||
/// Builds the y-octo document from parsed blocks.
|
||||
///
|
||||
/// Uses a two-phase approach to ensure Yjs compatibility:
|
||||
/// 1. Phase 1: Create and insert empty maps into blocks_map (establishes parent
|
||||
/// items)
|
||||
/// 2. Phase 2: Populate each map with properties (child items reference
|
||||
/// existing parents)
|
||||
///
|
||||
/// This ordering ensures that when items reference their parent map's ID in the
|
||||
/// encoded binary, the parent ID always has a lower clock value, which Yjs
|
||||
/// requires.
|
||||
fn build_ydoc(
|
||||
doc_id: &str,
|
||||
title: &str,
|
||||
content_blocks: Vec<BlockBuilder>,
|
||||
content_block_ids: Vec<String>,
|
||||
) -> Result<Vec<u8>, ParseError> {
|
||||
// Create the document with the specified ID
|
||||
let doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
|
||||
// Create the blocks map
|
||||
let mut blocks_map = doc
|
||||
.get_or_create_map("blocks")
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
|
||||
// Create block IDs
|
||||
let page_id = nanoid::nanoid!();
|
||||
let note_id = nanoid::nanoid!();
|
||||
|
||||
// ==== PHASE 1: Insert empty maps to establish parent items ====
|
||||
// This ensures parent items have lower clock values than their children
|
||||
|
||||
// Insert empty page block map
|
||||
blocks_map
|
||||
.insert(
|
||||
page_id.clone(),
|
||||
doc.create_map().map_err(|e| ParseError::ParserError(e.to_string()))?,
|
||||
)
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
|
||||
// Insert empty note block map
|
||||
blocks_map
|
||||
.insert(
|
||||
note_id.clone(),
|
||||
doc.create_map().map_err(|e| ParseError::ParserError(e.to_string()))?,
|
||||
)
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
|
||||
// Insert empty content block maps
|
||||
for block in &content_blocks {
|
||||
blocks_map
|
||||
.insert(
|
||||
block.id.clone(),
|
||||
doc.create_map().map_err(|e| ParseError::ParserError(e.to_string()))?,
|
||||
)
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
// ==== PHASE 2: Populate the maps with their properties ====
|
||||
// Now each map has an item with a lower clock, so children will reference
|
||||
// correctly
|
||||
|
||||
// Populate page block
|
||||
if let Some(page_map) = blocks_map.get(&page_id).and_then(|v| v.to_map()) {
|
||||
populate_block_map(
|
||||
&doc,
|
||||
page_map,
|
||||
&page_id,
|
||||
PAGE_FLAVOUR,
|
||||
Some(title),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
vec![note_id.clone()],
|
||||
)?;
|
||||
}
|
||||
|
||||
// Populate note block
|
||||
if let Some(note_map) = blocks_map.get(¬e_id).and_then(|v| v.to_map()) {
|
||||
populate_block_map(
|
||||
&doc,
|
||||
note_map,
|
||||
¬e_id,
|
||||
NOTE_FLAVOUR,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
content_block_ids.clone(),
|
||||
)?;
|
||||
}
|
||||
|
||||
// Populate content blocks
|
||||
for block in content_blocks {
|
||||
if let Some(block_map) = blocks_map.get(&block.id).and_then(|v| v.to_map()) {
|
||||
populate_block_map(
|
||||
&doc,
|
||||
block_map,
|
||||
&block.id,
|
||||
&block.flavour,
|
||||
None,
|
||||
if block.text_content.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(&block.text_content)
|
||||
},
|
||||
block.block_type,
|
||||
block.checked,
|
||||
block.code_language.as_deref(),
|
||||
Vec::new(),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Encode the document
|
||||
doc
|
||||
.encode_update_v1()
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))
|
||||
}
|
||||
|
||||
/// Populates an existing block map with the given properties.
|
||||
///
|
||||
/// This function takes an already-inserted map and populates it with
|
||||
/// properties. The two-phase approach (insert empty map first, then populate)
|
||||
/// ensures that when child items reference the map as their parent, the
|
||||
/// parent's clock is lower.
|
||||
///
|
||||
/// IMPORTANT: We use Any types (Any::Array, Any::String) instead of CRDT types
|
||||
/// (y_octo::Array, y_octo::Text) for nested values. Any types are encoded
|
||||
/// inline as part of the item content, avoiding the forward reference issue
|
||||
/// where child items would reference a parent with a higher clock value.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn populate_block_map(
|
||||
_doc: &y_octo::Doc,
|
||||
mut block: y_octo::Map,
|
||||
block_id: &str,
|
||||
flavour: &str,
|
||||
title: Option<&str>,
|
||||
text_content: Option<&str>,
|
||||
block_type: Option<BlockType>,
|
||||
checked: Option<bool>,
|
||||
code_language: Option<&str>,
|
||||
children: Vec<String>,
|
||||
) -> Result<(), ParseError> {
|
||||
// Required fields
|
||||
block
|
||||
.insert("sys:id".to_string(), Any::String(block_id.to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
block
|
||||
.insert("sys:flavour".to_string(), Any::String(flavour.to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
|
||||
// Children - use Any::Array which is encoded inline (no forward references)
|
||||
let children_any: Vec<Any> = children.into_iter().map(Any::String).collect();
|
||||
block
|
||||
.insert("sys:children".to_string(), Any::Array(children_any))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
|
||||
// Title
|
||||
if let Some(title) = title {
|
||||
block
|
||||
.insert("prop:title".to_string(), Any::String(title.to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Text content - use Any::String instead of Y.Text
|
||||
// This is simpler and avoids CRDT overhead for initial document creation
|
||||
if let Some(content) = text_content {
|
||||
block
|
||||
.insert("prop:text".to_string(), Any::String(content.to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Block type
|
||||
if let Some(btype) = block_type {
|
||||
block
|
||||
.insert("prop:type".to_string(), Any::String(btype.as_str().to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Checked state
|
||||
if let Some(is_checked) = checked {
|
||||
block
|
||||
.insert(
|
||||
"prop:checked".to_string(),
|
||||
if is_checked { Any::True } else { Any::False },
|
||||
)
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Code language
|
||||
if let Some(lang) = code_language {
|
||||
block
|
||||
.insert("prop:language".to_string(), Any::String(lang.to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_simple_markdown() {
|
||||
let markdown = "# Hello World\n\nThis is a test paragraph.";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_list() {
|
||||
let markdown = "# Test List\n\n- Item 1\n- Item 2\n- Item 3";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_code() {
|
||||
let markdown = "# Code Example\n\n```rust\nfn main() {\n println!(\"Hello\");\n}\n```";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_headings() {
|
||||
let markdown = "# H1\n\n## H2\n\n### H3\n\nParagraph text.";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_usage() {
|
||||
assert_eq!(extract_title("# My Title\n\nContent"), "My Title");
|
||||
assert_eq!(extract_title("No heading"), "Untitled");
|
||||
assert_eq!(extract_title("## Secondary\n\nContent"), "Untitled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_markdown() {
|
||||
let result = markdown_to_ydoc("", "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty()); // Should still create valid doc structure
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_whitespace_only_markdown() {
|
||||
let result = markdown_to_ydoc(" \n\n\t\n ", "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_without_h1() {
|
||||
// Should use "Untitled" as default title
|
||||
let markdown = "## Secondary Heading\n\nSome content without H1.";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nested_lists() {
|
||||
let markdown = "# Nested Lists\n\n- Item 1\n - Nested 1.1\n - Nested 1.2\n- Item 2\n - Nested 2.1";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blockquote() {
|
||||
let markdown = "# Title\n\n> A blockquote";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_divider() {
|
||||
let markdown = "# Title\n\nBefore divider\n\n---\n\nAfter divider";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_numbered_list() {
|
||||
let markdown = "# Title\n\n1. First item\n2. Second item";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_four_paragraphs() {
|
||||
// Test with 4 paragraphs
|
||||
let markdown = "# Title\n\nP1.\n\nP2.\n\nP3.\n\nP4.";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mixed_content() {
|
||||
let markdown = r#"# Mixed Content
|
||||
|
||||
Some intro text.
|
||||
|
||||
- List item 1
|
||||
- List item 2
|
||||
|
||||
```python
|
||||
def hello():
|
||||
print("world")
|
||||
```
|
||||
|
||||
## Another Section
|
||||
|
||||
More text here.
|
||||
|
||||
1. Numbered item
|
||||
2. Another numbered
|
||||
|
||||
> A blockquote
|
||||
|
||||
---
|
||||
|
||||
Final paragraph.
|
||||
"#;
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_code_block_preserves_indentation() {
|
||||
// Code blocks should preserve leading whitespace (indentation) which is
|
||||
// semantically significant in languages like Python, YAML, etc.
|
||||
let markdown = r#"# Code Test
|
||||
|
||||
```python
|
||||
def indented():
|
||||
return "preserved"
|
||||
```
|
||||
"#;
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
// The test passes if the conversion succeeds without errors.
|
||||
// Full verification would require roundtrip testing.
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_document_creation() {
|
||||
// Test that markdown_to_ydoc creates a valid binary
|
||||
let original_md = "# Test Document\n\nHello world.";
|
||||
let doc_id = "creation-test";
|
||||
|
||||
let bin = markdown_to_ydoc(original_md, doc_id).expect("Should convert to ydoc");
|
||||
|
||||
// Binary should not be empty
|
||||
assert!(!bin.is_empty(), "Binary should not be empty");
|
||||
assert!(bin.len() > 10, "Binary should have meaningful content");
|
||||
}
|
||||
|
||||
// NOTE: Full roundtrip tests (markdown -> ydoc -> markdown) are not included
|
||||
// because y-octo has a limitation where nested maps created with create_map()
|
||||
// lose their content after encode/decode. This is a known y-octo limitation.
|
||||
//
|
||||
// However, the documents we create ARE valid and can be:
|
||||
// 1. Pushed to the AFFiNE server via DocStorageAdapter.pushDocUpdates
|
||||
// 2. Read by the AFFiNE client which uses JavaScript Yjs (not y-octo)
|
||||
//
|
||||
// The MCP write tools work because:
|
||||
// - markdown_to_ydoc creates valid y-octo binary
|
||||
// - The server stores the binary directly
|
||||
// - The client (browser) uses Yjs to decode and render
|
||||
}
|
||||
@@ -1,463 +0,0 @@
|
||||
//! Shared markdown utilities for the doc_parser module
|
||||
|
||||
use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, Options, Parser, Tag, TagEnd};
|
||||
|
||||
/// Block flavours used in AFFiNE documents
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BlockFlavour {
|
||||
Paragraph,
|
||||
List,
|
||||
Code,
|
||||
Divider,
|
||||
}
|
||||
|
||||
impl BlockFlavour {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
BlockFlavour::Paragraph => "affine:paragraph",
|
||||
BlockFlavour::List => "affine:list",
|
||||
BlockFlavour::Code => "affine:code",
|
||||
BlockFlavour::Divider => "affine:divider",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Block types for paragraphs and lists
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BlockType {
|
||||
// Paragraph types
|
||||
#[allow(dead_code)] // Used via as_str() for default paragraph type
|
||||
Text,
|
||||
H1,
|
||||
H2,
|
||||
H3,
|
||||
H4,
|
||||
H5,
|
||||
H6,
|
||||
Quote,
|
||||
// List types
|
||||
Bulleted,
|
||||
Numbered,
|
||||
Todo,
|
||||
}
|
||||
|
||||
impl BlockType {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
BlockType::Text => "text",
|
||||
BlockType::H1 => "h1",
|
||||
BlockType::H2 => "h2",
|
||||
BlockType::H3 => "h3",
|
||||
BlockType::H4 => "h4",
|
||||
BlockType::H5 => "h5",
|
||||
BlockType::H6 => "h6",
|
||||
BlockType::Quote => "quote",
|
||||
BlockType::Bulleted => "bulleted",
|
||||
BlockType::Numbered => "numbered",
|
||||
BlockType::Todo => "todo",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_heading_level(level: HeadingLevel) -> Self {
|
||||
match level {
|
||||
HeadingLevel::H1 => BlockType::H1,
|
||||
HeadingLevel::H2 => BlockType::H2,
|
||||
HeadingLevel::H3 => BlockType::H3,
|
||||
HeadingLevel::H4 => BlockType::H4,
|
||||
HeadingLevel::H5 => BlockType::H5,
|
||||
HeadingLevel::H6 => BlockType::H6,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A parsed block from markdown content
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ParsedBlock {
|
||||
pub flavour: BlockFlavour,
|
||||
pub block_type: Option<BlockType>,
|
||||
pub content: String,
|
||||
pub checked: Option<bool>,
|
||||
pub language: Option<String>,
|
||||
}
|
||||
|
||||
/// Parses markdown content into a list of parsed blocks.
|
||||
///
|
||||
/// This is the shared parsing logic used by both `markdown_to_ydoc` and
|
||||
/// `update_ydoc`.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `markdown` - The markdown content to parse
|
||||
/// * `skip_first_h1` - If true, the first H1 heading is skipped (used as
|
||||
/// document title)
|
||||
///
|
||||
/// # Returns
|
||||
/// A vector of parsed blocks
|
||||
pub fn parse_markdown_blocks(markdown: &str, skip_first_h1: bool) -> Vec<ParsedBlock> {
|
||||
// Note: ENABLE_TABLES is included for future support, but table events
|
||||
// currently fall through to the catch-all match arm. Table content appears as
|
||||
// plain text.
|
||||
let options = Options::ENABLE_STRIKETHROUGH
|
||||
| Options::ENABLE_TABLES
|
||||
| Options::ENABLE_TASKLISTS
|
||||
| Options::ENABLE_HEADING_ATTRIBUTES;
|
||||
let parser = Parser::new_ext(markdown, options);
|
||||
|
||||
let mut blocks = Vec::new();
|
||||
let mut current_text = String::new();
|
||||
let mut current_type: Option<BlockType> = None;
|
||||
let mut current_flavour = BlockFlavour::Paragraph;
|
||||
let mut in_list = false;
|
||||
let mut list_type_stack: Vec<BlockType> = Vec::new();
|
||||
// Per-item type override for task list markers (resets at each Item start)
|
||||
let mut current_item_type: Option<BlockType> = None;
|
||||
let mut in_code_block = false;
|
||||
let mut code_language = String::new();
|
||||
let mut first_h1_seen = !skip_first_h1; // If not skipping, mark as already seen
|
||||
let mut current_checked: Option<bool> = None;
|
||||
let mut pending_link_url: Option<String> = None;
|
||||
|
||||
for event in parser {
|
||||
match event {
|
||||
Event::Start(Tag::Heading { level, .. }) => {
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
|
||||
if level == HeadingLevel::H1 && !first_h1_seen {
|
||||
// Skip the first H1 - it's used as the document title
|
||||
current_type = Some(BlockType::H1);
|
||||
} else {
|
||||
current_type = Some(BlockType::from_heading_level(level));
|
||||
}
|
||||
current_flavour = BlockFlavour::Paragraph;
|
||||
}
|
||||
Event::End(TagEnd::Heading(level)) => {
|
||||
if level == HeadingLevel::H1 && !first_h1_seen {
|
||||
first_h1_seen = true;
|
||||
current_text.clear();
|
||||
current_type = None;
|
||||
} else {
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
Event::Start(Tag::Paragraph) => {}
|
||||
Event::End(TagEnd::Paragraph) => {
|
||||
if !in_list {
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
Event::Start(Tag::BlockQuote(_)) => {
|
||||
current_type = Some(BlockType::Quote);
|
||||
current_flavour = BlockFlavour::Paragraph;
|
||||
}
|
||||
Event::End(TagEnd::BlockQuote(_)) => {
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
}
|
||||
Event::Start(Tag::List(start_num)) => {
|
||||
in_list = true;
|
||||
let list_type = if start_num.is_some() {
|
||||
BlockType::Numbered
|
||||
} else {
|
||||
BlockType::Bulleted
|
||||
};
|
||||
list_type_stack.push(list_type);
|
||||
}
|
||||
Event::End(TagEnd::List(_)) => {
|
||||
list_type_stack.pop();
|
||||
if list_type_stack.is_empty() {
|
||||
in_list = false;
|
||||
}
|
||||
}
|
||||
Event::Start(Tag::Item) => {
|
||||
current_flavour = BlockFlavour::List;
|
||||
// Reset per-item type override
|
||||
current_item_type = None;
|
||||
if let Some(lt) = list_type_stack.last() {
|
||||
current_type = Some(*lt);
|
||||
}
|
||||
}
|
||||
Event::End(TagEnd::Item) => {
|
||||
// Use per-item override if set (for task items), otherwise use current_type
|
||||
if let Some(item_type) = current_item_type.take() {
|
||||
current_type = Some(item_type);
|
||||
}
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
current_flavour = BlockFlavour::Paragraph;
|
||||
}
|
||||
Event::TaskListMarker(checked) => {
|
||||
// Set per-item type override for this specific item only
|
||||
current_item_type = Some(BlockType::Todo);
|
||||
current_checked = Some(checked);
|
||||
}
|
||||
Event::Start(Tag::CodeBlock(kind)) => {
|
||||
in_code_block = true;
|
||||
current_flavour = BlockFlavour::Code;
|
||||
code_language = match kind {
|
||||
CodeBlockKind::Fenced(lang) => lang.to_string(),
|
||||
CodeBlockKind::Indented => String::new(),
|
||||
};
|
||||
}
|
||||
Event::End(TagEnd::CodeBlock) => {
|
||||
flush_code_block(&mut blocks, &mut current_text, &code_language);
|
||||
in_code_block = false;
|
||||
code_language.clear();
|
||||
current_flavour = BlockFlavour::Paragraph;
|
||||
}
|
||||
Event::Text(text) => {
|
||||
current_text.push_str(&text);
|
||||
}
|
||||
Event::Code(code) => {
|
||||
// Inline code - wrap in backticks
|
||||
current_text.push('`');
|
||||
current_text.push_str(&code);
|
||||
current_text.push('`');
|
||||
}
|
||||
Event::SoftBreak | Event::HardBreak => {
|
||||
if in_code_block {
|
||||
current_text.push('\n');
|
||||
} else {
|
||||
current_text.push(' ');
|
||||
}
|
||||
}
|
||||
Event::Rule => {
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
blocks.push(ParsedBlock {
|
||||
flavour: BlockFlavour::Divider,
|
||||
block_type: None,
|
||||
content: String::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
});
|
||||
}
|
||||
Event::Start(Tag::Strong) => current_text.push_str("**"),
|
||||
Event::End(TagEnd::Strong) => current_text.push_str("**"),
|
||||
Event::Start(Tag::Emphasis) => current_text.push('_'),
|
||||
Event::End(TagEnd::Emphasis) => current_text.push('_'),
|
||||
Event::Start(Tag::Strikethrough) => current_text.push_str("~~"),
|
||||
Event::End(TagEnd::Strikethrough) => current_text.push_str("~~"),
|
||||
Event::Start(Tag::Link { dest_url, .. }) => {
|
||||
current_text.push('[');
|
||||
pending_link_url = Some(dest_url.to_string());
|
||||
}
|
||||
Event::End(TagEnd::Link) => {
|
||||
if let Some(url) = pending_link_url.take() {
|
||||
current_text.push_str(&format!("]({})", url));
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush any remaining content
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type,
|
||||
current_checked,
|
||||
None,
|
||||
);
|
||||
|
||||
blocks
|
||||
}
|
||||
|
||||
fn flush_block(
|
||||
blocks: &mut Vec<ParsedBlock>,
|
||||
text: &mut String,
|
||||
flavour: BlockFlavour,
|
||||
block_type: Option<BlockType>,
|
||||
checked: Option<bool>,
|
||||
language: Option<String>,
|
||||
) {
|
||||
let trimmed = text.trim();
|
||||
if !trimmed.is_empty() || flavour == BlockFlavour::Divider {
|
||||
blocks.push(ParsedBlock {
|
||||
flavour,
|
||||
block_type,
|
||||
content: trimmed.to_string(),
|
||||
checked,
|
||||
language,
|
||||
});
|
||||
}
|
||||
text.clear();
|
||||
}
|
||||
|
||||
fn flush_code_block(blocks: &mut Vec<ParsedBlock>, text: &mut String, language: &str) {
|
||||
// Preserve leading whitespace (indentation) in code blocks as it may be
|
||||
// semantically significant (e.g., Python, YAML). Only strip leading/trailing
|
||||
// newlines which are typically artifacts from code fence parsing.
|
||||
let content = text.trim_matches('\n');
|
||||
if !content.is_empty() {
|
||||
blocks.push(ParsedBlock {
|
||||
flavour: BlockFlavour::Code,
|
||||
block_type: None,
|
||||
content: content.to_string(),
|
||||
checked: None,
|
||||
language: if language.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(language.to_string())
|
||||
},
|
||||
});
|
||||
}
|
||||
text.clear();
|
||||
}
|
||||
|
||||
/// Extracts the title from the first H1 heading in markdown content.
|
||||
///
|
||||
/// Returns "Untitled" if no H1 heading is found.
|
||||
pub(crate) fn extract_title(markdown: &str) -> String {
|
||||
let parser = Parser::new(markdown);
|
||||
let mut in_heading = false;
|
||||
let mut title = String::new();
|
||||
|
||||
for event in parser {
|
||||
match event {
|
||||
Event::Start(Tag::Heading {
|
||||
level: HeadingLevel::H1,
|
||||
..
|
||||
}) => {
|
||||
in_heading = true;
|
||||
}
|
||||
Event::Text(text) if in_heading => {
|
||||
title.push_str(&text);
|
||||
}
|
||||
Event::Code(code) if in_heading => {
|
||||
title.push_str(&code);
|
||||
}
|
||||
Event::End(TagEnd::Heading(_)) if in_heading => {
|
||||
break;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if title.is_empty() {
|
||||
"Untitled".to_string()
|
||||
} else {
|
||||
title.trim().to_string()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_simple() {
|
||||
assert_eq!(extract_title("# Hello World\n\nContent"), "Hello World");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_with_code() {
|
||||
assert_eq!(extract_title("# Hello `code` World"), "Hello code World");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_empty() {
|
||||
assert_eq!(extract_title("No heading here"), "Untitled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_h2_not_used() {
|
||||
assert_eq!(extract_title("## H2 heading\n\nContent"), "Untitled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_simple() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\nParagraph text.", true);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert_eq!(blocks[0].flavour, BlockFlavour::Paragraph);
|
||||
assert_eq!(blocks[0].content, "Paragraph text.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_with_headings() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\n## Section\n\nText.", true);
|
||||
assert_eq!(blocks.len(), 2);
|
||||
assert_eq!(blocks[0].block_type, Some(BlockType::H2));
|
||||
assert_eq!(blocks[0].content, "Section");
|
||||
assert_eq!(blocks[1].content, "Text.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_lists() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\n- Item 1\n- Item 2", true);
|
||||
assert_eq!(blocks.len(), 2);
|
||||
assert_eq!(blocks[0].flavour, BlockFlavour::List);
|
||||
assert_eq!(blocks[0].block_type, Some(BlockType::Bulleted));
|
||||
assert_eq!(blocks[0].content, "Item 1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_task_list() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\n- [ ] Unchecked\n- [x] Checked", true);
|
||||
assert_eq!(blocks.len(), 2);
|
||||
assert_eq!(blocks[0].block_type, Some(BlockType::Todo));
|
||||
assert_eq!(blocks[0].checked, Some(false));
|
||||
assert_eq!(blocks[1].block_type, Some(BlockType::Todo));
|
||||
assert_eq!(blocks[1].checked, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_code() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\n```rust\nfn main() {}\n```", true);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert_eq!(blocks[0].flavour, BlockFlavour::Code);
|
||||
assert_eq!(blocks[0].language, Some("rust".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_divider() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\nBefore\n\n---\n\nAfter", true);
|
||||
assert_eq!(blocks.len(), 3);
|
||||
assert_eq!(blocks[1].flavour, BlockFlavour::Divider);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_code_preserves_indentation() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\n```python\n def indented():\n pass\n```", true);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert!(blocks[0].content.starts_with(" def"));
|
||||
}
|
||||
}
|
||||
@@ -1,19 +1,21 @@
|
||||
mod affine;
|
||||
mod block_spec;
|
||||
mod blocksuite;
|
||||
mod delta_markdown;
|
||||
#[cfg(feature = "ydoc-loader")]
|
||||
mod markdown_to_ydoc;
|
||||
#[cfg(feature = "ydoc-loader")]
|
||||
mod markdown_utils;
|
||||
#[cfg(feature = "ydoc-loader")]
|
||||
mod update_ydoc;
|
||||
mod doc_loader;
|
||||
mod error;
|
||||
mod markdown;
|
||||
mod read;
|
||||
#[cfg(test)]
|
||||
mod roundtrip_tests;
|
||||
mod schema;
|
||||
mod table;
|
||||
mod value;
|
||||
mod write;
|
||||
|
||||
pub use affine::{
|
||||
BlockInfo, CrawlResult, MarkdownResult, PageDocContent, ParseError, WorkspaceDocContent, add_doc_to_root_doc,
|
||||
get_doc_ids_from_binary, parse_doc_from_binary, parse_doc_to_markdown, parse_page_doc, parse_workspace_doc,
|
||||
pub use error::ParseError;
|
||||
pub use read::{
|
||||
BlockInfo, CrawlResult, MarkdownResult, PageDocContent, WorkspaceDocContent, get_doc_ids_from_binary,
|
||||
parse_doc_from_binary, parse_doc_to_markdown, parse_page_doc, parse_workspace_doc,
|
||||
};
|
||||
pub use write::{
|
||||
add_doc_to_root_doc, build_full_doc, update_doc, update_doc_properties, update_doc_title, update_root_doc_meta_title,
|
||||
};
|
||||
#[cfg(feature = "ydoc-loader")]
|
||||
pub use markdown_to_ydoc::markdown_to_ydoc;
|
||||
#[cfg(feature = "ydoc-loader")]
|
||||
pub use update_ydoc::update_ydoc;
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use y_octo::{Any, Map, Value};
|
||||
|
||||
use super::{text_content, text_content_for_summary};
|
||||
use crate::doc_parser::{
|
||||
blocksuite::{DocContext, collect_child_ids, get_string},
|
||||
markdown::{
|
||||
DeltaToMdOptions, InlineReferencePayload, delta_value_to_inline_markdown, extract_inline_references_from_value,
|
||||
text_to_inline_markdown,
|
||||
},
|
||||
table::{MarkdownTableOptions, render_markdown_table},
|
||||
value::{any_as_string, value_to_string},
|
||||
};
|
||||
|
||||
pub(super) struct DatabaseTable {
|
||||
pub(super) columns: Vec<DatabaseColumn>,
|
||||
pub(super) rows: Vec<Vec<String>>,
|
||||
}
|
||||
|
||||
pub(super) struct DatabaseOption {
|
||||
id: Option<String>,
|
||||
value: Option<String>,
|
||||
color: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) struct DatabaseColumn {
|
||||
pub(super) id: String,
|
||||
pub(super) name: Option<String>,
|
||||
pub(super) col_type: String,
|
||||
pub(super) options: Vec<DatabaseOption>,
|
||||
}
|
||||
|
||||
pub(super) fn build_database_table(
|
||||
block: &Map,
|
||||
context: &DocContext,
|
||||
md_options: &DeltaToMdOptions,
|
||||
) -> Option<DatabaseTable> {
|
||||
let columns = parse_database_columns(block)?;
|
||||
let cells_map = block.get("prop:cells").and_then(|v| v.to_map())?;
|
||||
let child_ids = collect_child_ids(block);
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for child_id in child_ids {
|
||||
let row_cells = cells_map.get(&child_id).and_then(|v| v.to_map());
|
||||
let mut row = Vec::new();
|
||||
|
||||
for column in columns.iter() {
|
||||
let mut cell_text = String::new();
|
||||
if column.col_type == "title" {
|
||||
if let Some(child_block) = context.block_pool.get(&child_id) {
|
||||
if let Some(text_md) = text_to_inline_markdown(child_block, "prop:text", md_options) {
|
||||
cell_text = text_md;
|
||||
} else if let Some((text, _)) = text_content(child_block, "prop:text") {
|
||||
cell_text = text;
|
||||
} else if let Some((text, _)) = text_content_for_summary(child_block, "prop:text") {
|
||||
cell_text = text;
|
||||
}
|
||||
}
|
||||
} else if let Some(row_cells) = &row_cells
|
||||
&& let Some(cell_val) = row_cells.get(&column.id).and_then(|v| v.to_map())
|
||||
&& let Some(value) = cell_val.get("value")
|
||||
{
|
||||
if let Some(text_md) = delta_value_to_inline_markdown(&value, md_options) {
|
||||
cell_text = text_md;
|
||||
} else {
|
||||
cell_text = format_cell_value(&value, column);
|
||||
}
|
||||
}
|
||||
|
||||
row.push(cell_text);
|
||||
}
|
||||
if row.iter().any(|cell| !cell.is_empty()) {
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
Some(DatabaseTable { columns, rows })
|
||||
}
|
||||
|
||||
pub(super) fn database_table_markdown(table: DatabaseTable) -> Option<String> {
|
||||
let mut rows = Vec::with_capacity(table.rows.len() + 1);
|
||||
let header = table
|
||||
.columns
|
||||
.iter()
|
||||
.map(|column| column.name.as_deref().unwrap_or_default().to_string())
|
||||
.collect::<Vec<_>>();
|
||||
rows.push(header);
|
||||
rows.extend(table.rows);
|
||||
|
||||
let options = MarkdownTableOptions::new(false, "<br />", true);
|
||||
render_markdown_table(&rows, options)
|
||||
}
|
||||
|
||||
pub(super) fn database_summary_text(block: &Map, context: &DocContext) -> Option<String> {
|
||||
let md_options = DeltaToMdOptions::new(None);
|
||||
let table = build_database_table(block, context, &md_options)?;
|
||||
let mut summary = String::new();
|
||||
|
||||
if let Some(title) = get_string(block, "prop:title")
|
||||
&& !title.is_empty()
|
||||
{
|
||||
summary.push_str(&title);
|
||||
summary.push('|');
|
||||
}
|
||||
|
||||
for column in table.columns.iter() {
|
||||
if let Some(name) = column.name.as_ref()
|
||||
&& !name.is_empty()
|
||||
{
|
||||
summary.push_str(name);
|
||||
summary.push('|');
|
||||
}
|
||||
for option in column.options.iter() {
|
||||
if let Some(value) = option.value.as_ref()
|
||||
&& !value.is_empty()
|
||||
{
|
||||
summary.push_str(value);
|
||||
summary.push('|');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for row in table.rows.iter() {
|
||||
for cell_text in row.iter() {
|
||||
if !cell_text.is_empty() {
|
||||
summary.push_str(cell_text);
|
||||
summary.push('|');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if summary.is_empty() { None } else { Some(summary) }
|
||||
}
|
||||
|
||||
pub(super) fn gather_database_texts(block: &Map) -> (Vec<String>, Option<String>) {
|
||||
let mut texts = Vec::new();
|
||||
let database_title = get_string(block, "prop:title");
|
||||
if let Some(title) = &database_title {
|
||||
texts.push(title.clone());
|
||||
}
|
||||
|
||||
if let Some(columns) = parse_database_columns(block) {
|
||||
for column in columns.iter() {
|
||||
if let Some(name) = column.name.as_ref() {
|
||||
texts.push(name.clone());
|
||||
}
|
||||
for option in column.options.iter() {
|
||||
if let Some(value) = option.value.as_ref() {
|
||||
texts.push(value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(texts, database_title)
|
||||
}
|
||||
|
||||
pub(super) fn collect_database_cell_references(block: &Map) -> Vec<InlineReferencePayload> {
|
||||
let cells_map = match block.get("prop:cells").and_then(|value| value.to_map()) {
|
||||
Some(map) => map,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
|
||||
let mut refs = Vec::new();
|
||||
let mut seen: HashSet<(String, String)> = HashSet::new();
|
||||
|
||||
for row in cells_map.values() {
|
||||
let Some(row_map) = row.to_map() else {
|
||||
continue;
|
||||
};
|
||||
for cell in row_map.values() {
|
||||
let Some(cell_map) = cell.to_map() else {
|
||||
continue;
|
||||
};
|
||||
let Some(value) = cell_map.get("value") else {
|
||||
continue;
|
||||
};
|
||||
for reference in extract_inline_references_from_value(&value) {
|
||||
let key = (reference.doc_id.clone(), reference.payload.clone());
|
||||
if seen.insert(key) {
|
||||
refs.push(reference);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
refs
|
||||
}
|
||||
|
||||
fn parse_database_columns(block: &Map) -> Option<Vec<DatabaseColumn>> {
|
||||
let columns = block.get("prop:columns").and_then(|value| value.to_array())?;
|
||||
let mut parsed = Vec::new();
|
||||
for column_value in columns.iter() {
|
||||
if let Some(column) = column_value.to_map() {
|
||||
let id = column
|
||||
.get("id")
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
let name = column.get("name").and_then(|value| value_to_string(&value));
|
||||
let col_type = column
|
||||
.get("type")
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
let options = parse_database_options(&column);
|
||||
parsed.push(DatabaseColumn {
|
||||
id,
|
||||
name,
|
||||
col_type,
|
||||
options,
|
||||
});
|
||||
}
|
||||
}
|
||||
Some(parsed)
|
||||
}
|
||||
|
||||
fn parse_database_options(column: &Map) -> Vec<DatabaseOption> {
|
||||
let Some(data) = column.get("data").and_then(|value| value.to_map()) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Some(options) = data.get("options").and_then(|value| value.to_array()) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut parsed = Vec::new();
|
||||
for option_value in options.iter() {
|
||||
if let Some(option) = option_value.to_map() {
|
||||
parsed.push(DatabaseOption {
|
||||
id: option.get("id").and_then(|value| value_to_string(&value)),
|
||||
value: option.get("value").and_then(|value| value_to_string(&value)),
|
||||
color: option.get("color").and_then(|value| value_to_string(&value)),
|
||||
});
|
||||
}
|
||||
}
|
||||
parsed
|
||||
}
|
||||
|
||||
fn format_option_tag(option: &DatabaseOption) -> String {
|
||||
let id = option.id.as_deref().unwrap_or_default();
|
||||
let value = option.value.as_deref().unwrap_or_default();
|
||||
let color = option.color.as_deref().unwrap_or_default();
|
||||
|
||||
format!("<span data-affine-option data-value=\"{id}\" data-option-color=\"{color}\">{value}</span>")
|
||||
}
|
||||
|
||||
fn format_cell_value(value: &Value, column: &DatabaseColumn) -> String {
|
||||
match column.col_type.as_str() {
|
||||
"select" => {
|
||||
let id = match value {
|
||||
Value::Any(any) => any_as_string(any).map(str::to_string),
|
||||
Value::Text(text) => Some(text.to_string()),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(id) = id {
|
||||
for option in column.options.iter() {
|
||||
if option.id.as_deref() == Some(id.as_str()) {
|
||||
return format_option_tag(option);
|
||||
}
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
"multi-select" => {
|
||||
let ids: Vec<String> = match value {
|
||||
Value::Any(Any::Array(ids)) => ids.iter().filter_map(any_as_string).map(str::to_string).collect(),
|
||||
Value::Array(array) => array.iter().filter_map(|id_val| value_to_string(&id_val)).collect(),
|
||||
_ => Vec::new(),
|
||||
};
|
||||
|
||||
if ids.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let mut selected = Vec::new();
|
||||
for id in ids.iter() {
|
||||
for option in column.options.iter() {
|
||||
if option.id.as_deref() == Some(id.as_str()) {
|
||||
selected.push(format_option_tag(option));
|
||||
}
|
||||
}
|
||||
}
|
||||
selected.join("")
|
||||
}
|
||||
_ => value_to_string(value).unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,795 @@
|
||||
mod database;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map as JsonMap, Value as JsonValue};
|
||||
use y_octo::{Any, Map};
|
||||
|
||||
use self::database::{
|
||||
build_database_table, collect_database_cell_references, database_summary_text, database_table_markdown,
|
||||
gather_database_texts,
|
||||
};
|
||||
use super::{
|
||||
ParseError,
|
||||
block_spec::{BlockFlavour, BlockSpec, ImageSpec},
|
||||
blocksuite::{DocContext, get_block_id, get_flavour, get_list_depth, get_string, nearest_by_flavour},
|
||||
doc_loader::load_doc,
|
||||
markdown::{DeltaToMdOptions, MarkdownRenderer, MarkdownWriter, extract_inline_references},
|
||||
schema::{NOTE_FLAVOUR, PAGE_FLAVOUR},
|
||||
value::{any_as_string, any_truthy, build_reference_payload, params_value_to_json, value_to_string},
|
||||
};
|
||||
|
||||
const SUMMARY_LIMIT: usize = 1000;
|
||||
const DEFAULT_PAGE_TITLE: &str = "Untitled";
|
||||
|
||||
const BOOKMARK_FLAVOURS: [&str; 6] = [
|
||||
"affine:bookmark",
|
||||
"affine:embed-youtube",
|
||||
"affine:embed-iframe",
|
||||
"affine:embed-figma",
|
||||
"affine:embed-github",
|
||||
"affine:embed-loom",
|
||||
];
|
||||
|
||||
struct SummaryBuilder {
|
||||
summary: String,
|
||||
remaining: Option<isize>,
|
||||
}
|
||||
|
||||
impl SummaryBuilder {
|
||||
fn new(limit: isize) -> Self {
|
||||
let remaining = if limit < 0 { None } else { Some(limit) };
|
||||
Self {
|
||||
summary: String::new(),
|
||||
remaining,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_unlimited(&self) -> bool {
|
||||
self.remaining.is_none()
|
||||
}
|
||||
|
||||
fn push_text(&mut self, text: &str, len: usize) {
|
||||
match self.remaining {
|
||||
None => self.summary.push_str(text),
|
||||
Some(remaining) if remaining > 0 => {
|
||||
self.summary.push_str(text);
|
||||
self.remaining = Some(remaining - len as isize);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn push_raw(&mut self, text: &str) {
|
||||
self.summary.push_str(text);
|
||||
}
|
||||
|
||||
fn into_string(self) -> String {
|
||||
self.summary
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BlockInfo {
|
||||
pub block_id: String,
|
||||
pub flavour: String,
|
||||
pub content: Option<Vec<String>>,
|
||||
pub blob: Option<Vec<String>>,
|
||||
pub ref_doc_id: Option<Vec<String>>,
|
||||
pub ref_info: Option<Vec<String>>,
|
||||
pub parent_flavour: Option<String>,
|
||||
pub parent_block_id: Option<String>,
|
||||
pub additional: Option<String>,
|
||||
}
|
||||
|
||||
impl BlockInfo {
|
||||
fn base(
|
||||
block_id: &str,
|
||||
flavour: &str,
|
||||
parent_flavour: Option<&String>,
|
||||
parent_block_id: Option<&String>,
|
||||
additional: Option<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
block_id: block_id.to_string(),
|
||||
flavour: flavour.to_string(),
|
||||
content: None,
|
||||
blob: None,
|
||||
ref_doc_id: None,
|
||||
ref_info: None,
|
||||
parent_flavour: parent_flavour.cloned(),
|
||||
parent_block_id: parent_block_id.cloned(),
|
||||
additional,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CrawlResult {
|
||||
pub blocks: Vec<BlockInfo>,
|
||||
pub title: String,
|
||||
pub summary: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PageDocContent {
|
||||
pub title: String,
|
||||
pub summary: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WorkspaceDocContent {
|
||||
pub name: String,
|
||||
#[serde(rename = "avatarKey")]
|
||||
pub avatar_key: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MarkdownResult {
|
||||
pub title: String,
|
||||
pub markdown: String,
|
||||
}
|
||||
|
||||
pub fn parse_workspace_doc(doc_bin: Vec<u8>) -> Result<Option<WorkspaceDocContent>, ParseError> {
|
||||
let doc = load_doc(&doc_bin, None)?;
|
||||
|
||||
let meta = match doc.get_map("meta") {
|
||||
Ok(meta) => meta,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
|
||||
let name = get_string(&meta, "name").unwrap_or_default();
|
||||
let avatar_key = get_string(&meta, "avatar").unwrap_or_default();
|
||||
|
||||
Ok(Some(WorkspaceDocContent { name, avatar_key }))
|
||||
}
|
||||
|
||||
pub fn parse_page_doc(
|
||||
doc_bin: Vec<u8>,
|
||||
max_summary_length: Option<isize>,
|
||||
) -> Result<Option<PageDocContent>, ParseError> {
|
||||
let doc = load_doc(&doc_bin, None)?;
|
||||
|
||||
let blocks_map = match doc.get_map("blocks") {
|
||||
Ok(map) => map,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
|
||||
if blocks_map.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let Some(context) = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let mut walker = context.walker();
|
||||
let mut content = PageDocContent {
|
||||
title: context
|
||||
.block_pool
|
||||
.get(&context.root_block_id)
|
||||
.and_then(|block| get_string(block, "prop:title"))
|
||||
.unwrap_or_default(),
|
||||
summary: String::new(),
|
||||
};
|
||||
let mut summary = SummaryBuilder::new(max_summary_length.unwrap_or(150));
|
||||
|
||||
while let Some((_parent_block_id, block_id)) = walker.next() {
|
||||
let Some(block) = context.block_pool.get(&block_id) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let Some(flavour) = get_flavour(block) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match flavour.as_str() {
|
||||
"affine:page" | "affine:note" => {
|
||||
walker.enqueue_children(&block_id, block);
|
||||
}
|
||||
"affine:attachment" | "affine:transcription" | "affine:callout" => {
|
||||
if summary.is_unlimited() {
|
||||
walker.enqueue_children(&block_id, block);
|
||||
}
|
||||
}
|
||||
"affine:database" => {
|
||||
if summary.is_unlimited()
|
||||
&& let Some(text) = database_summary_text(block, &context)
|
||||
{
|
||||
summary.push_raw(&text);
|
||||
}
|
||||
}
|
||||
"affine:table" => {
|
||||
if summary.is_unlimited() {
|
||||
let contents = table_cell_texts(block);
|
||||
if !contents.is_empty() {
|
||||
summary.push_raw(&contents.join("|"));
|
||||
}
|
||||
}
|
||||
}
|
||||
"affine:paragraph" | "affine:list" | "affine:code" => {
|
||||
walker.enqueue_children(&block_id, block);
|
||||
if let Some((text, len)) = text_content_for_summary(block, "prop:text") {
|
||||
summary.push_text(&text, len);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
content.summary = summary.into_string();
|
||||
Ok(Some(content))
|
||||
}
|
||||
|
||||
pub fn parse_doc_to_markdown(
|
||||
doc_bin: Vec<u8>,
|
||||
doc_id: String,
|
||||
ai_editable: bool,
|
||||
doc_url_prefix: Option<String>,
|
||||
) -> Result<MarkdownResult, ParseError> {
|
||||
let doc = load_doc(&doc_bin, Some(doc_id.as_str()))?;
|
||||
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Ok(MarkdownResult {
|
||||
title: "".into(),
|
||||
markdown: "".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("root block not found".into()))?;
|
||||
let root_block_id = context.root_block_id.clone();
|
||||
let mut walker = context.walker();
|
||||
let mut doc_title = String::from(DEFAULT_PAGE_TITLE);
|
||||
let mut markdown = String::new();
|
||||
let md_options = DeltaToMdOptions::new(doc_url_prefix);
|
||||
let renderer = MarkdownRenderer::new(&md_options);
|
||||
|
||||
while let Some((_parent_block_id, block_id)) = walker.next() {
|
||||
let block = match context.block_pool.get(&block_id) {
|
||||
Some(block) => block,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flavour = match get_flavour(block) {
|
||||
Some(flavour) => flavour,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let parent_id = context.parent_lookup.get(&block_id);
|
||||
let parent_flavour = parent_id
|
||||
.and_then(|id| context.block_pool.get(id))
|
||||
.and_then(get_flavour);
|
||||
|
||||
if parent_flavour.as_deref() == Some("affine:database") {
|
||||
continue;
|
||||
}
|
||||
|
||||
// enqueue children first to keep traversal order similar to JS implementation
|
||||
walker.enqueue_children(&block_id, block);
|
||||
|
||||
if flavour == PAGE_FLAVOUR {
|
||||
let title = get_string(block, "prop:title").unwrap_or_default();
|
||||
doc_title = title.clone();
|
||||
continue;
|
||||
}
|
||||
|
||||
let block_level = if ai_editable {
|
||||
block_level(&block_id, &root_block_id, &context.parent_lookup)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let ai_block = ai_editable && block_level == 2;
|
||||
|
||||
let mut block_markdown = String::new();
|
||||
|
||||
match flavour.as_str() {
|
||||
"affine:database" => {
|
||||
let title = get_string(block, "prop:title").unwrap_or_default();
|
||||
block_markdown.push_str(&format!("\n### {title}\n"));
|
||||
|
||||
if let Some(table) = build_database_table(block, &context, &md_options)
|
||||
&& let Some(table_md) = database_table_markdown(table)
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut block_markdown);
|
||||
writer.push_table(&table_md);
|
||||
}
|
||||
}
|
||||
"affine:note" | "affine:surface" | "affine:frame" => {}
|
||||
_ => {
|
||||
if let Some(block_flavour) = BlockFlavour::from_str(flavour.as_str()) {
|
||||
let spec = BlockSpec::from_block_map_with_flavour(block, block_flavour);
|
||||
let list_depth = if block_flavour == BlockFlavour::List {
|
||||
get_list_depth(&block_id, &context.parent_lookup, &context.block_pool)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
renderer.write_block(&mut block_markdown, &spec, list_depth);
|
||||
} else {
|
||||
return Err(ParseError::ParserError(format!("unsupported_block_flavour:{flavour}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ai_block {
|
||||
markdown.push_str(&format!("<!-- block_id={block_id} flavour={flavour} -->\n"));
|
||||
}
|
||||
markdown.push_str(&block_markdown);
|
||||
}
|
||||
|
||||
Ok(MarkdownResult {
|
||||
title: doc_title,
|
||||
markdown,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlResult, ParseError> {
|
||||
let doc = load_doc(&doc_bin, Some(doc_id.as_str()))?;
|
||||
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Err(ParseError::ParserError("blocks map is empty".into()));
|
||||
}
|
||||
|
||||
let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("root block not found".into()))?;
|
||||
let mut walker = context.walker();
|
||||
let mut blocks: Vec<BlockInfo> = Vec::with_capacity(context.block_pool.len());
|
||||
let mut doc_title = String::new();
|
||||
let mut summary = SummaryBuilder::new(SUMMARY_LIMIT as isize);
|
||||
|
||||
while let Some((parent_block_id, block_id)) = walker.next() {
|
||||
let block = match context.block_pool.get(&block_id) {
|
||||
Some(block) => block,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flavour = match get_flavour(block) {
|
||||
Some(flavour) => flavour,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let parent_block = parent_block_id.as_ref().and_then(|id| context.block_pool.get(id));
|
||||
let parent_flavour = parent_block.and_then(get_flavour);
|
||||
|
||||
let note_block = nearest_by_flavour(&block_id, NOTE_FLAVOUR, &context.parent_lookup, &context.block_pool);
|
||||
let note_block_id = note_block.as_ref().and_then(get_block_id);
|
||||
let display_mode = determine_display_mode(note_block.as_ref());
|
||||
|
||||
// enqueue children first to keep traversal order similar to JS implementation
|
||||
walker.enqueue_children(&block_id, block);
|
||||
|
||||
let build_block = |database_name: Option<&String>| {
|
||||
BlockInfo::base(
|
||||
&block_id,
|
||||
&flavour,
|
||||
parent_flavour.as_ref(),
|
||||
parent_block_id.as_ref(),
|
||||
compose_additional(&display_mode, note_block_id.as_ref(), database_name),
|
||||
)
|
||||
};
|
||||
|
||||
if flavour == PAGE_FLAVOUR {
|
||||
let title = get_string(block, "prop:title").unwrap_or_default();
|
||||
doc_title = title.clone();
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(vec![title]);
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if matches!(flavour.as_str(), "affine:paragraph" | "affine:list" | "affine:code") {
|
||||
if let Some(text) = block.get("prop:text").and_then(|value| value.to_text()) {
|
||||
let database_name = if flavour == "affine:paragraph" && parent_flavour.as_deref() == Some("affine:database") {
|
||||
parent_block.and_then(|map| get_string(map, "prop:title"))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let content = text.to_string();
|
||||
let text_len = text.len() as usize;
|
||||
let refs = extract_inline_references(&text.to_delta());
|
||||
|
||||
let mut info = build_block(database_name.as_ref());
|
||||
info.content = Some(vec![content.clone()]);
|
||||
if !refs.is_empty() {
|
||||
info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect());
|
||||
info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect());
|
||||
}
|
||||
blocks.push(info);
|
||||
summary.push_text(&content, text_len);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if matches!(flavour.as_str(), "affine:embed-linked-doc" | "affine:embed-synced-doc") {
|
||||
if let Some(page_id) = get_string(block, "prop:pageId") {
|
||||
let mut info = build_block(None);
|
||||
let payload = embed_ref_payload(block, &page_id);
|
||||
apply_doc_ref(&mut info, page_id, payload);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:attachment" {
|
||||
if let Some(blob_id) = get_string(block, "prop:sourceId") {
|
||||
let mut info = build_block(None);
|
||||
let name = get_string(block, "prop:name").unwrap_or_default();
|
||||
apply_blob_info(&mut info, blob_id, name);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:image" {
|
||||
let image = ImageSpec::from_block_map(block);
|
||||
if !image.source_id.is_empty() {
|
||||
let mut info = build_block(None);
|
||||
let caption = image.caption.unwrap_or_default();
|
||||
apply_blob_info(&mut info, image.source_id, caption);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:surface" {
|
||||
let texts = gather_surface_texts(block);
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(texts);
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:database" {
|
||||
let (texts, database_name) = gather_database_texts(block);
|
||||
let mut info = BlockInfo::base(
|
||||
&block_id,
|
||||
&flavour,
|
||||
parent_flavour.as_ref(),
|
||||
parent_block_id.as_ref(),
|
||||
compose_additional(&display_mode, note_block_id.as_ref(), database_name.as_ref()),
|
||||
);
|
||||
info.content = Some(texts);
|
||||
let refs = collect_database_cell_references(block);
|
||||
if !refs.is_empty() {
|
||||
info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect());
|
||||
info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect());
|
||||
}
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:latex" {
|
||||
if let Some(content) = get_string(block, "prop:latex") {
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(vec![content]);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:table" {
|
||||
let contents = table_cell_texts(block);
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(contents);
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if BOOKMARK_FLAVOURS.contains(&flavour.as_str()) {
|
||||
blocks.push(build_block(None));
|
||||
}
|
||||
}
|
||||
|
||||
if doc_title.is_empty() {
|
||||
doc_title = DEFAULT_PAGE_TITLE.into();
|
||||
}
|
||||
|
||||
Ok(CrawlResult {
|
||||
blocks,
|
||||
title: doc_title,
|
||||
summary: summary.into_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_doc_ids_from_binary(doc_bin: Vec<u8>, include_trash: bool) -> Result<Vec<String>, ParseError> {
|
||||
let doc = load_doc(&doc_bin, None)?;
|
||||
|
||||
let mut doc_ids = Vec::new();
|
||||
let meta = doc.get_map("meta")?;
|
||||
let pages_value = meta.get("pages");
|
||||
if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) {
|
||||
for page_val in pages.iter() {
|
||||
if let Some(page) = page_val.to_map() {
|
||||
let id = get_string(&page, "id");
|
||||
if let Some(id) = id {
|
||||
let trash = page
|
||||
.get("trash")
|
||||
.and_then(|v| match v.to_any() {
|
||||
Some(Any::True) => Some(true),
|
||||
Some(Any::False) => Some(false),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or(false);
|
||||
|
||||
if include_trash || !trash {
|
||||
doc_ids.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Ok(doc_ids);
|
||||
}
|
||||
|
||||
if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) {
|
||||
for entry in entries {
|
||||
let Any::Object(map) = entry else {
|
||||
continue;
|
||||
};
|
||||
let id = map.get("id").and_then(any_as_string).map(str::to_string);
|
||||
if let Some(id) = id {
|
||||
let trash = map.get("trash").map(any_truthy).unwrap_or(false);
|
||||
if include_trash || !trash {
|
||||
doc_ids.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(doc_ids)
|
||||
}
|
||||
|
||||
fn block_level(block_id: &str, root_id: &str, parent_lookup: &HashMap<String, String>) -> usize {
|
||||
let mut level = 0;
|
||||
let mut cursor = block_id;
|
||||
while let Some(parent) = parent_lookup.get(cursor) {
|
||||
level += 1;
|
||||
if parent == root_id {
|
||||
break;
|
||||
}
|
||||
cursor = parent;
|
||||
}
|
||||
level
|
||||
}
|
||||
|
||||
pub(super) fn text_content(block: &Map, key: &str) -> Option<(String, usize)> {
|
||||
block.get(key).and_then(|value| {
|
||||
value.to_text().map(|text| {
|
||||
let content = text.to_string();
|
||||
let len = text.len() as usize;
|
||||
(content, len)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn determine_display_mode(note_block: Option<&Map>) -> String {
|
||||
match note_block.and_then(|block| get_string(block, "prop:displayMode")) {
|
||||
Some(mode) if mode == "both" => "page".into(),
|
||||
Some(mode) => mode,
|
||||
None => "edgeless".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn compose_additional(
|
||||
display_mode: &str,
|
||||
note_block_id: Option<&String>,
|
||||
database_name: Option<&String>,
|
||||
) -> Option<String> {
|
||||
let mut payload = JsonMap::new();
|
||||
payload.insert("displayMode".into(), JsonValue::String(display_mode.to_string()));
|
||||
if let Some(note_id) = note_block_id {
|
||||
payload.insert("noteBlockId".into(), JsonValue::String(note_id.clone()));
|
||||
}
|
||||
if let Some(name) = database_name {
|
||||
payload.insert("databaseName".into(), JsonValue::String(name.clone()));
|
||||
}
|
||||
Some(JsonValue::Object(payload).to_string())
|
||||
}
|
||||
|
||||
fn apply_blob_info(info: &mut BlockInfo, blob_id: String, content: String) {
|
||||
info.blob = Some(vec![blob_id]);
|
||||
info.content = Some(vec![content]);
|
||||
}
|
||||
|
||||
fn apply_doc_ref(info: &mut BlockInfo, page_id: String, payload: Option<String>) {
|
||||
info.ref_doc_id = Some(vec![page_id]);
|
||||
if let Some(payload) = payload {
|
||||
info.ref_info = Some(vec![payload]);
|
||||
}
|
||||
}
|
||||
|
||||
fn embed_ref_payload(block: &Map, page_id: &str) -> Option<String> {
|
||||
let params = block.get("prop:params").as_ref().and_then(params_value_to_json);
|
||||
Some(build_reference_payload(page_id, params))
|
||||
}
|
||||
|
||||
fn gather_surface_texts(block: &Map) -> Vec<String> {
|
||||
let mut texts = Vec::new();
|
||||
let elements = match block.get("prop:elements").and_then(|value| value.to_map()) {
|
||||
Some(map) => map,
|
||||
None => return texts,
|
||||
};
|
||||
|
||||
if elements
|
||||
.get("type")
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.as_deref()
|
||||
!= Some("$blocksuite:internal:native$")
|
||||
{
|
||||
return texts;
|
||||
}
|
||||
|
||||
if let Some(value_map) = elements.get("value").and_then(|value| value.to_map()) {
|
||||
for value in value_map.values() {
|
||||
if let Some(element) = value.to_map()
|
||||
&& let Some(text) = element.get("text").and_then(|value| value.to_text())
|
||||
{
|
||||
texts.push(text.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
texts.sort();
|
||||
texts
|
||||
}
|
||||
|
||||
fn table_cell_texts(block: &Map) -> Vec<String> {
|
||||
let mut contents = Vec::new();
|
||||
for key in block.keys() {
|
||||
if key.starts_with("prop:cells.")
|
||||
&& key.ends_with(".text")
|
||||
&& let Some(value) = block.get(key).and_then(|value| value_to_string(&value))
|
||||
&& !value.is_empty()
|
||||
{
|
||||
contents.push(value);
|
||||
}
|
||||
}
|
||||
contents
|
||||
}
|
||||
|
||||
pub(super) fn text_content_for_summary(block: &Map, key: &str) -> Option<(String, usize)> {
|
||||
if let Some((text, len)) = text_content(block, key) {
|
||||
return Some((text, len));
|
||||
}
|
||||
|
||||
block.get(key).and_then(|value| {
|
||||
value_to_string(&value).map(|text| {
|
||||
let len = text.chars().count();
|
||||
(text, len)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
use y_octo::{AHashMap, Any, DocOptions, TextAttributes, TextDeltaOp, TextInsert, Value};
|
||||
|
||||
use super::*;
|
||||
use crate::doc_parser::build_full_doc;
|
||||
|
||||
#[test]
|
||||
fn test_parse_doc_from_binary() {
|
||||
let json = include_bytes!("../../../fixtures/demo.ydoc.json");
|
||||
let doc_bin = include_bytes!("../../../fixtures/demo.ydoc").to_vec();
|
||||
let doc_id = "dYpV7PPhk8amRkY5IAcVO".to_string();
|
||||
|
||||
let result = parse_doc_from_binary(doc_bin, doc_id).unwrap();
|
||||
let config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict)
|
||||
.numeric_mode(assert_json_diff::NumericMode::AssumeFloat);
|
||||
assert_json_diff::assert_json_matches!(
|
||||
serde_json::from_slice::<serde_json::Value>(json).unwrap(),
|
||||
serde_json::json!(result),
|
||||
config
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_database_cell_references() {
|
||||
let doc_id = "doc-with-db".to_string();
|
||||
let doc = DocOptions::new().with_guid(doc_id.clone()).build();
|
||||
let mut blocks = doc.get_or_create_map("blocks").unwrap();
|
||||
|
||||
let mut page = doc.create_map().unwrap();
|
||||
page.insert("sys:id".into(), "page").unwrap();
|
||||
page.insert("sys:flavour".into(), "affine:page").unwrap();
|
||||
let mut page_children = doc.create_array().unwrap();
|
||||
page_children.push("note").unwrap();
|
||||
page.insert("sys:children".into(), Value::Array(page_children)).unwrap();
|
||||
let mut page_title = doc.create_text().unwrap();
|
||||
page_title.insert(0, "Page").unwrap();
|
||||
page.insert("prop:title".into(), Value::Text(page_title)).unwrap();
|
||||
blocks.insert("page".into(), Value::Map(page)).unwrap();
|
||||
|
||||
let mut note = doc.create_map().unwrap();
|
||||
note.insert("sys:id".into(), "note").unwrap();
|
||||
note.insert("sys:flavour".into(), "affine:note").unwrap();
|
||||
let mut note_children = doc.create_array().unwrap();
|
||||
note_children.push("db").unwrap();
|
||||
note.insert("sys:children".into(), Value::Array(note_children)).unwrap();
|
||||
note.insert("prop:displayMode".into(), "page").unwrap();
|
||||
blocks.insert("note".into(), Value::Map(note)).unwrap();
|
||||
|
||||
let mut db = doc.create_map().unwrap();
|
||||
db.insert("sys:id".into(), "db").unwrap();
|
||||
db.insert("sys:flavour".into(), "affine:database").unwrap();
|
||||
db.insert("sys:children".into(), Value::Array(doc.create_array().unwrap()))
|
||||
.unwrap();
|
||||
let mut db_title = doc.create_text().unwrap();
|
||||
db_title.insert(0, "Database").unwrap();
|
||||
db.insert("prop:title".into(), Value::Text(db_title)).unwrap();
|
||||
|
||||
let mut columns = doc.create_array().unwrap();
|
||||
let mut column = doc.create_map().unwrap();
|
||||
column.insert("id".into(), "col1").unwrap();
|
||||
column.insert("name".into(), "Text").unwrap();
|
||||
column.insert("type".into(), "rich-text").unwrap();
|
||||
column
|
||||
.insert("data".into(), Value::Map(doc.create_map().unwrap()))
|
||||
.unwrap();
|
||||
columns.push(Value::Map(column)).unwrap();
|
||||
db.insert("prop:columns".into(), Value::Array(columns)).unwrap();
|
||||
|
||||
let mut cell_text = doc.create_text().unwrap();
|
||||
let mut reference = AHashMap::default();
|
||||
reference.insert("pageId".into(), Any::String("target-doc".into()));
|
||||
let mut params = AHashMap::default();
|
||||
params.insert("mode".into(), Any::String("page".into()));
|
||||
reference.insert("params".into(), Any::Object(params));
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert("reference".into(), Any::Object(reference));
|
||||
cell_text
|
||||
.apply_delta(&[
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("See ".into()),
|
||||
format: None,
|
||||
},
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Target".into()),
|
||||
format: Some(attrs),
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let mut cell = doc.create_map().unwrap();
|
||||
cell.insert("columnId".into(), "col1").unwrap();
|
||||
cell.insert("value".into(), Value::Text(cell_text)).unwrap();
|
||||
let mut row = doc.create_map().unwrap();
|
||||
row.insert("col1".into(), Value::Map(cell)).unwrap();
|
||||
let mut cells = doc.create_map().unwrap();
|
||||
cells.insert("row1".into(), Value::Map(row)).unwrap();
|
||||
db.insert("prop:cells".into(), Value::Map(cells)).unwrap();
|
||||
|
||||
blocks.insert("db".into(), Value::Map(db)).unwrap();
|
||||
|
||||
let doc_bin = doc.encode_update_v1().unwrap();
|
||||
let result = parse_doc_from_binary(doc_bin, doc_id).unwrap();
|
||||
let db_block = result.blocks.iter().find(|block| block.block_id == "db").unwrap();
|
||||
assert_eq!(db_block.ref_doc_id, Some(vec!["target-doc".to_string()]));
|
||||
assert_eq!(
|
||||
db_block.ref_info,
|
||||
Some(vec![build_reference_payload(
|
||||
"target-doc",
|
||||
Some(json!({"mode": "page"}))
|
||||
)])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_doc_to_markdown_ai_editable_image_table() {
|
||||
let doc_id = "ai-editable-doc";
|
||||
let markdown = "\n\n| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let doc_bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let result = parse_doc_to_markdown(doc_bin, doc_id.to_string(), true, None).expect("parse doc");
|
||||
let md = result.markdown;
|
||||
|
||||
assert!(md.contains("flavour=affine:image"));
|
||||
assert!(md.contains("blob://image-id"));
|
||||
assert!(md.contains("|A|B|"));
|
||||
assert!(md.contains("|---|---|"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
use super::{build_full_doc, parse_doc_to_markdown};
|
||||
|
||||
fn assert_markdown_roundtrip(markdown: &str, expected: &str) {
|
||||
let doc_id = "roundtrip-doc";
|
||||
let title = "Roundtrip Title";
|
||||
let bin = build_full_doc(title, markdown, doc_id).expect("create doc");
|
||||
let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc");
|
||||
assert_eq!(result.title, title);
|
||||
assert_eq!(result.markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_inline_styles() {
|
||||
let markdown = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com).";
|
||||
let expected = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com).\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_list_items() {
|
||||
let markdown = "- Item 1\n- Item 2\n- [ ] Task\n- [x] Done";
|
||||
let expected = "* Item 1\n* Item 2\n- [ ] Task\n- [x] Done\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_code_block() {
|
||||
let markdown = "```rust\nfn main() {}\n```";
|
||||
let expected = "```rust\nfn main() {}\n\n```\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_code_block_indentation() {
|
||||
let markdown = "```python\n def indented():\n return \"ok\"\n```";
|
||||
let doc_id = "roundtrip-indent";
|
||||
let title = "Roundtrip Title";
|
||||
let bin = build_full_doc(title, markdown, doc_id).expect("create doc");
|
||||
let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc");
|
||||
assert!(result.markdown.contains("\n def indented():"));
|
||||
assert!(result.markdown.contains("\n return \"ok\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_table() {
|
||||
let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let expected = "|A|B|\n|---|---|\n|1|2|\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_image_with_caption() {
|
||||
let markdown = "";
|
||||
let expected = "<img\n src=\"blob://image-id\"\n alt=\"Alt\"\n width=\"auto\"\n height=\"auto\"\n/>\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
pub(super) const PAGE_FLAVOUR: &str = "affine:page";
|
||||
pub(super) const NOTE_FLAVOUR: &str = "affine:note";
|
||||
pub(super) const SURFACE_FLAVOUR: &str = "affine:surface";
|
||||
|
||||
pub(super) const SYS_ID: &str = "sys:id";
|
||||
pub(super) const SYS_FLAVOUR: &str = "sys:flavour";
|
||||
pub(super) const SYS_VERSION: &str = "sys:version";
|
||||
pub(super) const SYS_CHILDREN: &str = "sys:children";
|
||||
|
||||
pub(super) const PROP_TITLE: &str = "prop:title";
|
||||
pub(super) const PROP_TEXT: &str = "prop:text";
|
||||
pub(super) const PROP_TYPE: &str = "prop:type";
|
||||
pub(super) const PROP_CHECKED: &str = "prop:checked";
|
||||
pub(super) const PROP_LANGUAGE: &str = "prop:language";
|
||||
pub(super) const PROP_ORDER: &str = "prop:order";
|
||||
|
||||
pub(super) const PROP_ELEMENTS: &str = "prop:elements";
|
||||
pub(super) const PROP_BACKGROUND: &str = "prop:background";
|
||||
pub(super) const PROP_XYWH: &str = "prop:xywh";
|
||||
pub(super) const PROP_INDEX: &str = "prop:index";
|
||||
pub(super) const PROP_HIDDEN: &str = "prop:hidden";
|
||||
pub(super) const PROP_DISPLAY_MODE: &str = "prop:displayMode";
|
||||
|
||||
pub(super) const PROP_SOURCE_ID: &str = "prop:sourceId";
|
||||
pub(super) const PROP_CAPTION: &str = "prop:caption";
|
||||
pub(super) const PROP_WIDTH: &str = "prop:width";
|
||||
pub(super) const PROP_HEIGHT: &str = "prop:height";
|
||||
pub(super) const PROP_URL: &str = "prop:url";
|
||||
pub(super) const PROP_VIDEO_ID: &str = "prop:videoId";
|
||||
|
||||
pub(super) const PROP_ROWS_PREFIX: &str = "prop:rows.";
|
||||
pub(super) const PROP_COLUMNS_PREFIX: &str = "prop:columns.";
|
||||
pub(super) const PROP_CELLS_PREFIX: &str = "prop:cells.";
|
||||
pub(super) const PROP_ROW_ID_SUFFIX: &str = ".rowId";
|
||||
pub(super) const PROP_COLUMN_ID_SUFFIX: &str = ".columnId";
|
||||
pub(super) const PROP_ORDER_SUFFIX: &str = ".order";
|
||||
pub(super) const PROP_TEXT_SUFFIX: &str = ".text";
|
||||
|
||||
pub(super) fn table_row_id_key(row_id: &str) -> String {
|
||||
format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ROW_ID_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_row_order_key(row_id: &str) -> String {
|
||||
format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ORDER_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_column_id_key(column_id: &str) -> String {
|
||||
format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_COLUMN_ID_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_column_order_key(column_id: &str) -> String {
|
||||
format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_ORDER_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_cell_text_key(row_id: &str, column_id: &str) -> String {
|
||||
format!("{PROP_CELLS_PREFIX}{row_id}:{column_id}{PROP_TEXT_SUFFIX}")
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct MarkdownTableOptions {
|
||||
pub(super) escape_pipes: bool,
|
||||
pub(super) newline_replacement: &'static str,
|
||||
pub(super) trim: bool,
|
||||
}
|
||||
|
||||
impl MarkdownTableOptions {
|
||||
pub(super) const fn new(escape_pipes: bool, newline_replacement: &'static str, trim: bool) -> Self {
|
||||
Self {
|
||||
escape_pipes,
|
||||
newline_replacement,
|
||||
trim,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn render_markdown_table(rows: &[Vec<String>], options: MarkdownTableOptions) -> Option<String> {
|
||||
let (header, body) = rows.split_first()?;
|
||||
let header_line = format_table_row(header, options);
|
||||
let separator_line = format_table_row(&vec!["---".to_string(); header.len()], options);
|
||||
let mut lines = vec![header_line, separator_line];
|
||||
for row in body {
|
||||
lines.push(format_table_row(row, options));
|
||||
}
|
||||
Some(lines.join("\n"))
|
||||
}
|
||||
|
||||
fn format_table_row(row: &[String], options: MarkdownTableOptions) -> String {
|
||||
let cells = row
|
||||
.iter()
|
||||
.map(|cell| format_table_cell(cell, options))
|
||||
.collect::<Vec<_>>();
|
||||
format!("|{}|", cells.join("|"))
|
||||
}
|
||||
|
||||
fn format_table_cell(cell: &str, options: MarkdownTableOptions) -> String {
|
||||
let mut value = if options.trim {
|
||||
cell.trim().to_string()
|
||||
} else {
|
||||
cell.to_string()
|
||||
};
|
||||
|
||||
if options.escape_pipes {
|
||||
value = value.replace('|', "\\|");
|
||||
}
|
||||
if !options.newline_replacement.is_empty() {
|
||||
value = collapse_newlines(&value, options.newline_replacement);
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
fn collapse_newlines(value: &str, replacement: &str) -> String {
|
||||
if replacement.is_empty() {
|
||||
return value.to_string();
|
||||
}
|
||||
let mut out = String::with_capacity(value.len());
|
||||
let mut in_newline = false;
|
||||
for ch in value.chars() {
|
||||
if ch == '\n' {
|
||||
if !in_newline {
|
||||
out.push_str(replacement);
|
||||
in_newline = true;
|
||||
}
|
||||
} else {
|
||||
in_newline = false;
|
||||
out.push(ch);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -81,6 +81,16 @@ pub(super) fn value_to_any(value: &Value) -> Option<Any> {
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn value_to_f64(value: Value) -> Option<f64> {
|
||||
value.to_any().and_then(|any| match any {
|
||||
Any::Integer(v) => Some(v as f64),
|
||||
Any::BigInt64(v) => Some(v as f64),
|
||||
Any::Float32(v) => Some(v.0 as f64),
|
||||
Any::Float64(v) => Some(v.0),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn any_to_string(any: &Any) -> Option<String> {
|
||||
match any {
|
||||
Any::String(value) => Some(value.to_string()),
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
use y_octo::{TextDeltaOp, TextInsert};
|
||||
|
||||
use super::{
|
||||
super::schema::{
|
||||
PROP_CAPTION, PROP_CELLS_PREFIX, PROP_CHECKED, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER,
|
||||
PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL, PROP_VIDEO_ID, PROP_WIDTH, SYS_CHILDREN,
|
||||
SYS_FLAVOUR, SYS_ID, SYS_VERSION, table_cell_text_key, table_column_id_key, table_column_order_key,
|
||||
table_row_id_key, table_row_order_key,
|
||||
},
|
||||
*,
|
||||
};
|
||||
|
||||
pub(super) const BOXED_NATIVE_TYPE: &str = "$blocksuite:internal:native$";
|
||||
pub(super) const NOTE_BG_LIGHT: &str = "#ffffff";
|
||||
pub(super) const NOTE_BG_DARK: &str = "#252525";
|
||||
const TABLE_ORDER_WIDTH: usize = 6;
|
||||
|
||||
pub(super) fn block_version(flavour: &str) -> i32 {
|
||||
match flavour {
|
||||
"affine:page" => 2,
|
||||
"affine:surface" => 5,
|
||||
"affine:note" => 1,
|
||||
"affine:paragraph" => 1,
|
||||
"affine:list" => 1,
|
||||
"affine:code" => 1,
|
||||
"affine:divider" => 1,
|
||||
"affine:image" => 1,
|
||||
"affine:table" => 1,
|
||||
"affine:bookmark" => 1,
|
||||
"affine:embed-youtube" => 1,
|
||||
"affine:embed-iframe" => 1,
|
||||
"affine:callout" => 1,
|
||||
_ => 1,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct TextBlockProps<'a> {
|
||||
pub block_type: Option<&'a str>,
|
||||
pub checked: Option<bool>,
|
||||
pub language: Option<&'a str>,
|
||||
pub order: Option<i64>,
|
||||
pub text: &'a [TextDeltaOp],
|
||||
}
|
||||
|
||||
pub(super) struct ImageBlockProps<'a> {
|
||||
pub source_id: &'a str,
|
||||
pub caption: Option<&'a str>,
|
||||
pub width: Option<f64>,
|
||||
pub height: Option<f64>,
|
||||
}
|
||||
|
||||
pub(super) struct BookmarkBlockProps<'a> {
|
||||
pub url: &'a str,
|
||||
pub caption: Option<&'a str>,
|
||||
}
|
||||
|
||||
pub(super) struct EmbedYoutubeBlockProps<'a> {
|
||||
pub video_id: &'a str,
|
||||
}
|
||||
|
||||
pub(super) struct EmbedIframeBlockProps<'a> {
|
||||
pub url: &'a str,
|
||||
}
|
||||
|
||||
pub(super) fn insert_text(doc: &Doc, block: &mut Map, key: &str, ops: &[TextDeltaOp]) -> Result<(), ParseError> {
|
||||
let mut text = doc.create_text()?;
|
||||
// Attach first so updates encode parent types before their contents.
|
||||
block.insert(key.to_string(), Value::Text(text.clone()))?;
|
||||
if !ops.is_empty() {
|
||||
text.apply_delta(ops)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn text_ops_from_plain(text: &str) -> Vec<TextDeltaOp> {
|
||||
if text.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text.to_string()),
|
||||
format: None,
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn insert_children(doc: &Doc, block: &mut Map, children: &[String]) -> Result<(), ParseError> {
|
||||
let mut array = doc.create_array()?;
|
||||
// Attach first so updates encode parent types before their contents.
|
||||
block.insert(SYS_CHILDREN.to_string(), Value::Array(array.clone()))?;
|
||||
for child_id in children {
|
||||
array.push(child_id.to_string())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn insert_block_map(doc: &Doc, blocks_map: &mut Map, block_id: &str) -> Result<Map, ParseError> {
|
||||
let empty_map = doc.create_map()?;
|
||||
blocks_map.insert(block_id.to_string(), Value::Map(empty_map))?;
|
||||
|
||||
blocks_map
|
||||
.get(block_id)
|
||||
.and_then(|value| value.to_map())
|
||||
.ok_or_else(|| ParseError::ParserError("Failed to retrieve inserted block map".into()))
|
||||
}
|
||||
|
||||
pub(super) fn insert_sys_fields(block: &mut Map, block_id: &str, flavour: &str) -> Result<(), ParseError> {
|
||||
block.insert(SYS_ID.to_string(), Any::String(block_id.to_string()))?;
|
||||
block.insert(SYS_FLAVOUR.to_string(), Any::String(flavour.to_string()))?;
|
||||
block.insert(SYS_VERSION.to_string(), Any::Integer(block_version(flavour)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_text_block_props(
|
||||
doc: &Doc,
|
||||
block: &mut Map,
|
||||
props: &TextBlockProps<'_>,
|
||||
preserve_text: bool,
|
||||
clear_missing: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
match props.block_type {
|
||||
Some(block_type) => {
|
||||
block.insert(PROP_TYPE.to_string(), Any::String(block_type.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_TYPE).is_some() {
|
||||
block.remove(PROP_TYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !preserve_text && !props.text.is_empty() {
|
||||
insert_text(doc, block, PROP_TEXT, props.text)?;
|
||||
} else if !preserve_text && clear_missing && block.get(PROP_TEXT).is_some() {
|
||||
block.remove(PROP_TEXT);
|
||||
}
|
||||
|
||||
match props.checked {
|
||||
Some(checked) => {
|
||||
block.insert(PROP_CHECKED.to_string(), if checked { Any::True } else { Any::False })?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_CHECKED).is_some() {
|
||||
block.remove(PROP_CHECKED);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.language {
|
||||
Some(language) => {
|
||||
block.insert(PROP_LANGUAGE.to_string(), Any::String(language.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_LANGUAGE).is_some() {
|
||||
block.remove(PROP_LANGUAGE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.order {
|
||||
Some(order) => {
|
||||
block.insert(PROP_ORDER.to_string(), Any::Float64((order as f64).into()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_ORDER).is_some() {
|
||||
block.remove(PROP_ORDER);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_image_block_props(
|
||||
block: &mut Map,
|
||||
props: &ImageBlockProps<'_>,
|
||||
clear_missing: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_SOURCE_ID.to_string(), Any::String(props.source_id.to_string()))?;
|
||||
|
||||
match props.caption {
|
||||
Some(caption) => {
|
||||
block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_CAPTION).is_some() {
|
||||
block.remove(PROP_CAPTION);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.width {
|
||||
Some(width) => {
|
||||
block.insert(PROP_WIDTH.to_string(), Any::Float64(width.into()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_WIDTH).is_some() {
|
||||
block.remove(PROP_WIDTH);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.height {
|
||||
Some(height) => {
|
||||
block.insert(PROP_HEIGHT.to_string(), Any::Float64(height.into()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_HEIGHT).is_some() {
|
||||
block.remove(PROP_HEIGHT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_bookmark_block_props(
|
||||
block: &mut Map,
|
||||
props: &BookmarkBlockProps<'_>,
|
||||
clear_missing: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?;
|
||||
|
||||
match props.caption {
|
||||
Some(caption) => {
|
||||
block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_CAPTION).is_some() {
|
||||
block.remove(PROP_CAPTION);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_embed_youtube_block_props(
|
||||
block: &mut Map,
|
||||
props: &EmbedYoutubeBlockProps<'_>,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_VIDEO_ID.to_string(), Any::String(props.video_id.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_embed_iframe_block_props(
|
||||
block: &mut Map,
|
||||
props: &EmbedIframeBlockProps<'_>,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_table_block_props(block: &mut Map, rows: &[Vec<String>]) -> Result<(), ParseError> {
|
||||
clear_table_props(block);
|
||||
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let column_count = rows.iter().map(|row| row.len()).max().unwrap_or(0);
|
||||
let column_ids: Vec<String> = (0..column_count).map(|_| nanoid::nanoid!()).collect();
|
||||
|
||||
for (col_idx, column_id) in column_ids.iter().enumerate() {
|
||||
let order = format_table_order(col_idx);
|
||||
block.insert(table_column_id_key(column_id), Any::String(column_id.to_string()))?;
|
||||
block.insert(table_column_order_key(column_id), Any::String(order))?;
|
||||
}
|
||||
|
||||
for (row_idx, row) in rows.iter().enumerate() {
|
||||
let row_id = nanoid::nanoid!();
|
||||
let order = format_table_order(row_idx);
|
||||
block.insert(table_row_id_key(&row_id), Any::String(row_id.to_string()))?;
|
||||
block.insert(table_row_order_key(&row_id), Any::String(order))?;
|
||||
|
||||
for (col_idx, column_id) in column_ids.iter().enumerate() {
|
||||
let cell_text = row.get(col_idx).cloned().unwrap_or_default();
|
||||
block.insert(table_cell_text_key(&row_id, column_id), Any::String(cell_text))?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) struct ApplyBlockOptions {
|
||||
pub preserve_text: bool,
|
||||
pub clear_missing: bool,
|
||||
}
|
||||
|
||||
pub(super) fn apply_block_spec(
|
||||
doc: &Doc,
|
||||
block: &mut Map,
|
||||
spec: &BlockSpec,
|
||||
options: ApplyBlockOptions,
|
||||
) -> Result<(), ParseError> {
|
||||
match spec.flavour {
|
||||
BlockFlavour::Image => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let image = spec
|
||||
.image
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("image spec missing".into()))?;
|
||||
let props = ImageBlockProps {
|
||||
source_id: &image.source_id,
|
||||
caption: image.caption.as_deref(),
|
||||
width: image.width,
|
||||
height: image.height,
|
||||
};
|
||||
apply_image_block_props(block, &props, options.clear_missing)?;
|
||||
}
|
||||
BlockFlavour::Bookmark => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let bookmark = spec
|
||||
.bookmark
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("bookmark spec missing".into()))?;
|
||||
let props = BookmarkBlockProps {
|
||||
url: &bookmark.url,
|
||||
caption: bookmark.caption.as_deref(),
|
||||
};
|
||||
apply_bookmark_block_props(block, &props, options.clear_missing)?;
|
||||
}
|
||||
BlockFlavour::EmbedYoutube => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let embed = spec
|
||||
.embed_youtube
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?;
|
||||
let props = EmbedYoutubeBlockProps {
|
||||
video_id: &embed.video_id,
|
||||
};
|
||||
apply_embed_youtube_block_props(block, &props)?;
|
||||
}
|
||||
BlockFlavour::EmbedIframe => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let embed = spec
|
||||
.embed_iframe
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?;
|
||||
let props = EmbedIframeBlockProps { url: &embed.url };
|
||||
apply_embed_iframe_block_props(block, &props)?;
|
||||
}
|
||||
BlockFlavour::Callout => {
|
||||
return Ok(());
|
||||
}
|
||||
BlockFlavour::Table => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let table = spec
|
||||
.table
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("table spec missing".into()))?;
|
||||
apply_table_block_props(block, &table.rows)?;
|
||||
}
|
||||
_ => {
|
||||
let props = TextBlockProps {
|
||||
block_type: spec.block_type_str(),
|
||||
checked: spec.checked,
|
||||
language: spec.language.as_deref(),
|
||||
order: spec.order,
|
||||
text: &spec.text,
|
||||
};
|
||||
apply_text_block_props(doc, block, &props, options.preserve_text, options.clear_missing)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn insert_block_tree(doc: &Doc, blocks_map: &mut Map, node: &BlockNode) -> Result<String, ParseError> {
|
||||
let block_id = nanoid::nanoid!();
|
||||
let mut block_map = insert_block_map(doc, blocks_map, &block_id)?;
|
||||
|
||||
insert_sys_fields(&mut block_map, &block_id, node.spec.flavour.as_str())?;
|
||||
apply_block_spec(
|
||||
doc,
|
||||
&mut block_map,
|
||||
&node.spec,
|
||||
ApplyBlockOptions {
|
||||
preserve_text: false,
|
||||
clear_missing: false,
|
||||
},
|
||||
)?;
|
||||
|
||||
let child_ids = node
|
||||
.children
|
||||
.iter()
|
||||
.map(|child| insert_block_tree(doc, blocks_map, child))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
insert_children(doc, &mut block_map, &child_ids)?;
|
||||
|
||||
Ok(block_id)
|
||||
}
|
||||
|
||||
fn clear_table_props(block: &mut Map) {
|
||||
let keys = block
|
||||
.keys()
|
||||
.filter(|key| {
|
||||
key.starts_with(PROP_ROWS_PREFIX) || key.starts_with(PROP_COLUMNS_PREFIX) || key.starts_with(PROP_CELLS_PREFIX)
|
||||
})
|
||||
.map(|s| s.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
for key in keys {
|
||||
block.remove(&key);
|
||||
}
|
||||
}
|
||||
|
||||
fn format_table_order(index: usize) -> String {
|
||||
format!("{index:0width$}", width = TABLE_ORDER_WIDTH)
|
||||
}
|
||||
|
||||
pub(super) fn boxed_empty_map(doc: &Doc) -> Result<Map, ParseError> {
|
||||
doc.create_map().map_err(ParseError::from)
|
||||
}
|
||||
|
||||
pub(super) fn note_background_map(doc: &Doc) -> Result<Map, ParseError> {
|
||||
doc.create_map().map_err(ParseError::from)
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
//! Markdown to YDoc conversion module
|
||||
//!
|
||||
//! Converts markdown content into AFFiNE-compatible y-octo document binary
|
||||
//! format.
|
||||
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::{
|
||||
super::{
|
||||
markdown::parse_markdown_blocks,
|
||||
schema::{PROP_BACKGROUND, PROP_DISPLAY_MODE, PROP_ELEMENTS, PROP_HIDDEN, PROP_INDEX, PROP_XYWH, SURFACE_FLAVOUR},
|
||||
},
|
||||
builder::{
|
||||
BOXED_NATIVE_TYPE, NOTE_BG_DARK, NOTE_BG_LIGHT, boxed_empty_map, insert_block_map, insert_block_tree,
|
||||
insert_children, insert_sys_fields, insert_text, note_background_map, text_ops_from_plain,
|
||||
},
|
||||
*,
|
||||
};
|
||||
|
||||
/// Converts markdown into an AFFiNE-compatible y-octo document binary.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `title` - The document title
|
||||
/// * `markdown` - The markdown content to convert
|
||||
/// * `doc_id` - The document ID to use
|
||||
///
|
||||
/// # Returns
|
||||
/// A binary vector containing the y-octo encoded document update
|
||||
pub fn build_full_doc(title: &str, markdown: &str, doc_id: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let nodes = parse_markdown_blocks(markdown)?;
|
||||
build_doc_update(doc_id, title, &nodes)
|
||||
}
|
||||
|
||||
fn build_doc_update(doc_id: &str, title: &str, blocks: &[BlockNode]) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
let mut blocks_map = doc.get_or_create_map("blocks")?;
|
||||
|
||||
let page_id = nanoid::nanoid!();
|
||||
let surface_id = nanoid::nanoid!();
|
||||
let note_id = nanoid::nanoid!();
|
||||
|
||||
// Insert root blocks first to establish stable IDs.
|
||||
let mut page_map = insert_block_map(&doc, &mut blocks_map, &page_id)?;
|
||||
let mut surface_map = insert_block_map(&doc, &mut blocks_map, &surface_id)?;
|
||||
let mut note_map = insert_block_map(&doc, &mut blocks_map, ¬e_id)?;
|
||||
|
||||
// Create content blocks under note.
|
||||
let content_ids = insert_block_trees(&doc, &mut blocks_map, blocks)?;
|
||||
|
||||
// Page block
|
||||
insert_sys_fields(&mut page_map, &page_id, PAGE_FLAVOUR)?;
|
||||
insert_children(&doc, &mut page_map, &[surface_id.clone(), note_id.clone()])?;
|
||||
insert_text(&doc, &mut page_map, PROP_TITLE, &text_ops_from_plain(title))?;
|
||||
|
||||
// Surface block
|
||||
insert_sys_fields(&mut surface_map, &surface_id, SURFACE_FLAVOUR)?;
|
||||
insert_children(&doc, &mut surface_map, &[])?;
|
||||
let mut boxed = boxed_empty_map(&doc)?;
|
||||
surface_map.insert(PROP_ELEMENTS.to_string(), Value::Map(boxed.clone()))?;
|
||||
boxed.insert("type".to_string(), Any::String(BOXED_NATIVE_TYPE.to_string()))?;
|
||||
let value = doc.create_map()?;
|
||||
boxed.insert("value".to_string(), Value::Map(value))?;
|
||||
|
||||
// Note block
|
||||
insert_sys_fields(&mut note_map, ¬e_id, NOTE_FLAVOUR)?;
|
||||
insert_children(&doc, &mut note_map, &content_ids)?;
|
||||
let mut background = note_background_map(&doc)?;
|
||||
note_map.insert(PROP_BACKGROUND.to_string(), Value::Map(background.clone()))?;
|
||||
background.insert("light".to_string(), Any::String(NOTE_BG_LIGHT.to_string()))?;
|
||||
background.insert("dark".to_string(), Any::String(NOTE_BG_DARK.to_string()))?;
|
||||
note_map.insert(PROP_XYWH.to_string(), Any::String("[0,0,800,95]".to_string()))?;
|
||||
note_map.insert(PROP_INDEX.to_string(), Any::String("a0".to_string()))?;
|
||||
note_map.insert(PROP_HIDDEN.to_string(), Any::False)?;
|
||||
note_map.insert(PROP_DISPLAY_MODE.to_string(), Any::String("both".to_string()))?;
|
||||
|
||||
Ok(doc.encode_update_v1()?)
|
||||
}
|
||||
|
||||
fn insert_block_trees(doc: &Doc, blocks_map: &mut Map, blocks: &[BlockNode]) -> Result<Vec<String>, ParseError> {
|
||||
let mut ids = Vec::with_capacity(blocks.len());
|
||||
for block in blocks {
|
||||
let id = insert_block_tree(doc, blocks_map, block)?;
|
||||
ids.push(id);
|
||||
}
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::{Any, DocOptions};
|
||||
|
||||
use super::{
|
||||
super::super::{
|
||||
blocksuite::get_string,
|
||||
markdown::{MAX_BLOCKS, MAX_MARKDOWN_CHARS},
|
||||
schema::PAGE_FLAVOUR,
|
||||
},
|
||||
*,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn test_simple_markdown() {
|
||||
let markdown = "# Hello World\n\nThis is a test paragraph.";
|
||||
let result = build_full_doc("Hello World", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_title_from_param() {
|
||||
let markdown = "# Markdown Title\n\nContent.";
|
||||
let doc_id = "title-param-test";
|
||||
let bin = build_full_doc("External Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut title = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
|
||||
{
|
||||
title = get_string(&block_map, "prop:title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(title.as_deref(), Some("External Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_list() {
|
||||
let markdown = "# Test List\n\n- Item 1\n- Item 2\n- Item 3";
|
||||
let result = build_full_doc("Test List", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_code() {
|
||||
let markdown = "# Code Example\n\n```rust\nfn main() {\n println!(\"Hello\");\n}\n```";
|
||||
let result = build_full_doc("Code Example", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_headings() {
|
||||
let markdown = "# H1\n\n## H2\n\n### H3\n\nParagraph text.";
|
||||
let result = build_full_doc("H1", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_markdown() {
|
||||
let result = build_full_doc("Untitled", "", "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_whitespace_only_markdown() {
|
||||
let result = build_full_doc("Untitled", " \n\n\t\n ", "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_without_h1() {
|
||||
let markdown = "## Secondary Heading\n\nSome content without H1.";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nested_lists() {
|
||||
let markdown = "# Nested Lists\n\n- Item 1\n - Nested 1.1\n - Nested 1.2\n- Item 2\n - Nested 2.1";
|
||||
let result = build_full_doc("Nested Lists", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blockquote() {
|
||||
let markdown = "# Title\n\n> A blockquote";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_divider() {
|
||||
let markdown = "# Title\n\nBefore divider\n\n---\n\nAfter divider";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_numbered_list() {
|
||||
let markdown = "# Title\n\n1. First item\n2. Second item";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_too_large() {
|
||||
let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1);
|
||||
let result = build_full_doc("Title", &markdown, "test-doc-id");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_block_limit() {
|
||||
let mut markdown = String::from("# Title\n\n");
|
||||
for i in 0..=MAX_BLOCKS {
|
||||
markdown.push_str(&format!("Paragraph {i}\n\n"));
|
||||
}
|
||||
let result = build_full_doc("Title", &markdown, "test-doc-id");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_image() {
|
||||
let markdown = "";
|
||||
let doc_id = "image-doc";
|
||||
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut found = false;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image")
|
||||
{
|
||||
let source_id = get_string(&block_map, "prop:sourceId");
|
||||
assert_eq!(source_id.as_deref(), Some("image-id"));
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_table() {
|
||||
let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let doc_id = "table-doc";
|
||||
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut found_cell = false;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table")
|
||||
{
|
||||
for key in block_map.keys() {
|
||||
if key.starts_with("prop:cells.") && key.ends_with(".text") {
|
||||
let value = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::String(value) => Some(value),
|
||||
_ => None,
|
||||
});
|
||||
if let Some(value) = value
|
||||
&& (value == "A" || value == "1")
|
||||
{
|
||||
found_cell = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found_cell);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
use super::{
|
||||
builder::{insert_text, text_ops_from_plain},
|
||||
root_doc::ensure_pages_array,
|
||||
*,
|
||||
};
|
||||
|
||||
pub fn update_doc_title(existing_binary: &[u8], doc_id: &str, title: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = load_doc(existing_binary, Some(doc_id))?;
|
||||
|
||||
let state_before = doc.get_state_vector();
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Err(ParseError::ParserError("blocks map is empty".into()));
|
||||
}
|
||||
|
||||
let mut page_block = find_page_block(&blocks_map)?;
|
||||
let current = get_string(&page_block, PROP_TITLE).unwrap_or_default();
|
||||
if current != title {
|
||||
insert_text(&doc, &mut page_block, PROP_TITLE, &text_ops_from_plain(title))?;
|
||||
}
|
||||
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
pub fn update_root_doc_meta_title(root_doc_bin: &[u8], doc_id: &str, title: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = load_doc_or_new(root_doc_bin)?;
|
||||
|
||||
let state_before = doc.get_state_vector();
|
||||
let mut meta = doc.get_or_create_map("meta")?;
|
||||
let mut pages = ensure_pages_array(&doc, &mut meta)?;
|
||||
|
||||
let mut found = false;
|
||||
for idx in 0..pages.len() {
|
||||
let Some(mut page) = pages.get(idx).and_then(|v| v.to_map()) else {
|
||||
continue;
|
||||
};
|
||||
if get_string(&page, "id").as_deref() == Some(doc_id) {
|
||||
page.insert("title".to_string(), Any::String(title.to_string()))?;
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
let page_map = doc.create_map()?;
|
||||
|
||||
let idx = pages.len();
|
||||
pages.insert(idx, Value::Map(page_map))?;
|
||||
|
||||
if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) {
|
||||
inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?;
|
||||
inserted_page.insert("title".to_string(), Any::String(title.to_string()))?;
|
||||
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as i64)
|
||||
.unwrap_or(0);
|
||||
inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?;
|
||||
|
||||
let tags = doc.create_array()?;
|
||||
inserted_page.insert("tags".to_string(), Value::Array(tags))?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
fn find_page_block(blocks_map: &Map) -> Result<Map, ParseError> {
|
||||
let index = build_block_index(blocks_map);
|
||||
let page_id = find_block_id_by_flavour(&index.block_pool, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
|
||||
blocks_map
|
||||
.get(&page_id)
|
||||
.and_then(|value| value.to_map())
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::*;
|
||||
use crate::doc_parser::{add_doc_to_root_doc, build_full_doc};
|
||||
|
||||
#[test]
|
||||
fn test_update_doc_title() {
|
||||
let doc_id = "doc-meta-title-test";
|
||||
let initial = build_full_doc("Old Title", "Content.", doc_id).expect("create doc");
|
||||
let delta = update_doc_title(&initial, doc_id, "New Title").expect("update title");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&initial).expect("apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut title = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
|
||||
{
|
||||
title = get_string(&block_map, "prop:title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(title.as_deref(), Some("New Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_root_doc_meta_title() {
|
||||
let doc_id = "root-meta-title-test";
|
||||
let root_bin = add_doc_to_root_doc(Vec::new(), doc_id, Some("Old Title")).expect("create root meta");
|
||||
let delta = update_root_doc_meta_title(&root_bin, doc_id, "New Title").expect("update meta");
|
||||
|
||||
let mut doc = DocOptions::new().build();
|
||||
doc.apply_update_from_binary_v1(&root_bin).expect("apply root");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let meta = doc.get_map("meta").expect("meta map");
|
||||
let pages = meta.get("pages").and_then(|v| v.to_array()).expect("pages array");
|
||||
let mut title = None;
|
||||
for page in pages.iter() {
|
||||
if let Some(page_map) = page.to_map()
|
||||
&& get_string(&page_map, "id").as_deref() == Some(doc_id)
|
||||
{
|
||||
title = get_string(&page_map, "title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(title.as_deref(), Some("New Title"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
use y_octo::{Any, DocOptions, Map};
|
||||
|
||||
use super::{
|
||||
super::{doc_loader::is_empty_doc, value::value_to_string},
|
||||
ParseError,
|
||||
};
|
||||
|
||||
pub fn update_doc_properties(
|
||||
existing_binary: &[u8],
|
||||
properties_doc_id: &str,
|
||||
target_doc_id: &str,
|
||||
created_by: Option<&str>,
|
||||
updated_by: Option<&str>,
|
||||
) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = if is_empty_doc(existing_binary) {
|
||||
DocOptions::new().with_guid(properties_doc_id.to_string()).build()
|
||||
} else {
|
||||
super::load_doc(existing_binary, Some(properties_doc_id))?
|
||||
};
|
||||
|
||||
let state_before = doc.get_state_vector();
|
||||
let mut record = doc.get_or_create_map(target_doc_id)?;
|
||||
let mut changed = false;
|
||||
|
||||
if record.get("id").is_none() {
|
||||
record.insert("id".to_string(), Any::String(target_doc_id.to_string()))?;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if let Some(created_by) = created_by
|
||||
&& get_record_string(&record, "createdBy").as_deref() != Some(created_by)
|
||||
{
|
||||
record.insert("createdBy".to_string(), Any::String(created_by.to_string()))?;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if let Some(updated_by) = updated_by
|
||||
&& get_record_string(&record, "updatedBy").as_deref() != Some(updated_by)
|
||||
{
|
||||
record.insert("updatedBy".to_string(), Any::String(updated_by.to_string()))?;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if !changed {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
fn get_record_string(record: &Map, key: &str) -> Option<String> {
|
||||
record.get(key).and_then(|value| value_to_string(&value))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_update_doc_properties_creates_record() {
|
||||
let properties_doc_id = "doc-properties";
|
||||
let target_doc_id = "doc-1";
|
||||
let update = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-1"))
|
||||
.expect("update properties");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(properties_doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&update).expect("apply");
|
||||
|
||||
let record = doc.get_map(target_doc_id).expect("record map");
|
||||
assert_eq!(get_record_string(&record, "id").as_deref(), Some(target_doc_id));
|
||||
assert_eq!(get_record_string(&record, "createdBy").as_deref(), Some("user-1"));
|
||||
assert_eq!(get_record_string(&record, "updatedBy").as_deref(), Some("user-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_doc_properties_no_change() {
|
||||
let properties_doc_id = "doc-properties-no-change";
|
||||
let target_doc_id = "doc-2";
|
||||
let initial = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-2"))
|
||||
.expect("initial update");
|
||||
|
||||
let delta = update_doc_properties(
|
||||
&initial,
|
||||
properties_doc_id,
|
||||
target_doc_id,
|
||||
Some("user-1"),
|
||||
Some("user-2"),
|
||||
)
|
||||
.expect("no change update");
|
||||
|
||||
assert!(delta.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
pub mod builder;
|
||||
mod create;
|
||||
mod doc_meta;
|
||||
mod doc_properties;
|
||||
mod root_doc;
|
||||
mod update;
|
||||
|
||||
pub use create::build_full_doc;
|
||||
pub use doc_meta::{update_doc_title, update_root_doc_meta_title};
|
||||
pub use doc_properties::update_doc_properties;
|
||||
pub use root_doc::add_doc_to_root_doc;
|
||||
pub use update::update_doc;
|
||||
use y_octo::{Any, Doc, Map, Value};
|
||||
|
||||
use super::{
|
||||
ParseError,
|
||||
block_spec::{BlockFlavour, BlockNode, BlockSpec},
|
||||
blocksuite::{build_block_index, find_block_id_by_flavour, get_string},
|
||||
doc_loader::{load_doc, load_doc_or_new},
|
||||
schema::{NOTE_FLAVOUR, PAGE_FLAVOUR, PROP_TITLE},
|
||||
};
|
||||
@@ -0,0 +1,106 @@
|
||||
use y_octo::Array;
|
||||
|
||||
use super::*;
|
||||
|
||||
const DEFAULT_DOC_TITLE: &str = "Untitled";
|
||||
|
||||
fn any_to_value(doc: &Doc, any: Any) -> Result<Value, ParseError> {
|
||||
match any {
|
||||
Any::Array(values) => {
|
||||
let mut array = doc.create_array()?;
|
||||
for value in values {
|
||||
let item = any_to_value(doc, value)?;
|
||||
array.push(item)?;
|
||||
}
|
||||
Ok(Value::Array(array))
|
||||
}
|
||||
Any::Object(values) => {
|
||||
let mut map = doc.create_map()?;
|
||||
for (key, value) in values {
|
||||
let item = any_to_value(doc, value)?;
|
||||
map.insert(key, item)?;
|
||||
}
|
||||
Ok(Value::Map(map))
|
||||
}
|
||||
_ => Ok(Value::Any(any)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn ensure_pages_array(doc: &Doc, meta: &mut Map) -> Result<Array, ParseError> {
|
||||
let pages_value = meta.get("pages");
|
||||
if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) {
|
||||
return Ok(pages);
|
||||
}
|
||||
|
||||
if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) {
|
||||
let mut pages = doc.create_array()?;
|
||||
for entry in entries {
|
||||
let value = any_to_value(doc, entry)?;
|
||||
pages.push(value)?;
|
||||
}
|
||||
meta.insert("pages".to_string(), Value::Array(pages.clone()))?;
|
||||
return Ok(pages);
|
||||
}
|
||||
|
||||
let pages = doc.create_array()?;
|
||||
meta.insert("pages".to_string(), Value::Array(pages.clone()))?;
|
||||
Ok(pages)
|
||||
}
|
||||
|
||||
/// Adds a document ID to the root doc's meta.pages array.
|
||||
/// Returns a binary update that can be applied to the root doc.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `root_doc_bin` - The current root doc binary
|
||||
/// * `doc_id` - The document ID to add
|
||||
/// * `title` - Optional title for the document
|
||||
///
|
||||
/// # Returns
|
||||
/// A Vec<u8> containing the y-octo update binary to add the doc
|
||||
pub fn add_doc_to_root_doc(root_doc_bin: Vec<u8>, doc_id: &str, title: Option<&str>) -> Result<Vec<u8>, ParseError> {
|
||||
// Handle empty or minimal root doc - create a new one
|
||||
let doc = load_doc_or_new(&root_doc_bin)?;
|
||||
|
||||
// Capture state before modifications to encode only the delta
|
||||
let state_before = doc.get_state_vector();
|
||||
|
||||
// Get or create the meta map
|
||||
let mut meta = doc.get_or_create_map("meta")?;
|
||||
|
||||
let mut pages = ensure_pages_array(&doc, &mut meta)?;
|
||||
|
||||
// Check if doc already exists
|
||||
let doc_exists = pages.iter().any(|page_val| {
|
||||
page_val
|
||||
.to_map()
|
||||
.and_then(|page| get_string(&page, "id"))
|
||||
.map(|id| id == doc_id)
|
||||
.unwrap_or(false)
|
||||
});
|
||||
|
||||
if !doc_exists {
|
||||
let page_map = doc.create_map()?;
|
||||
|
||||
let idx = pages.len();
|
||||
pages.insert(idx, Value::Map(page_map))?;
|
||||
|
||||
if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) {
|
||||
inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?;
|
||||
|
||||
let page_title = title.unwrap_or(DEFAULT_DOC_TITLE);
|
||||
inserted_page.insert("title".to_string(), Any::String(page_title.to_string()))?;
|
||||
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as i64)
|
||||
.unwrap_or(0);
|
||||
inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?;
|
||||
|
||||
let tags = doc.create_array()?;
|
||||
inserted_page.insert("tags".to_string(), Value::Array(tags))?;
|
||||
}
|
||||
}
|
||||
|
||||
// Encode only the changes (delta) since state_before
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
@@ -0,0 +1,671 @@
|
||||
//! Update YDoc module
|
||||
//!
|
||||
//! Provides functionality to update existing AFFiNE documents by applying
|
||||
//! surgical y-octo operations based on content differences.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use super::{
|
||||
super::{
|
||||
block_spec::{TreeNode, count_tree_nodes, text_delta_eq},
|
||||
blocksuite::{collect_child_ids, find_child_id_by_flavour},
|
||||
markdown::{MAX_BLOCKS, parse_markdown_blocks},
|
||||
},
|
||||
builder::{ApplyBlockOptions, apply_block_spec, insert_block_tree, insert_children},
|
||||
*,
|
||||
};
|
||||
|
||||
const MAX_LCS_CELLS: usize = 2_000_000;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct StoredNode {
|
||||
id: String,
|
||||
spec: BlockSpec,
|
||||
children: Vec<StoredNode>,
|
||||
}
|
||||
|
||||
impl TreeNode for StoredNode {
|
||||
fn children(&self) -> &[StoredNode] {
|
||||
&self.children
|
||||
}
|
||||
}
|
||||
|
||||
struct DocState {
|
||||
doc: Doc,
|
||||
note_id: String,
|
||||
blocks: Vec<StoredNode>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum PatchOp {
|
||||
Keep(usize, usize),
|
||||
Delete(usize),
|
||||
Insert(usize),
|
||||
Update(usize, usize),
|
||||
}
|
||||
|
||||
/// Updates an existing document with new markdown content.
|
||||
///
|
||||
/// This function performs structural diffing between the existing document
|
||||
/// and the new markdown content, then applies block-level replacements
|
||||
/// for changed blocks. This enables proper CRDT merging with concurrent
|
||||
/// edits from other clients.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `existing_binary` - The current document binary
|
||||
/// * `new_markdown` - The new markdown content (document title is not updated)
|
||||
/// * `doc_id` - The document ID
|
||||
///
|
||||
/// # Returns
|
||||
/// A binary vector representing only the delta (changes) to apply
|
||||
pub fn update_doc(existing_binary: &[u8], new_markdown: &str, doc_id: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let mut new_nodes = parse_markdown_blocks(new_markdown)?;
|
||||
let state = load_doc_state(existing_binary, doc_id)?;
|
||||
|
||||
check_limits(&state.blocks, &new_nodes)?;
|
||||
|
||||
let state_before = state.doc.get_state_vector();
|
||||
|
||||
let mut blocks_map = state.doc.get_map("blocks")?;
|
||||
|
||||
let new_children = sync_nodes(&state.doc, &mut blocks_map, &state.blocks, &mut new_nodes)?;
|
||||
sync_children(&state.doc, &mut blocks_map, &state.note_id, &new_children)?;
|
||||
|
||||
Ok(state.doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
fn load_doc_state(binary: &[u8], doc_id: &str) -> Result<DocState, ParseError> {
|
||||
let doc = load_doc(binary, Some(doc_id))?;
|
||||
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Err(ParseError::ParserError("blocks map is empty".into()));
|
||||
}
|
||||
|
||||
let block_index = build_block_index(&blocks_map);
|
||||
let page_id = find_block_id_by_flavour(&block_index.block_pool, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
|
||||
let page_block = block_index
|
||||
.block_pool
|
||||
.get(&page_id)
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
|
||||
let note_id = find_child_id_by_flavour(page_block, &block_index.block_pool, NOTE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("note block not found".into()))?;
|
||||
let note_block = block_index
|
||||
.block_pool
|
||||
.get(¬e_id)
|
||||
.ok_or_else(|| ParseError::ParserError("note block not found".into()))?;
|
||||
let content_ids = collect_child_ids(note_block);
|
||||
|
||||
let mut blocks = Vec::new();
|
||||
for block_id in content_ids {
|
||||
let block = block_index
|
||||
.block_pool
|
||||
.get(&block_id)
|
||||
.ok_or_else(|| ParseError::ParserError("content block not found".into()))?;
|
||||
blocks.push(build_stored_tree(&block_id, block, &block_index.block_pool)?);
|
||||
}
|
||||
|
||||
Ok(DocState { doc, note_id, blocks })
|
||||
}
|
||||
|
||||
fn build_stored_tree(block_id: &str, block: &Map, pool: &HashMap<String, Map>) -> Result<StoredNode, ParseError> {
|
||||
let spec = BlockSpec::from_block_map(block)?;
|
||||
|
||||
let child_ids = collect_child_ids(block);
|
||||
if !child_ids.is_empty() && !matches!(spec.flavour, BlockFlavour::List | BlockFlavour::Callout) {
|
||||
return Err(ParseError::ParserError(format!(
|
||||
"unsupported children on block: {block_id}"
|
||||
)));
|
||||
}
|
||||
let mut children = Vec::new();
|
||||
for child_id in child_ids {
|
||||
let child_block = pool
|
||||
.get(&child_id)
|
||||
.ok_or_else(|| ParseError::ParserError("child block not found".into()))?;
|
||||
children.push(build_stored_tree(&child_id, child_block, pool)?);
|
||||
}
|
||||
|
||||
Ok(StoredNode {
|
||||
id: block_id.to_string(),
|
||||
spec,
|
||||
children,
|
||||
})
|
||||
}
|
||||
|
||||
fn sync_nodes(
|
||||
doc: &Doc,
|
||||
blocks_map: &mut Map,
|
||||
current: &[StoredNode],
|
||||
target: &mut [BlockNode],
|
||||
) -> Result<Vec<String>, ParseError> {
|
||||
let ops = diff_blocks(current, target);
|
||||
let mut new_children = Vec::new();
|
||||
let mut to_remove = Vec::new();
|
||||
|
||||
for op in ops {
|
||||
match op {
|
||||
PatchOp::Keep(old_idx, new_idx) => {
|
||||
let old_node = ¤t[old_idx];
|
||||
let new_node = &target[new_idx];
|
||||
update_block_props(doc, blocks_map, old_node, &new_node.spec, true)?;
|
||||
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
|
||||
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
|
||||
new_children.push(old_node.id.clone());
|
||||
}
|
||||
PatchOp::Update(old_idx, new_idx) => {
|
||||
let old_node = ¤t[old_idx];
|
||||
let new_node = &target[new_idx];
|
||||
update_block_props(doc, blocks_map, old_node, &new_node.spec, false)?;
|
||||
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
|
||||
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
|
||||
new_children.push(old_node.id.clone());
|
||||
}
|
||||
PatchOp::Insert(new_idx) => {
|
||||
let new_id = insert_block_tree(doc, blocks_map, &target[new_idx])?;
|
||||
new_children.push(new_id);
|
||||
}
|
||||
PatchOp::Delete(old_idx) => {
|
||||
let node = ¤t[old_idx];
|
||||
if node.spec.flavour == BlockFlavour::Callout {
|
||||
new_children.push(node.id.clone());
|
||||
} else {
|
||||
collect_tree_ids(node, &mut to_remove);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for id in to_remove {
|
||||
blocks_map.remove(&id);
|
||||
}
|
||||
|
||||
Ok(new_children)
|
||||
}
|
||||
|
||||
fn diff_blocks(current: &[StoredNode], target: &[BlockNode]) -> Vec<PatchOp> {
|
||||
let old_len = current.len();
|
||||
let new_len = target.len();
|
||||
|
||||
if old_len == 0 {
|
||||
return (0..new_len).map(PatchOp::Insert).collect();
|
||||
}
|
||||
if new_len == 0 {
|
||||
return (0..old_len).map(PatchOp::Delete).collect();
|
||||
}
|
||||
|
||||
let mut lcs = vec![vec![0usize; new_len + 1]; old_len + 1];
|
||||
|
||||
for i in 1..=old_len {
|
||||
for j in 1..=new_len {
|
||||
let old_spec = ¤t[i - 1].spec;
|
||||
let new_spec = &target[j - 1].spec;
|
||||
|
||||
if old_spec.is_exact(new_spec) {
|
||||
lcs[i][j] = lcs[i - 1][j - 1] + 1;
|
||||
} else {
|
||||
lcs[i][j] = std::cmp::max(lcs[i - 1][j], lcs[i][j - 1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut ops = Vec::new();
|
||||
let mut i = old_len;
|
||||
let mut j = new_len;
|
||||
|
||||
while i > 0 || j > 0 {
|
||||
if i > 0 && j > 0 {
|
||||
let old_spec = ¤t[i - 1].spec;
|
||||
let new_spec = &target[j - 1].spec;
|
||||
|
||||
if old_spec.is_exact(new_spec) {
|
||||
ops.push(PatchOp::Keep(i - 1, j - 1));
|
||||
i -= 1;
|
||||
j -= 1;
|
||||
} else if old_spec.is_similar(new_spec)
|
||||
&& lcs[i - 1][j - 1] >= lcs[i - 1][j]
|
||||
&& lcs[i - 1][j - 1] >= lcs[i][j - 1]
|
||||
{
|
||||
ops.push(PatchOp::Update(i - 1, j - 1));
|
||||
i -= 1;
|
||||
j -= 1;
|
||||
} else if lcs[i][j - 1] >= lcs[i - 1][j] {
|
||||
ops.push(PatchOp::Insert(j - 1));
|
||||
j -= 1;
|
||||
} else {
|
||||
ops.push(PatchOp::Delete(i - 1));
|
||||
i -= 1;
|
||||
}
|
||||
} else if j > 0 {
|
||||
ops.push(PatchOp::Insert(j - 1));
|
||||
j -= 1;
|
||||
} else {
|
||||
ops.push(PatchOp::Delete(i - 1));
|
||||
i -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
ops.reverse();
|
||||
ops
|
||||
}
|
||||
|
||||
fn update_block_props(
|
||||
doc: &Doc,
|
||||
blocks_map: &mut Map,
|
||||
node: &StoredNode,
|
||||
target: &BlockSpec,
|
||||
preserve_text: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
let Some(mut block) = blocks_map.get(&node.id).and_then(|v| v.to_map()) else {
|
||||
return Err(ParseError::ParserError(format!("Block {} not found", node.id)));
|
||||
};
|
||||
|
||||
let preserve = match target.flavour {
|
||||
BlockFlavour::Image
|
||||
| BlockFlavour::Table
|
||||
| BlockFlavour::Bookmark
|
||||
| BlockFlavour::EmbedYoutube
|
||||
| BlockFlavour::EmbedIframe => preserve_text,
|
||||
_ => preserve_text || text_delta_eq(&node.spec.text, &target.text),
|
||||
};
|
||||
|
||||
apply_block_spec(
|
||||
doc,
|
||||
&mut block,
|
||||
target,
|
||||
ApplyBlockOptions {
|
||||
preserve_text: preserve,
|
||||
clear_missing: true,
|
||||
},
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sync_children(doc: &Doc, blocks_map: &mut Map, block_id: &str, children: &[String]) -> Result<(), ParseError> {
|
||||
let Some(mut block) = blocks_map.get(block_id).and_then(|v| v.to_map()) else {
|
||||
return Err(ParseError::ParserError("Block not found".into()));
|
||||
};
|
||||
|
||||
let current_children = collect_child_ids(&block);
|
||||
if current_children != children {
|
||||
insert_children(doc, &mut block, children)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_tree_ids(node: &StoredNode, output: &mut Vec<String>) {
|
||||
output.push(node.id.clone());
|
||||
for child in &node.children {
|
||||
collect_tree_ids(child, output);
|
||||
}
|
||||
}
|
||||
|
||||
fn check_limits(current: &[StoredNode], target: &[BlockNode]) -> Result<(), ParseError> {
|
||||
let current_count = count_tree_nodes(current);
|
||||
let target_count = count_tree_nodes(target);
|
||||
|
||||
if current_count > MAX_BLOCKS || target_count > MAX_BLOCKS {
|
||||
return Err(ParseError::ParserError("block_count_too_large".into()));
|
||||
}
|
||||
|
||||
if current_count.saturating_mul(target_count) > MAX_LCS_CELLS {
|
||||
return Err(ParseError::ParserError("diff_matrix_too_large".into()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::{Any, DocOptions, TextDeltaOp, TextInsert};
|
||||
|
||||
use super::{super::builder::text_ops_from_plain, *};
|
||||
use crate::doc_parser::{
|
||||
block_spec::BlockType, blocksuite::get_string, build_full_doc, markdown::MAX_MARKDOWN_CHARS, parse_doc_to_markdown,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn test_compute_text_diff_simple() {
|
||||
let ops = text_ops_from_plain("hello world");
|
||||
assert_eq!(ops.len(), 1);
|
||||
match &ops[0] {
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text),
|
||||
format: None,
|
||||
} => {
|
||||
assert_eq!(text, "hello world");
|
||||
}
|
||||
_ => panic!("unexpected delta op"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_content_block_similarity() {
|
||||
let b1 = BlockSpec {
|
||||
flavour: BlockFlavour::Paragraph,
|
||||
block_type: Some(BlockType::H1),
|
||||
text: text_ops_from_plain("Hello"),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
let b2 = BlockSpec {
|
||||
flavour: BlockFlavour::Paragraph,
|
||||
block_type: Some(BlockType::H1),
|
||||
text: text_ops_from_plain("World"),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
let b3 = BlockSpec {
|
||||
flavour: BlockFlavour::Paragraph,
|
||||
block_type: Some(BlockType::H2),
|
||||
text: text_ops_from_plain("Hello"),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
|
||||
assert!(b1.is_similar(&b2));
|
||||
assert!(!b1.is_similar(&b3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_roundtrip() {
|
||||
let initial_md = "# Test Document\n\nFirst paragraph.\n\nSecond paragraph.";
|
||||
let doc_id = "update-test";
|
||||
|
||||
let initial_bin = build_full_doc("Test Document", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let updated_md = "# Test Document\n\nFirst paragraph.\n\nModified second paragraph.\n\nNew third paragraph.";
|
||||
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
assert!(!delta.is_empty(), "Delta should contain changes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_does_not_update_page_title() {
|
||||
let initial_md = "# Original Title\n\nContent here.";
|
||||
let doc_id = "title-test";
|
||||
|
||||
let initial_bin = build_full_doc("Original Title", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let updated_md = "# New Title\n\nContent here.";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("Should apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map exists");
|
||||
let mut title = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
|
||||
{
|
||||
title = get_string(&block_map, "prop:title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(title.as_deref(), Some("Original Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_no_changes() {
|
||||
let markdown = "# Same Title\n\nSame content.";
|
||||
let doc_id = "no-change-test";
|
||||
|
||||
let initial_bin = build_full_doc("Same Title", markdown, doc_id).expect("Should create initial doc");
|
||||
let delta = update_doc(&initial_bin, markdown, doc_id).expect("Should compute delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
doc
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("Should apply delta even with no changes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_ignores_ai_editable_comments() {
|
||||
let markdown = "Plain paragraph.";
|
||||
let doc_id = "ai-comment-test";
|
||||
|
||||
let initial_bin = build_full_doc("Title", markdown, doc_id).expect("Should create initial doc");
|
||||
|
||||
let ai_markdown = parse_doc_to_markdown(initial_bin.clone(), doc_id.to_string(), true, None)
|
||||
.expect("parse doc")
|
||||
.markdown;
|
||||
assert!(ai_markdown.contains("block_id="));
|
||||
|
||||
let delta = update_doc(&initial_bin, &ai_markdown, doc_id).expect("Should compute delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("Should apply delta");
|
||||
|
||||
let before = parse_doc_to_markdown(initial_bin, doc_id.to_string(), false, None)
|
||||
.expect("parse before")
|
||||
.markdown;
|
||||
let after = parse_doc_to_markdown(doc.encode_update_v1().unwrap(), doc_id.to_string(), false, None)
|
||||
.expect("parse after")
|
||||
.markdown;
|
||||
|
||||
assert_eq!(after, before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_add_block() {
|
||||
let initial_md = "# Add Block Test\n\nOriginal paragraph.";
|
||||
let doc_id = "add-block-test";
|
||||
|
||||
let initial_bin = build_full_doc("Add Block Test", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
initial_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
|
||||
let updated_md = "# Add Block Test\n\nOriginal paragraph.\n\nNew paragraph added.";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
assert!(!delta.is_empty(), "Delta should contain changes");
|
||||
|
||||
let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("Should apply delta with new block");
|
||||
|
||||
let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
assert!(
|
||||
updated_count > initial_count,
|
||||
"Expected more blocks after insert, got {updated_count} vs {initial_count}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_delete_block() {
|
||||
let initial_md = "# Delete Block Test\n\nFirst paragraph.\n\nSecond paragraph to delete.";
|
||||
let doc_id = "delete-block-test";
|
||||
|
||||
let initial_bin = build_full_doc("Delete Block Test", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
initial_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
|
||||
let updated_md = "# Delete Block Test\n\nFirst paragraph.";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
assert!(!delta.is_empty(), "Delta should contain changes");
|
||||
|
||||
let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("Should apply delta with block deletion");
|
||||
|
||||
let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
assert!(
|
||||
updated_count < initial_count,
|
||||
"Expected fewer blocks after deletion, got {updated_count} vs {initial_count}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_update_image_caption() {
|
||||
let initial_md = "";
|
||||
let doc_id = "image-update-test";
|
||||
let initial_bin = build_full_doc("Image", initial_md, doc_id).expect("create doc");
|
||||
|
||||
let updated_md = "";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut caption = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image")
|
||||
{
|
||||
caption = get_string(&block_map, "prop:caption");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(caption.as_deref(), Some("New Caption"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_update_table_cell() {
|
||||
let initial_md = "| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let doc_id = "table-update-test";
|
||||
let initial_bin = build_full_doc("Table", initial_md, doc_id).expect("create doc");
|
||||
|
||||
let updated_md = "| A | B |\n| --- | --- |\n| 1 | 9 |";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut found = false;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table")
|
||||
{
|
||||
for key in block_map.keys() {
|
||||
if key.starts_with("prop:cells.")
|
||||
&& key.ends_with(".text")
|
||||
&& let Some(value) = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::String(value) => Some(value),
|
||||
_ => None,
|
||||
})
|
||||
&& value == "9"
|
||||
{
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_concurrent_merge_simulation() {
|
||||
let base_md = "# Concurrent Test\n\nBase paragraph.";
|
||||
let doc_id = "concurrent-test";
|
||||
|
||||
let base_bin = build_full_doc("Concurrent Test", base_md, doc_id).expect("Should create base doc");
|
||||
|
||||
let mut base_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
base_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base");
|
||||
let base_count = base_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
|
||||
let client_a_md = "# Concurrent Test\n\nModified by client A.";
|
||||
let delta_a = update_doc(&base_bin, client_a_md, doc_id).expect("Delta A");
|
||||
|
||||
let client_b_md = "# Concurrent Test\n\nBase paragraph.\n\nAdded by client B.";
|
||||
let delta_b = update_doc(&base_bin, client_b_md, doc_id).expect("Delta B");
|
||||
|
||||
let mut final_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
final_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base");
|
||||
final_doc.apply_update_from_binary_v1(&delta_a).expect("Apply delta A");
|
||||
final_doc.apply_update_from_binary_v1(&delta_b).expect("Apply delta B");
|
||||
|
||||
let final_count = final_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
assert!(
|
||||
final_count > base_count,
|
||||
"Expected merged blocks after concurrent updates, got {final_count} vs {base_count}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_empty_binary_errors() {
|
||||
let markdown = "# New Document\n\nCreated from empty binary.";
|
||||
let doc_id = "empty-fallback-test";
|
||||
|
||||
let result = update_doc(&[], markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
|
||||
let result = update_doc(&[0, 0], markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_markdown_too_large() {
|
||||
let initial_md = "# Title\n\nContent.";
|
||||
let doc_id = "size-limit-test";
|
||||
let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1);
|
||||
let result = update_doc(&initial_bin, &markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_rejects_unsupported_markdown() {
|
||||
let initial_md = "# Title\n\nContent.";
|
||||
let doc_id = "unsupported-test";
|
||||
let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let markdown = "# Title\n\n<div>HTML</div>";
|
||||
let result = update_doc(&initial_bin, markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user