mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-14 08:36:22 +08:00
feat(core): import progress & perf (#15197)
#### PR Dependency Tree * **PR #15197** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added a new import pipeline with “plan then commit” batch handling for Markdown, Notion HTML, Obsidian, and Bear backups. * Enabled native import sessions with progress, cancellation, and batch-by-batch committing (including assets, folders, icons, and tags). * Added web preflight limits for ZIP and multi-file imports. * **Bug Fixes** * Improved import error/warning reporting and continued processing when some items fail. * Strengthened snapshot-based file/directory picking to preserve paths. * **Chores** * Updated project packaging/configuration for the new import workflow. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
@@ -1,601 +0,0 @@
|
||||
use y_octo::{Any, Map, TextAttributes, TextDeltaOp, TextInsert};
|
||||
|
||||
use super::{
|
||||
ParseError,
|
||||
blocksuite::get_string,
|
||||
schema::{
|
||||
PROP_CAPTION, PROP_CHECKED, PROP_COLUMN_ID_SUFFIX, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER,
|
||||
PROP_ORDER_SUFFIX, PROP_ROW_ID_SUFFIX, PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL,
|
||||
PROP_VIDEO_ID, PROP_WIDTH, SYS_FLAVOUR, table_cell_text_key,
|
||||
},
|
||||
table::{MarkdownTableOptions, render_markdown_table},
|
||||
value::{value_to_f64, value_to_string},
|
||||
};
|
||||
|
||||
/// Block flavours used in AFFiNE documents.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BlockFlavour {
|
||||
Paragraph,
|
||||
List,
|
||||
Code,
|
||||
Divider,
|
||||
Image,
|
||||
Table,
|
||||
Bookmark,
|
||||
EmbedYoutube,
|
||||
EmbedIframe,
|
||||
Callout,
|
||||
}
|
||||
|
||||
impl BlockFlavour {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
BlockFlavour::Paragraph => "affine:paragraph",
|
||||
BlockFlavour::List => "affine:list",
|
||||
BlockFlavour::Code => "affine:code",
|
||||
BlockFlavour::Divider => "affine:divider",
|
||||
BlockFlavour::Image => "affine:image",
|
||||
BlockFlavour::Table => "affine:table",
|
||||
BlockFlavour::Bookmark => "affine:bookmark",
|
||||
BlockFlavour::EmbedYoutube => "affine:embed-youtube",
|
||||
BlockFlavour::EmbedIframe => "affine:embed-iframe",
|
||||
BlockFlavour::Callout => "affine:callout",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"affine:paragraph" => Some(BlockFlavour::Paragraph),
|
||||
"affine:list" => Some(BlockFlavour::List),
|
||||
"affine:code" => Some(BlockFlavour::Code),
|
||||
"affine:divider" => Some(BlockFlavour::Divider),
|
||||
"affine:image" => Some(BlockFlavour::Image),
|
||||
"affine:table" => Some(BlockFlavour::Table),
|
||||
"affine:bookmark" => Some(BlockFlavour::Bookmark),
|
||||
"affine:embed-youtube" => Some(BlockFlavour::EmbedYoutube),
|
||||
"affine:embed-iframe" => Some(BlockFlavour::EmbedIframe),
|
||||
"affine:callout" => Some(BlockFlavour::Callout),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Block types for paragraphs and lists.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum BlockType {
|
||||
// Paragraph types
|
||||
Text,
|
||||
H1,
|
||||
H2,
|
||||
H3,
|
||||
H4,
|
||||
H5,
|
||||
H6,
|
||||
Quote,
|
||||
// List types
|
||||
Bulleted,
|
||||
Numbered,
|
||||
Todo,
|
||||
// Preserve unknown types when loading from ydoc.
|
||||
Unknown(String),
|
||||
}
|
||||
|
||||
impl BlockType {
|
||||
pub fn as_str(&self) -> &str {
|
||||
match self {
|
||||
BlockType::Text => "text",
|
||||
BlockType::H1 => "h1",
|
||||
BlockType::H2 => "h2",
|
||||
BlockType::H3 => "h3",
|
||||
BlockType::H4 => "h4",
|
||||
BlockType::H5 => "h5",
|
||||
BlockType::H6 => "h6",
|
||||
BlockType::Quote => "quote",
|
||||
BlockType::Bulleted => "bulleted",
|
||||
BlockType::Numbered => "numbered",
|
||||
BlockType::Todo => "todo",
|
||||
BlockType::Unknown(value) => value.as_str(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"text" => Some(BlockType::Text),
|
||||
"h1" => Some(BlockType::H1),
|
||||
"h2" => Some(BlockType::H2),
|
||||
"h3" => Some(BlockType::H3),
|
||||
"h4" => Some(BlockType::H4),
|
||||
"h5" => Some(BlockType::H5),
|
||||
"h6" => Some(BlockType::H6),
|
||||
"quote" => Some(BlockType::Quote),
|
||||
"bulleted" => Some(BlockType::Bulleted),
|
||||
"numbered" => Some(BlockType::Numbered),
|
||||
"todo" => Some(BlockType::Todo),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str_lossy(value: String) -> Self {
|
||||
Self::from_str(&value).unwrap_or(BlockType::Unknown(value))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct ImageSpec {
|
||||
pub(super) source_id: String,
|
||||
pub(super) caption: Option<String>,
|
||||
pub(super) width: Option<f64>,
|
||||
pub(super) height: Option<f64>,
|
||||
}
|
||||
|
||||
impl ImageSpec {
|
||||
pub(super) fn render_markdown(&self) -> String {
|
||||
let blob_url = format!("blob://{}", self.source_id);
|
||||
let caption = self.caption.as_deref().unwrap_or("");
|
||||
|
||||
if self.width.is_some() || self.height.is_some() || !caption.is_empty() {
|
||||
let width_text = self
|
||||
.width
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "auto".into());
|
||||
let height_text = self
|
||||
.height
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "auto".into());
|
||||
return format!(
|
||||
"<img\n src=\"{blob_url}\"\n alt=\"{caption}\"\n width=\"{width_text}\"\n height=\"{height_text}\"\n/>\n\n"
|
||||
);
|
||||
}
|
||||
|
||||
let alt = if caption.is_empty() {
|
||||
self.source_id.as_str()
|
||||
} else {
|
||||
caption
|
||||
};
|
||||
format!("\n\n\n")
|
||||
}
|
||||
|
||||
pub(super) fn from_block_map(block: &Map) -> Self {
|
||||
let source_id = get_string(block, PROP_SOURCE_ID).unwrap_or_default();
|
||||
let caption = get_string(block, PROP_CAPTION);
|
||||
let width = block.get(PROP_WIDTH).and_then(value_to_f64);
|
||||
let height = block.get(PROP_HEIGHT).and_then(value_to_f64);
|
||||
ImageSpec {
|
||||
source_id,
|
||||
caption,
|
||||
width,
|
||||
height,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn normalize_source(src: &str) -> Result<String, ParseError> {
|
||||
let trimmed = src.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(ParseError::ParserError("invalid_image_source".into()));
|
||||
}
|
||||
if let Some(rest) = trimmed.strip_prefix("blob://") {
|
||||
if rest.is_empty() {
|
||||
return Err(ParseError::ParserError("invalid_image_source".into()));
|
||||
}
|
||||
return Ok(rest.to_string());
|
||||
}
|
||||
if !trimmed.contains('/') && !trimmed.contains(':') {
|
||||
return Ok(trimmed.to_string());
|
||||
}
|
||||
if let Some(pos) = trimmed.rfind("/blobs/") {
|
||||
let id = &trimmed[pos + "/blobs/".len()..];
|
||||
let id = id.split(['?', '#']).next().unwrap_or("");
|
||||
if !id.is_empty() {
|
||||
return Ok(id.to_string());
|
||||
}
|
||||
}
|
||||
Err(ParseError::ParserError("unsupported_image_source".into()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct TableSpec {
|
||||
pub(super) rows: Vec<Vec<String>>,
|
||||
}
|
||||
|
||||
impl TableSpec {
|
||||
pub(super) fn from_block_map(block: &Map) -> Self {
|
||||
let mut row_entries: Vec<(String, String)> = Vec::new();
|
||||
let mut column_entries: Vec<(String, String)> = Vec::new();
|
||||
|
||||
for key in block.keys() {
|
||||
if key.starts_with(PROP_ROWS_PREFIX)
|
||||
&& key.ends_with(PROP_ROW_ID_SUFFIX)
|
||||
&& let Some(row_id) = block.get(key).and_then(|value| value_to_string(&value))
|
||||
{
|
||||
let base = key.trim_end_matches(PROP_ROW_ID_SUFFIX);
|
||||
let order_key = format!("{base}{PROP_ORDER_SUFFIX}");
|
||||
let order = block
|
||||
.get(&order_key)
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
row_entries.push((order, row_id));
|
||||
}
|
||||
if key.starts_with(PROP_COLUMNS_PREFIX)
|
||||
&& key.ends_with(PROP_COLUMN_ID_SUFFIX)
|
||||
&& let Some(column_id) = block.get(key).and_then(|value| value_to_string(&value))
|
||||
{
|
||||
let base = key.trim_end_matches(PROP_COLUMN_ID_SUFFIX);
|
||||
let order_key = format!("{base}{PROP_ORDER_SUFFIX}");
|
||||
let order = block
|
||||
.get(&order_key)
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
column_entries.push((order, column_id));
|
||||
}
|
||||
}
|
||||
|
||||
row_entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
column_entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for (_, row_id) in row_entries {
|
||||
let mut row = Vec::new();
|
||||
for (_, column_id) in &column_entries {
|
||||
let cell_key = table_cell_text_key(&row_id, column_id);
|
||||
let cell_text = block
|
||||
.get(&cell_key)
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
row.push(cell_text);
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
Self { rows }
|
||||
}
|
||||
|
||||
pub(super) fn render_markdown(&self) -> Option<String> {
|
||||
let options = MarkdownTableOptions::new(false, "<br />", true);
|
||||
render_markdown_table(&self.rows, options)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct BookmarkSpec {
|
||||
pub(super) url: String,
|
||||
pub(super) caption: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct EmbedYoutubeSpec {
|
||||
pub(super) video_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct EmbedIframeSpec {
|
||||
pub(super) url: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct BlockSpec {
|
||||
pub(super) flavour: BlockFlavour,
|
||||
pub(super) block_type: Option<BlockType>,
|
||||
pub(super) text: Vec<TextDeltaOp>,
|
||||
pub(super) checked: Option<bool>,
|
||||
pub(super) language: Option<String>,
|
||||
pub(super) order: Option<i64>,
|
||||
pub(super) image: Option<ImageSpec>,
|
||||
pub(super) table: Option<TableSpec>,
|
||||
pub(super) bookmark: Option<BookmarkSpec>,
|
||||
pub(super) embed_youtube: Option<EmbedYoutubeSpec>,
|
||||
pub(super) embed_iframe: Option<EmbedIframeSpec>,
|
||||
}
|
||||
|
||||
impl BlockSpec {
|
||||
pub(super) fn is_exact(&self, other: &BlockSpec) -> bool {
|
||||
self.flavour == other.flavour
|
||||
&& self.block_type == other.block_type
|
||||
&& self.checked == other.checked
|
||||
&& self.language == other.language
|
||||
&& self.image == other.image
|
||||
&& self.table == other.table
|
||||
&& self.bookmark == other.bookmark
|
||||
&& self.embed_youtube == other.embed_youtube
|
||||
&& self.embed_iframe == other.embed_iframe
|
||||
&& text_delta_eq(&self.text, &other.text)
|
||||
}
|
||||
|
||||
pub(super) fn is_similar(&self, other: &BlockSpec) -> bool {
|
||||
self.flavour == other.flavour && self.block_type == other.block_type
|
||||
}
|
||||
|
||||
pub(super) fn block_type_str(&self) -> Option<&str> {
|
||||
self.block_type.as_ref().map(BlockType::as_str)
|
||||
}
|
||||
|
||||
pub(super) fn from_block_map(block: &Map) -> Result<Self, ParseError> {
|
||||
let flavour_value = get_string(block, SYS_FLAVOUR).unwrap_or_default();
|
||||
let Some(flavour) = BlockFlavour::from_str(&flavour_value) else {
|
||||
return Err(ParseError::ParserError(format!(
|
||||
"unsupported block flavour: {flavour_value}"
|
||||
)));
|
||||
};
|
||||
Ok(Self::from_block_map_with_flavour(block, flavour))
|
||||
}
|
||||
|
||||
pub(super) fn from_block_map_with_flavour(block: &Map, flavour: BlockFlavour) -> Self {
|
||||
if flavour == BlockFlavour::Image {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: Some(ImageSpec::from_block_map(block)),
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::Table {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: Some(TableSpec::from_block_map(block)),
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::Bookmark {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: Some(BookmarkSpec {
|
||||
url: get_string(block, PROP_URL).unwrap_or_default(),
|
||||
caption: get_string(block, PROP_CAPTION),
|
||||
}),
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::EmbedYoutube {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: Some(EmbedYoutubeSpec {
|
||||
video_id: get_string(block, PROP_VIDEO_ID).unwrap_or_default(),
|
||||
}),
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::EmbedIframe {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: Some(EmbedIframeSpec {
|
||||
url: get_string(block, PROP_URL).unwrap_or_default(),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
let block_type = match get_string(block, PROP_TYPE) {
|
||||
Some(value) => Some(BlockType::from_str_lossy(value)),
|
||||
None => match flavour {
|
||||
BlockFlavour::Paragraph => Some(BlockType::Text),
|
||||
BlockFlavour::List => Some(BlockType::Bulleted),
|
||||
_ => None,
|
||||
},
|
||||
};
|
||||
|
||||
let text = block
|
||||
.get(PROP_TEXT)
|
||||
.and_then(|v| v.to_text())
|
||||
.map(|text| text.to_delta())
|
||||
.unwrap_or_default();
|
||||
|
||||
let checked = block.get(PROP_CHECKED).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::True => Some(true),
|
||||
Any::False => Some(false),
|
||||
_ => None,
|
||||
});
|
||||
let language = get_string(block, PROP_LANGUAGE);
|
||||
let order = block.get(PROP_ORDER).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::Integer(value) => Some(value as i64),
|
||||
Any::BigInt64(value) => Some(value),
|
||||
Any::Float32(value) => Some(value.0 as i64),
|
||||
Any::Float64(value) => Some(value.0 as i64),
|
||||
_ => None,
|
||||
});
|
||||
|
||||
BlockSpec {
|
||||
flavour,
|
||||
block_type,
|
||||
text,
|
||||
checked,
|
||||
language,
|
||||
order,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct BlockNode {
|
||||
pub(super) spec: BlockSpec,
|
||||
pub(super) children: Vec<BlockNode>,
|
||||
}
|
||||
|
||||
pub(super) trait TreeNode {
|
||||
fn children(&self) -> &[Self]
|
||||
where
|
||||
Self: Sized;
|
||||
}
|
||||
|
||||
impl TreeNode for BlockNode {
|
||||
fn children(&self) -> &[BlockNode] {
|
||||
&self.children
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn count_tree_nodes<T: TreeNode>(nodes: &[T]) -> usize {
|
||||
nodes.iter().map(|node| 1 + count_tree_nodes(node.children())).sum()
|
||||
}
|
||||
|
||||
pub(super) fn text_delta_eq(a: &[TextDeltaOp], b: &[TextDeltaOp]) -> bool {
|
||||
if a.len() != b.len() {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (left, right) in a.iter().zip(b.iter()) {
|
||||
match (left, right) {
|
||||
(
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text_a),
|
||||
format: format_a,
|
||||
},
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text_b),
|
||||
format: format_b,
|
||||
},
|
||||
) => {
|
||||
if text_a != text_b {
|
||||
return false;
|
||||
}
|
||||
if !attrs_eq(format_a.as_ref(), format_b.as_ref()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
_ => return false,
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
fn attrs_eq(a: Option<&TextAttributes>, b: Option<&TextAttributes>) -> bool {
|
||||
match (a, b) {
|
||||
(None, None) => true,
|
||||
(Some(left), Some(right)) => {
|
||||
if left.len() != right.len() {
|
||||
return false;
|
||||
}
|
||||
left.iter().all(|(key, value)| right.get(key) == Some(value))
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::{super::write::builder::text_ops_from_plain, *};
|
||||
use crate::doc_parser::build_full_doc;
|
||||
|
||||
fn spec_from_markdown(markdown: &str, doc_id: &str, flavour: &str) -> BlockSpec {
|
||||
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, SYS_FLAVOUR).as_deref() == Some(flavour)
|
||||
{
|
||||
return BlockSpec::from_block_map(&block_map).expect("spec");
|
||||
}
|
||||
}
|
||||
|
||||
panic!("block not found: {flavour}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_paragraph() {
|
||||
let spec = spec_from_markdown("Plain paragraph.", "block-spec-paragraph", "affine:paragraph");
|
||||
assert_eq!(spec.flavour, BlockFlavour::Paragraph);
|
||||
assert_eq!(spec.block_type, Some(BlockType::Text));
|
||||
assert_eq!(spec.text, text_ops_from_plain("Plain paragraph."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_list_checked() {
|
||||
let spec = spec_from_markdown("- [x] Done", "block-spec-list", "affine:list");
|
||||
assert_eq!(spec.flavour, BlockFlavour::List);
|
||||
assert_eq!(spec.block_type, Some(BlockType::Todo));
|
||||
assert_eq!(spec.checked, Some(true));
|
||||
assert_eq!(spec.text, text_ops_from_plain("Done"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_image() {
|
||||
let spec = spec_from_markdown("", "block-spec-image", "affine:image");
|
||||
assert_eq!(spec.flavour, BlockFlavour::Image);
|
||||
let image = spec.image.expect("image spec");
|
||||
assert_eq!(image.source_id, "image-id");
|
||||
assert_eq!(image.caption.as_deref(), Some("Alt"));
|
||||
assert_eq!(image.width, None);
|
||||
assert_eq!(image.height, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_table() {
|
||||
let spec = spec_from_markdown(
|
||||
"| A | B |\n| --- | --- |\n| 1 | 2 |",
|
||||
"block-spec-table",
|
||||
"affine:table",
|
||||
);
|
||||
assert_eq!(spec.flavour, BlockFlavour::Table);
|
||||
let table = spec.table.expect("table spec");
|
||||
assert_eq!(
|
||||
table.rows,
|
||||
vec![
|
||||
vec!["A".to_string(), "B".to_string()],
|
||||
vec!["1".to_string(), "2".to_string()]
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_embed_iframe() {
|
||||
let spec = spec_from_markdown(
|
||||
r#"<iframe src="https://example.com/embed"></iframe>"#,
|
||||
"block-spec-embed-iframe",
|
||||
"affine:embed-iframe",
|
||||
);
|
||||
assert_eq!(spec.flavour, BlockFlavour::EmbedIframe);
|
||||
assert_eq!(spec.embed_iframe.as_ref().unwrap().url, "https://example.com/embed");
|
||||
}
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use y_octo::Map;
|
||||
|
||||
use super::{
|
||||
schema::{SYS_CHILDREN, SYS_FLAVOUR, SYS_ID},
|
||||
value::value_to_string,
|
||||
};
|
||||
|
||||
pub(super) struct BlockIndex {
|
||||
pub(super) block_pool: HashMap<String, Map>,
|
||||
pub(super) parent_lookup: HashMap<String, String>,
|
||||
}
|
||||
|
||||
pub(super) struct DocContext {
|
||||
pub(super) block_pool: HashMap<String, Map>,
|
||||
pub(super) parent_lookup: HashMap<String, String>,
|
||||
pub(super) root_block_id: String,
|
||||
}
|
||||
|
||||
pub(super) struct BlockWalker {
|
||||
queue: Vec<(Option<String>, String)>,
|
||||
visited: HashSet<String>,
|
||||
}
|
||||
|
||||
pub(super) fn build_block_index(blocks_map: &Map) -> BlockIndex {
|
||||
let mut block_pool: HashMap<String, Map> = HashMap::new();
|
||||
let mut parent_lookup: HashMap<String, String> = HashMap::new();
|
||||
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& let Some(block_id) = get_block_id(&block_map)
|
||||
{
|
||||
for child_id in collect_child_ids(&block_map) {
|
||||
parent_lookup.insert(child_id, block_id.clone());
|
||||
}
|
||||
block_pool.insert(block_id, block_map);
|
||||
}
|
||||
}
|
||||
|
||||
BlockIndex {
|
||||
block_pool,
|
||||
parent_lookup,
|
||||
}
|
||||
}
|
||||
|
||||
impl DocContext {
|
||||
pub(super) fn from_blocks_map(blocks_map: &Map, root_flavour: &str) -> Option<Self> {
|
||||
let BlockIndex {
|
||||
block_pool,
|
||||
parent_lookup,
|
||||
} = build_block_index(blocks_map);
|
||||
|
||||
let root_block_id = find_block_id_by_flavour(&block_pool, root_flavour)?;
|
||||
Some(Self {
|
||||
block_pool,
|
||||
parent_lookup,
|
||||
root_block_id,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn walker(&self) -> BlockWalker {
|
||||
BlockWalker::new(&self.root_block_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl BlockWalker {
|
||||
fn new(root_block_id: &str) -> Self {
|
||||
let mut visited = HashSet::new();
|
||||
visited.insert(root_block_id.to_string());
|
||||
|
||||
Self {
|
||||
queue: vec![(None, root_block_id.to_string())],
|
||||
visited,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn next(&mut self) -> Option<(Option<String>, String)> {
|
||||
self.queue.pop()
|
||||
}
|
||||
|
||||
pub(super) fn enqueue_children(&mut self, parent_block_id: &str, block: &Map) {
|
||||
let mut child_ids = collect_child_ids(block);
|
||||
for child_id in child_ids.drain(..).rev() {
|
||||
if self.visited.insert(child_id.clone()) {
|
||||
self.queue.push((Some(parent_block_id.to_string()), child_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn find_block_id_by_flavour(block_pool: &HashMap<String, Map>, flavour: &str) -> Option<String> {
|
||||
block_pool.iter().find_map(|(id, block)| {
|
||||
get_flavour(block)
|
||||
.filter(|block_flavour| block_flavour == flavour)
|
||||
.map(|_| id.clone())
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn collect_child_ids(block: &Map) -> Vec<String> {
|
||||
block
|
||||
.get(SYS_CHILDREN)
|
||||
.and_then(|value| value.to_array())
|
||||
.map(|array| {
|
||||
array
|
||||
.iter()
|
||||
.filter_map(|value| value_to_string(&value))
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(super) fn get_block_id(block: &Map) -> Option<String> {
|
||||
get_string(block, SYS_ID)
|
||||
}
|
||||
|
||||
pub(super) fn get_flavour(block: &Map) -> Option<String> {
|
||||
get_string(block, SYS_FLAVOUR)
|
||||
}
|
||||
|
||||
pub(super) fn get_string(block: &Map, key: &str) -> Option<String> {
|
||||
block.get(key).and_then(|value| value_to_string(&value))
|
||||
}
|
||||
|
||||
pub(super) fn get_list_depth(
|
||||
block_id: &str,
|
||||
parent_lookup: &HashMap<String, String>,
|
||||
blocks: &HashMap<String, Map>,
|
||||
) -> usize {
|
||||
let mut depth = 0;
|
||||
let mut current_id = block_id.to_string();
|
||||
|
||||
while let Some(parent_id) = parent_lookup.get(¤t_id) {
|
||||
if let Some(parent_block) = blocks.get(parent_id)
|
||||
&& get_flavour(parent_block).as_deref() == Some("affine:list")
|
||||
{
|
||||
depth += 1;
|
||||
current_id = parent_id.clone();
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
depth
|
||||
}
|
||||
|
||||
pub(super) fn nearest_by_flavour(
|
||||
start: &str,
|
||||
flavour: &str,
|
||||
parent_lookup: &HashMap<String, String>,
|
||||
blocks: &HashMap<String, Map>,
|
||||
) -> Option<Map> {
|
||||
let mut cursor = Some(start.to_string());
|
||||
while let Some(node) = cursor {
|
||||
if let Some(block) = blocks.get(&node)
|
||||
&& get_flavour(block).as_deref() == Some(flavour)
|
||||
{
|
||||
return Some(block.clone());
|
||||
}
|
||||
cursor = parent_lookup.get(&node).cloned();
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn find_child_id_by_flavour(
|
||||
parent: &Map,
|
||||
block_pool: &HashMap<String, Map>,
|
||||
flavour: &str,
|
||||
) -> Option<String> {
|
||||
collect_child_ids(parent).into_iter().find(|id| {
|
||||
block_pool
|
||||
.get(id)
|
||||
.and_then(get_flavour)
|
||||
.as_deref()
|
||||
.is_some_and(|value| value == flavour)
|
||||
})
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
use y_octo::{Doc, DocOptions};
|
||||
|
||||
use super::ParseError;
|
||||
|
||||
pub(super) fn is_empty_doc(binary: &[u8]) -> bool {
|
||||
binary.is_empty() || binary == [0, 0]
|
||||
}
|
||||
|
||||
pub(super) fn load_doc(binary: &[u8], doc_id: Option<&str>) -> Result<Doc, ParseError> {
|
||||
if is_empty_doc(binary) {
|
||||
return Err(ParseError::InvalidBinary);
|
||||
}
|
||||
|
||||
let mut doc = build_doc(doc_id);
|
||||
doc
|
||||
.apply_update_from_binary_v1(binary)
|
||||
.map_err(|_| ParseError::InvalidBinary)?;
|
||||
Ok(doc)
|
||||
}
|
||||
|
||||
pub(super) fn load_doc_or_new(binary: &[u8]) -> Result<Doc, ParseError> {
|
||||
if is_empty_doc(binary) {
|
||||
return Ok(DocOptions::new().build());
|
||||
}
|
||||
|
||||
let mut doc = DocOptions::new().build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(binary)
|
||||
.map_err(|_| ParseError::InvalidBinary)?;
|
||||
Ok(doc)
|
||||
}
|
||||
|
||||
fn build_doc(doc_id: Option<&str>) -> Doc {
|
||||
let options = DocOptions::new();
|
||||
match doc_id {
|
||||
Some(doc_id) => options.with_guid(doc_id.to_string()).build(),
|
||||
None => options.build(),
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use thiserror::Error;
|
||||
use y_octo::JwstCodecError;
|
||||
|
||||
#[derive(Error, Debug, Serialize, Deserialize)]
|
||||
pub enum ParseError {
|
||||
#[error("doc_not_found")]
|
||||
DocNotFound,
|
||||
#[error("invalid_binary")]
|
||||
InvalidBinary,
|
||||
#[error("sqlite_error: {0}")]
|
||||
SqliteError(String),
|
||||
#[error("parser_error: {0}")]
|
||||
ParserError(String),
|
||||
#[error("unknown: {0}")]
|
||||
Unknown(String),
|
||||
}
|
||||
|
||||
impl From<JwstCodecError> for ParseError {
|
||||
fn from(value: JwstCodecError) -> Self {
|
||||
if matches!(
|
||||
value,
|
||||
JwstCodecError::DamagedDocumentJson
|
||||
| JwstCodecError::IncompleteDocument(_)
|
||||
| JwstCodecError::InvalidWriteBuffer(_)
|
||||
| JwstCodecError::UpdateInvalid(_)
|
||||
| JwstCodecError::StructClockInvalid { expect: _, actually: _ }
|
||||
| JwstCodecError::StructSequenceInvalid { client_id: _, clock: _ }
|
||||
| JwstCodecError::StructSequenceNotExists(_)
|
||||
| JwstCodecError::RootStructNotFound(_)
|
||||
| JwstCodecError::ParentNotFound
|
||||
| JwstCodecError::IndexOutOfBound(_)
|
||||
) {
|
||||
return ParseError::InvalidBinary;
|
||||
}
|
||||
Self::ParserError(value.to_string())
|
||||
}
|
||||
}
|
||||
@@ -1,842 +0,0 @@
|
||||
use std::{
|
||||
cell::RefCell,
|
||||
collections::{HashMap, HashSet},
|
||||
rc::{Rc, Weak},
|
||||
};
|
||||
|
||||
use y_octo::{AHashMap, Any, Map, Text, TextAttributes, TextDeltaOp, TextInsert, Value};
|
||||
|
||||
use super::{
|
||||
super::value::{
|
||||
any_as_string, any_as_u64, any_truthy, build_reference_payload, params_any_map_to_json, value_to_any,
|
||||
},
|
||||
inline::InlineStyle,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct InlineReference {
|
||||
ref_type: Option<String>,
|
||||
page_id: String,
|
||||
title: Option<String>,
|
||||
params: Option<AHashMap<String, Any>>,
|
||||
mode: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct InlineReferencePayload {
|
||||
pub(crate) doc_id: String,
|
||||
pub(crate) payload: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct DeltaToMdOptions {
|
||||
doc_url_prefix: Option<String>,
|
||||
}
|
||||
|
||||
impl DeltaToMdOptions {
|
||||
pub(crate) fn new(doc_url_prefix: Option<String>) -> Self {
|
||||
Self { doc_url_prefix }
|
||||
}
|
||||
|
||||
fn build_reference_link(&self, reference: &InlineReference) -> (String, String) {
|
||||
let title = reference.title.clone().unwrap_or_default();
|
||||
|
||||
if let Some(prefix) = self.doc_url_prefix.as_deref() {
|
||||
let prefix = prefix.trim_end_matches('/');
|
||||
return (title, format!("{}/{}", prefix, reference.page_id));
|
||||
}
|
||||
|
||||
let mut parts = Vec::new();
|
||||
parts.push(reference.ref_type.clone().unwrap_or_else(|| "LinkedPage".into()));
|
||||
parts.push(reference.page_id.clone());
|
||||
if let Some(mode) = reference.mode.as_ref() {
|
||||
parts.push(mode.clone());
|
||||
}
|
||||
|
||||
(title, parts.join(":"))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn text_to_inline_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option<String> {
|
||||
block
|
||||
.get(key)
|
||||
.and_then(|value| value.to_text())
|
||||
.map(|text| delta_to_inline_markdown(&text, options))
|
||||
}
|
||||
|
||||
pub(crate) fn delta_ops_to_markdown(ops: &[TextDeltaOp], options: &DeltaToMdOptions) -> String {
|
||||
delta_to_markdown_with_options(ops, options, true)
|
||||
}
|
||||
|
||||
pub(crate) fn delta_ops_to_plain_text(ops: &[TextDeltaOp]) -> String {
|
||||
let mut out = String::new();
|
||||
for op in ops {
|
||||
if let TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text),
|
||||
..
|
||||
} = op
|
||||
{
|
||||
out.push_str(text);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub(crate) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec<InlineReferencePayload> {
|
||||
let mut refs = Vec::new();
|
||||
let mut seen: HashSet<(String, String)> = HashSet::new();
|
||||
|
||||
for op in delta {
|
||||
let attrs = match op {
|
||||
TextDeltaOp::Insert {
|
||||
format: Some(format), ..
|
||||
} => format,
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let reference = match attrs.get(InlineStyle::Reference.key()).and_then(parse_inline_reference) {
|
||||
Some(reference) => reference,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let payload = match inline_reference_payload(&reference) {
|
||||
Some(payload) => payload,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let key = (reference.page_id.clone(), payload.clone());
|
||||
if seen.insert(key.clone()) {
|
||||
refs.push(InlineReferencePayload {
|
||||
doc_id: key.0,
|
||||
payload: key.1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
refs
|
||||
}
|
||||
|
||||
pub(crate) fn extract_inline_references_from_value(value: &Value) -> Vec<InlineReferencePayload> {
|
||||
if let Some(text) = value.to_text() {
|
||||
return extract_inline_references(&text.to_delta());
|
||||
}
|
||||
|
||||
let Some(any) = value_to_any(value) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
extract_inline_references_from_any(&any)
|
||||
}
|
||||
|
||||
fn extract_inline_references_from_any(value: &Any) -> Vec<InlineReferencePayload> {
|
||||
let Some(ops) = delta_ops_from_any(value) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut refs = Vec::new();
|
||||
let mut seen: HashSet<(String, String)> = HashSet::new();
|
||||
|
||||
for op in ops {
|
||||
let reference = match op
|
||||
.attributes
|
||||
.get(InlineStyle::Reference.key())
|
||||
.and_then(parse_inline_reference)
|
||||
{
|
||||
Some(reference) => reference,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let payload = match inline_reference_payload(&reference) {
|
||||
Some(payload) => payload,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let key = (reference.page_id.clone(), payload.clone());
|
||||
if seen.insert(key.clone()) {
|
||||
refs.push(InlineReferencePayload {
|
||||
doc_id: key.0,
|
||||
payload: key.1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
refs
|
||||
}
|
||||
|
||||
fn parse_inline_reference(value: &Any) -> Option<InlineReference> {
|
||||
let map = match value {
|
||||
Any::Object(map) => map,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let page_id = map.get("pageId").and_then(any_as_string).map(str::to_string)?;
|
||||
let title = map.get("title").and_then(any_as_string).map(str::to_string);
|
||||
let ref_type = map.get("type").and_then(any_as_string).map(str::to_string);
|
||||
let params = map.get("params").and_then(|value| match value {
|
||||
Any::Object(map) => Some(map.clone()),
|
||||
_ => None,
|
||||
});
|
||||
let mode = params
|
||||
.as_ref()
|
||||
.and_then(|params| params.get("mode"))
|
||||
.and_then(any_as_string)
|
||||
.map(str::to_string);
|
||||
|
||||
Some(InlineReference {
|
||||
ref_type,
|
||||
page_id,
|
||||
title,
|
||||
params,
|
||||
mode,
|
||||
})
|
||||
}
|
||||
|
||||
fn inline_reference_payload(reference: &InlineReference) -> Option<String> {
|
||||
let params = reference.params.as_ref().map(params_any_map_to_json);
|
||||
Some(build_reference_payload(&reference.page_id, params))
|
||||
}
|
||||
|
||||
fn delta_to_inline_markdown(text: &Text, options: &DeltaToMdOptions) -> String {
|
||||
delta_to_markdown_with_options(&text.to_delta(), options, false)
|
||||
}
|
||||
|
||||
fn delta_to_markdown_with_options(delta: &[TextDeltaOp], options: &DeltaToMdOptions, trailing_newline: bool) -> String {
|
||||
let ops = build_delta_ops(delta);
|
||||
delta_ops_to_markdown_with_options(&ops, options, trailing_newline)
|
||||
}
|
||||
|
||||
fn delta_ops_to_markdown_with_options(ops: &[DeltaOp], options: &DeltaToMdOptions, trailing_newline: bool) -> String {
|
||||
let root = convert_delta_ops(ops, options);
|
||||
let mut rendered = render_node(&root);
|
||||
rendered = rendered.trim_end().to_string();
|
||||
if trailing_newline {
|
||||
rendered.push('\n');
|
||||
}
|
||||
rendered
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct DeltaOp {
|
||||
insert: DeltaInsert,
|
||||
attributes: TextAttributes,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum DeltaInsert {
|
||||
Text(String),
|
||||
Embed(Vec<Any>),
|
||||
}
|
||||
|
||||
fn delta_ops_from_any(value: &Any) -> Option<Vec<DeltaOp>> {
|
||||
let map = match value {
|
||||
Any::Object(map) => map,
|
||||
_ => return None,
|
||||
};
|
||||
match map.get("$blocksuite:internal:text$") {
|
||||
Some(Any::True) => {}
|
||||
_ => return None,
|
||||
}
|
||||
|
||||
let delta = map.get("delta")?;
|
||||
let entries = match delta {
|
||||
Any::Array(entries) => entries,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let mut ops = Vec::new();
|
||||
for entry in entries {
|
||||
if let Some(op) = delta_op_from_any(entry) {
|
||||
ops.push(op);
|
||||
}
|
||||
}
|
||||
|
||||
Some(ops)
|
||||
}
|
||||
|
||||
fn delta_op_from_any(value: &Any) -> Option<DeltaOp> {
|
||||
let map = match value {
|
||||
Any::Object(map) => map,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let insert_value = map.get("insert")?;
|
||||
let insert = match insert_value {
|
||||
Any::String(text) => DeltaInsert::Text(text.clone()),
|
||||
Any::Array(values) => DeltaInsert::Embed(values.clone()),
|
||||
_ => DeltaInsert::Embed(vec![insert_value.clone()]),
|
||||
};
|
||||
|
||||
let attributes = map.get("attributes").and_then(any_to_attributes).unwrap_or_default();
|
||||
|
||||
Some(DeltaOp { insert, attributes })
|
||||
}
|
||||
|
||||
fn any_to_attributes(value: &Any) -> Option<TextAttributes> {
|
||||
let map = match value {
|
||||
Any::Object(map) => map,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let mut attrs = TextAttributes::new();
|
||||
for (key, value) in map.iter() {
|
||||
attrs.insert(key.clone(), value.clone());
|
||||
}
|
||||
Some(attrs)
|
||||
}
|
||||
|
||||
fn delta_any_to_inline_markdown(value: &Any, options: &DeltaToMdOptions) -> Option<String> {
|
||||
delta_ops_from_any(value).map(|ops| delta_ops_to_markdown_with_options(&ops, options, false))
|
||||
}
|
||||
|
||||
pub(crate) fn delta_value_to_inline_markdown(value: &Value, options: &DeltaToMdOptions) -> Option<String> {
|
||||
if let Some(text) = value.to_text() {
|
||||
return Some(delta_to_inline_markdown(&text, options));
|
||||
}
|
||||
|
||||
let any = value_to_any(value)?;
|
||||
delta_any_to_inline_markdown(&any, options)
|
||||
}
|
||||
|
||||
fn build_delta_ops(delta: &[TextDeltaOp]) -> Vec<DeltaOp> {
|
||||
let mut ops = Vec::new();
|
||||
|
||||
for op in delta {
|
||||
let (insert, attrs) = match op {
|
||||
TextDeltaOp::Insert { insert, format } => (insert, format.clone().unwrap_or_default()),
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
match insert {
|
||||
TextInsert::Text(text) => ops.push(DeltaOp {
|
||||
insert: DeltaInsert::Text(text.clone()),
|
||||
attributes: attrs,
|
||||
}),
|
||||
TextInsert::Embed(values) => ops.push(DeltaOp {
|
||||
insert: DeltaInsert::Embed(values.clone()),
|
||||
attributes: attrs,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
ops
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Group {
|
||||
node: Rc<RefCell<Node>>,
|
||||
kind: String,
|
||||
distance: usize,
|
||||
count: usize,
|
||||
}
|
||||
|
||||
fn convert_delta_ops(ops: &[DeltaOp], options: &DeltaToMdOptions) -> Rc<RefCell<Node>> {
|
||||
let root = Node::new_root();
|
||||
let mut group: Option<Group> = None;
|
||||
let mut active_inline: HashMap<String, Any> = HashMap::new();
|
||||
let mut beginning_of_line = false;
|
||||
|
||||
let mut line = Node::new_line();
|
||||
let mut el = line.clone();
|
||||
Node::append(&root, line.clone());
|
||||
|
||||
for index in 0..ops.len() {
|
||||
let op = &ops[index];
|
||||
let next_attrs = ops.get(index + 1).map(|next| &next.attributes);
|
||||
|
||||
match &op.insert {
|
||||
DeltaInsert::Embed(values) => {
|
||||
apply_inline_attributes(&mut el, &op.attributes, None, &mut active_inline, options);
|
||||
for value in values {
|
||||
match value {
|
||||
Any::Object(map) => {
|
||||
for (key, value) in map.iter() {
|
||||
match key.as_str() {
|
||||
"image" => {
|
||||
if let Some(src) = any_as_string(value) {
|
||||
let url = encode_link(src);
|
||||
Node::append(&el, Node::new_text(&format!("")));
|
||||
}
|
||||
}
|
||||
"thematic_break" => {
|
||||
let current_open = el.borrow().open.clone();
|
||||
el.borrow_mut().open = format!("\n---\n{current_open}");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Any::String(value) => {
|
||||
Node::append(&el, Node::new_text(value));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
DeltaInsert::Text(text) => {
|
||||
let lines: Vec<&str> = text.split('\n').collect();
|
||||
if has_block_level_attribute(&op.attributes) {
|
||||
for _ in 1..lines.len() {
|
||||
for (attr, value) in op.attributes.iter() {
|
||||
match attr.as_str() {
|
||||
"header" => {
|
||||
if let Some(level) = any_as_u64(value) {
|
||||
let prefix = "#".repeat(level as usize);
|
||||
let current_open = line.borrow().open.clone();
|
||||
line.borrow_mut().open = format!("{prefix} {current_open}");
|
||||
new_line(&root, &mut line, &mut el, &mut active_inline);
|
||||
break;
|
||||
}
|
||||
}
|
||||
"blockquote" => {
|
||||
let current_open = line.borrow().open.clone();
|
||||
line.borrow_mut().open = format!("> {current_open}");
|
||||
new_line(&root, &mut line, &mut el, &mut active_inline);
|
||||
break;
|
||||
}
|
||||
"list" => {
|
||||
if group.as_ref().is_some_and(|g| g.kind != attr.as_str()) {
|
||||
group = None;
|
||||
}
|
||||
if group.is_none() {
|
||||
let group_node = Node::new_line();
|
||||
Node::append(&root, group_node.clone());
|
||||
group = Some(Group {
|
||||
node: group_node,
|
||||
kind: attr.to_string(),
|
||||
distance: 0,
|
||||
count: 0,
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(group) = group.as_mut() {
|
||||
Node::append(&group.node, line.clone());
|
||||
group.distance = 0;
|
||||
match any_as_string(value) {
|
||||
Some("bullet") => {
|
||||
let current_open = line.borrow().open.clone();
|
||||
line.borrow_mut().open = format!("- {current_open}");
|
||||
}
|
||||
Some("checked") => {
|
||||
let current_open = line.borrow().open.clone();
|
||||
line.borrow_mut().open = format!("- [x] {current_open}");
|
||||
}
|
||||
Some("unchecked") => {
|
||||
let current_open = line.borrow().open.clone();
|
||||
line.borrow_mut().open = format!("- [ ] {current_open}");
|
||||
}
|
||||
Some("ordered") => {
|
||||
group.count += 1;
|
||||
let current_open = line.borrow().open.clone();
|
||||
line.borrow_mut().open = format!("{}. {}", group.count, current_open);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
new_line(&root, &mut line, &mut el, &mut active_inline);
|
||||
break;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
beginning_of_line = true;
|
||||
} else {
|
||||
for (line_index, segment) in lines.iter().enumerate() {
|
||||
if (line_index > 0 || beginning_of_line) && group.is_some() {
|
||||
let reset_group = group.as_mut().map(|group| {
|
||||
group.distance += 1;
|
||||
group.distance >= 2
|
||||
});
|
||||
if reset_group == Some(true) {
|
||||
group = None;
|
||||
}
|
||||
}
|
||||
|
||||
apply_inline_attributes(&mut el, &op.attributes, next_attrs, &mut active_inline, options);
|
||||
Node::append(&el, Node::new_text(segment));
|
||||
if line_index + 1 < lines.len() {
|
||||
new_line(&root, &mut line, &mut el, &mut active_inline);
|
||||
}
|
||||
}
|
||||
beginning_of_line = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
root
|
||||
}
|
||||
|
||||
fn apply_inline_attributes(
|
||||
el: &mut Rc<RefCell<Node>>,
|
||||
attrs: &TextAttributes,
|
||||
next: Option<&TextAttributes>,
|
||||
active_inline: &mut HashMap<String, Any>,
|
||||
options: &DeltaToMdOptions,
|
||||
) {
|
||||
let mut first = Vec::new();
|
||||
let mut then = Vec::new();
|
||||
let mut seen: HashSet<String> = HashSet::new();
|
||||
|
||||
let mut tag = el.clone();
|
||||
loop {
|
||||
let format = match tag.borrow().format.clone() {
|
||||
Some(format) => format,
|
||||
None => break,
|
||||
};
|
||||
seen.insert(format.clone());
|
||||
|
||||
let should_close = match attrs.get(&format) {
|
||||
Some(value) => !any_truthy(value) || tag.borrow().open != tag.borrow().close,
|
||||
None => true,
|
||||
};
|
||||
|
||||
if should_close {
|
||||
for key in seen.iter() {
|
||||
active_inline.remove(key);
|
||||
}
|
||||
let parent = {
|
||||
let tag_ref = tag.borrow();
|
||||
tag_ref.parent.as_ref().and_then(|p| p.upgrade())
|
||||
};
|
||||
if let Some(parent) = parent {
|
||||
*el = parent.clone();
|
||||
tag = parent;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
let parent = {
|
||||
let tag_ref = tag.borrow();
|
||||
tag_ref.parent.as_ref().and_then(|p| p.upgrade())
|
||||
};
|
||||
if let Some(parent) = parent {
|
||||
tag = parent;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
for (attr, value) in attrs.iter() {
|
||||
if !is_inline_attribute(attr) || !any_truthy(value) {
|
||||
continue;
|
||||
}
|
||||
if let Some(active) = active_inline.get(attr)
|
||||
&& active == value
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let next_matches = next
|
||||
.and_then(|next_attrs| next_attrs.get(attr))
|
||||
.map(|next_value| next_value == value)
|
||||
.unwrap_or(false);
|
||||
|
||||
if next_matches {
|
||||
first.push(attr.clone());
|
||||
} else {
|
||||
then.push(attr.clone());
|
||||
}
|
||||
active_inline.insert(attr.clone(), value.clone());
|
||||
}
|
||||
|
||||
for attr in first.into_iter().chain(then) {
|
||||
if let Some(node) = inline_node_for_attr(&attr, attrs, options) {
|
||||
node.borrow_mut().format = Some(attr.clone());
|
||||
Node::append(el, node.clone());
|
||||
*el = node;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn inline_node_for_attr(attr: &str, attrs: &TextAttributes, options: &DeltaToMdOptions) -> Option<Rc<RefCell<Node>>> {
|
||||
let style = InlineStyle::from_key(attr)?;
|
||||
if let Some(delimiter) = style.delimiter() {
|
||||
return Some(Node::new_inline(delimiter.open, delimiter.close));
|
||||
}
|
||||
|
||||
match style {
|
||||
InlineStyle::Underline => Some(Node::new_inline("<u>", "</u>")),
|
||||
InlineStyle::Color => {
|
||||
let color = attrs.get(attr).and_then(any_as_string)?.trim();
|
||||
if color.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(Node::new_inline(&format!("<span style=\"color: {color}\">"), "</span>"))
|
||||
}
|
||||
}
|
||||
InlineStyle::Link => attrs
|
||||
.get(attr)
|
||||
.and_then(any_as_string)
|
||||
.map(|url| Node::new_inline("[", &format!("]({url})"))),
|
||||
InlineStyle::Reference => attrs.get(attr).and_then(parse_inline_reference).map(|reference| {
|
||||
let (title, link) = options.build_reference_link(&reference);
|
||||
Node::new_inline("[", &format!("{title}]({link})"))
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn has_block_level_attribute(attrs: &TextAttributes) -> bool {
|
||||
attrs.contains_key("header") || attrs.contains_key("blockquote") || attrs.contains_key("list")
|
||||
}
|
||||
|
||||
fn is_inline_attribute(attr: &str) -> bool {
|
||||
InlineStyle::from_key(attr).is_some()
|
||||
}
|
||||
|
||||
fn encode_link(link: &str) -> String {
|
||||
const HEX: &[u8; 16] = b"0123456789abcdef";
|
||||
|
||||
#[inline]
|
||||
fn push_pct(out: &mut String, b: u8) {
|
||||
out.push('%');
|
||||
out.push(HEX[(b >> 4) as usize] as char);
|
||||
out.push(HEX[(b & 0x0f) as usize] as char);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_allowed(b: u8) -> bool {
|
||||
matches!(
|
||||
b,
|
||||
b'A'..=b'Z'
|
||||
| b'a'..=b'z'
|
||||
| b'0'..=b'9'
|
||||
| b'-'
|
||||
| b'_'
|
||||
| b'.'
|
||||
| b'!'
|
||||
| b'~'
|
||||
| b'*'
|
||||
| b'\''
|
||||
| b';'
|
||||
| b','
|
||||
| b'/'
|
||||
| b'?'
|
||||
| b':'
|
||||
| b'@'
|
||||
| b'&'
|
||||
| b'='
|
||||
| b'+'
|
||||
| b'$'
|
||||
| b'#'
|
||||
)
|
||||
}
|
||||
|
||||
let mut out = String::with_capacity(link.len());
|
||||
|
||||
for &b in link.as_bytes() {
|
||||
match b {
|
||||
b'(' | b')' => push_pct(&mut out, b),
|
||||
b if is_allowed(b) => out.push(b as char),
|
||||
b => push_pct(&mut out, b),
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(i) = out.find("?response-content-disposition=attachment") {
|
||||
out.truncate(i);
|
||||
} else if let Some(i) = out.find("&response-content-disposition=attachment") {
|
||||
out.truncate(i);
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Node {
|
||||
open: String,
|
||||
close: String,
|
||||
text: String,
|
||||
children: Vec<Rc<RefCell<Node>>>,
|
||||
parent: Option<Weak<RefCell<Node>>>,
|
||||
format: Option<String>,
|
||||
}
|
||||
|
||||
impl Node {
|
||||
fn new_root() -> Rc<RefCell<Node>> {
|
||||
Rc::new(RefCell::new(Node {
|
||||
open: String::new(),
|
||||
close: String::new(),
|
||||
text: String::new(),
|
||||
children: Vec::new(),
|
||||
parent: None,
|
||||
format: None,
|
||||
}))
|
||||
}
|
||||
|
||||
fn new_inline(open: &str, close: &str) -> Rc<RefCell<Node>> {
|
||||
Rc::new(RefCell::new(Node {
|
||||
open: open.to_string(),
|
||||
close: close.to_string(),
|
||||
text: String::new(),
|
||||
children: Vec::new(),
|
||||
parent: None,
|
||||
format: None,
|
||||
}))
|
||||
}
|
||||
|
||||
fn new_text(text: &str) -> Rc<RefCell<Node>> {
|
||||
Rc::new(RefCell::new(Node {
|
||||
open: String::new(),
|
||||
close: String::new(),
|
||||
text: text.to_string(),
|
||||
children: Vec::new(),
|
||||
parent: None,
|
||||
format: None,
|
||||
}))
|
||||
}
|
||||
|
||||
fn new_line() -> Rc<RefCell<Node>> {
|
||||
Rc::new(RefCell::new(Node {
|
||||
open: String::new(),
|
||||
close: "\n".to_string(),
|
||||
text: String::new(),
|
||||
children: Vec::new(),
|
||||
parent: None,
|
||||
format: None,
|
||||
}))
|
||||
}
|
||||
|
||||
fn append(parent: &Rc<RefCell<Node>>, child: Rc<RefCell<Node>>) {
|
||||
if let Some(old_parent) = child.borrow().parent.as_ref().and_then(|p| p.upgrade()) {
|
||||
let mut old_parent = old_parent.borrow_mut();
|
||||
old_parent.children.retain(|existing| !Rc::ptr_eq(existing, &child));
|
||||
}
|
||||
|
||||
child.borrow_mut().parent = Some(Rc::downgrade(parent));
|
||||
parent.borrow_mut().children.push(child);
|
||||
}
|
||||
}
|
||||
|
||||
fn render_node(node: &Rc<RefCell<Node>>) -> String {
|
||||
let node_ref = node.borrow();
|
||||
let mut inner = node_ref.text.clone();
|
||||
for child in node_ref.children.iter() {
|
||||
inner.push_str(&render_node(child));
|
||||
}
|
||||
|
||||
if inner.trim().is_empty()
|
||||
&& node_ref.open == node_ref.close
|
||||
&& !node_ref.open.is_empty()
|
||||
&& !node_ref.close.is_empty()
|
||||
{
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let wrapped = !node_ref.open.is_empty() && !node_ref.close.is_empty();
|
||||
let empty_inner = inner.trim().is_empty();
|
||||
let mut fragments = Vec::new();
|
||||
|
||||
if inner.starts_with(' ') && !empty_inner && wrapped {
|
||||
fragments.push(" ".to_string());
|
||||
}
|
||||
if !node_ref.open.is_empty() {
|
||||
fragments.push(node_ref.open.clone());
|
||||
}
|
||||
fragments.push(if wrapped {
|
||||
inner.trim().to_string()
|
||||
} else {
|
||||
inner.clone()
|
||||
});
|
||||
if !node_ref.close.is_empty() {
|
||||
fragments.push(node_ref.close.clone());
|
||||
}
|
||||
if inner.ends_with(' ') && !empty_inner && wrapped {
|
||||
fragments.push(" ".to_string());
|
||||
}
|
||||
|
||||
fragments.join("")
|
||||
}
|
||||
|
||||
fn new_line(
|
||||
root: &Rc<RefCell<Node>>,
|
||||
line: &mut Rc<RefCell<Node>>,
|
||||
el: &mut Rc<RefCell<Node>>,
|
||||
active_inline: &mut HashMap<String, Any>,
|
||||
) {
|
||||
*line = Node::new_line();
|
||||
*el = line.clone();
|
||||
Node::append(root, line.clone());
|
||||
active_inline.clear();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::Value;
|
||||
use y_octo::{Any, TextAttributes, TextDeltaOp, TextInsert};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_delta_to_inline_markdown_link() {
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert(
|
||||
InlineStyle::Link.key().into(),
|
||||
Any::String("https://example.com".into()),
|
||||
);
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("AFFiNE".into()),
|
||||
format: Some(attrs),
|
||||
}];
|
||||
|
||||
let options = DeltaToMdOptions::new(None);
|
||||
let rendered = delta_to_markdown_with_options(&delta, &options, false);
|
||||
assert_eq!(rendered, "[AFFiNE](https://example.com)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_inline_references_payload() {
|
||||
let mut ref_map = AHashMap::default();
|
||||
ref_map.insert("pageId".into(), Any::String("doc123".into()));
|
||||
ref_map.insert("title".into(), Any::String("Doc Title".into()));
|
||||
ref_map.insert("type".into(), Any::String("LinkedPage".into()));
|
||||
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert(InlineStyle::Reference.key().into(), Any::Object(ref_map));
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Doc Title".into()),
|
||||
format: Some(attrs),
|
||||
}];
|
||||
|
||||
let refs = extract_inline_references(&delta);
|
||||
assert_eq!(refs.len(), 1);
|
||||
assert_eq!(refs[0].doc_id, "doc123");
|
||||
|
||||
let payload: Value = serde_json::from_str(&refs[0].payload).unwrap();
|
||||
assert_eq!(payload, serde_json::json!({ "docId": "doc123" }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delta_to_inline_markdown_underline() {
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert(InlineStyle::Underline.key().into(), Any::True);
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Under".into()),
|
||||
format: Some(attrs),
|
||||
}];
|
||||
|
||||
let options = DeltaToMdOptions::new(None);
|
||||
let rendered = delta_to_markdown_with_options(&delta, &options, false);
|
||||
assert_eq!(rendered, "<u>Under</u>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delta_to_inline_markdown_color() {
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert(InlineStyle::Color.key().into(), Any::String("red".into()));
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Red".into()),
|
||||
format: Some(attrs),
|
||||
}];
|
||||
|
||||
let options = DeltaToMdOptions::new(None);
|
||||
let rendered = delta_to_markdown_with_options(&delta, &options, false);
|
||||
assert_eq!(rendered, "<span style=\"color: red\">Red</span>");
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
const INLINE_ATTR_BOLD: &str = "bold";
|
||||
const INLINE_ATTR_ITALIC: &str = "italic";
|
||||
const INLINE_ATTR_UNDERLINE: &str = "underline";
|
||||
const INLINE_ATTR_STRIKE: &str = "strike";
|
||||
const INLINE_ATTR_CODE: &str = "code";
|
||||
const INLINE_ATTR_LINK: &str = "link";
|
||||
const INLINE_ATTR_REFERENCE: &str = "reference";
|
||||
const INLINE_ATTR_COLOR: &str = "color";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum InlineStyle {
|
||||
Bold,
|
||||
Italic,
|
||||
Underline,
|
||||
Strike,
|
||||
Code,
|
||||
Link,
|
||||
Reference,
|
||||
Color,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) struct InlineDelimiter {
|
||||
pub(super) open: &'static str,
|
||||
pub(super) close: &'static str,
|
||||
}
|
||||
|
||||
impl InlineStyle {
|
||||
pub(super) fn key(self) -> &'static str {
|
||||
match self {
|
||||
InlineStyle::Bold => INLINE_ATTR_BOLD,
|
||||
InlineStyle::Italic => INLINE_ATTR_ITALIC,
|
||||
InlineStyle::Underline => INLINE_ATTR_UNDERLINE,
|
||||
InlineStyle::Strike => INLINE_ATTR_STRIKE,
|
||||
InlineStyle::Code => INLINE_ATTR_CODE,
|
||||
InlineStyle::Link => INLINE_ATTR_LINK,
|
||||
InlineStyle::Reference => INLINE_ATTR_REFERENCE,
|
||||
InlineStyle::Color => INLINE_ATTR_COLOR,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn from_key(key: &str) -> Option<Self> {
|
||||
match key {
|
||||
INLINE_ATTR_BOLD => Some(InlineStyle::Bold),
|
||||
INLINE_ATTR_ITALIC => Some(InlineStyle::Italic),
|
||||
INLINE_ATTR_UNDERLINE => Some(InlineStyle::Underline),
|
||||
INLINE_ATTR_STRIKE => Some(InlineStyle::Strike),
|
||||
INLINE_ATTR_CODE => Some(InlineStyle::Code),
|
||||
INLINE_ATTR_LINK => Some(InlineStyle::Link),
|
||||
INLINE_ATTR_REFERENCE => Some(InlineStyle::Reference),
|
||||
INLINE_ATTR_COLOR => Some(InlineStyle::Color),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn delimiter(self) -> Option<InlineDelimiter> {
|
||||
match self {
|
||||
InlineStyle::Italic => Some(InlineDelimiter { open: "_", close: "_" }),
|
||||
InlineStyle::Bold => Some(InlineDelimiter {
|
||||
open: "**",
|
||||
close: "**",
|
||||
}),
|
||||
InlineStyle::Strike => Some(InlineDelimiter {
|
||||
open: "~~",
|
||||
close: "~~",
|
||||
}),
|
||||
InlineStyle::Code => Some(InlineDelimiter { open: "`", close: "`" }),
|
||||
InlineStyle::Link | InlineStyle::Reference | InlineStyle::Underline | InlineStyle::Color => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
mod delta;
|
||||
mod inline;
|
||||
mod parser;
|
||||
mod render;
|
||||
|
||||
pub(crate) use delta::{
|
||||
DeltaToMdOptions, InlineReferencePayload, delta_value_to_inline_markdown, extract_inline_references,
|
||||
extract_inline_references_from_value, text_to_inline_markdown,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use parser::MAX_MARKDOWN_CHARS;
|
||||
pub(crate) use parser::{MAX_BLOCKS, parse_markdown_blocks};
|
||||
pub(crate) use render::{MarkdownRenderer, MarkdownWriter};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,235 +0,0 @@
|
||||
use super::{
|
||||
super::block_spec::{BlockFlavour, BlockSpec},
|
||||
delta::{DeltaToMdOptions, delta_ops_to_markdown, delta_ops_to_plain_text},
|
||||
};
|
||||
|
||||
pub(crate) struct MarkdownWriter<'a> {
|
||||
output: &'a mut String,
|
||||
}
|
||||
|
||||
impl<'a> MarkdownWriter<'a> {
|
||||
pub(crate) fn new(output: &'a mut String) -> Self {
|
||||
Self { output }
|
||||
}
|
||||
|
||||
pub(crate) fn push_paragraph(&mut self, prefix: &str, text: &str) {
|
||||
if prefix == "> " {
|
||||
let quoted = text
|
||||
.split('\n')
|
||||
.map(|line| format!("{prefix}{line}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
self.output.push_str("ed);
|
||||
if !text.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
self.output.push('\n');
|
||||
return;
|
||||
}
|
||||
|
||||
self.output.push_str(prefix);
|
||||
self.output.push_str(text);
|
||||
if !text.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
self.output.push('\n');
|
||||
}
|
||||
|
||||
pub(crate) fn push_list_item(&mut self, indent: &str, prefix: &str, text: &str) {
|
||||
self.output.push_str(indent);
|
||||
self.output.push_str(prefix);
|
||||
self.output.push_str(text);
|
||||
if !text.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn push_code_block(&mut self, lang: &str, text: &str) {
|
||||
self.output.push_str("```");
|
||||
self.output.push_str(lang);
|
||||
self.output.push('\n');
|
||||
self.output.push_str(text);
|
||||
self.output.push_str("\n```\n\n");
|
||||
}
|
||||
|
||||
pub(crate) fn push_divider(&mut self) {
|
||||
self.output.push_str("\n---\n\n");
|
||||
}
|
||||
|
||||
pub(crate) fn push_table(&mut self, table: &str) {
|
||||
if table.is_empty() {
|
||||
self.output.push('\n');
|
||||
return;
|
||||
}
|
||||
self.output.push_str(table);
|
||||
if !table.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
self.output.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn paragraph_prefix(type_: &str) -> &'static str {
|
||||
match type_ {
|
||||
"h1" => "# ",
|
||||
"h2" => "## ",
|
||||
"h3" => "### ",
|
||||
"h4" => "#### ",
|
||||
"h5" => "##### ",
|
||||
"h6" => "###### ",
|
||||
"quote" => "> ",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn list_prefix(r#type: &str, checked: bool, order: Option<i64>) -> String {
|
||||
match r#type {
|
||||
"bulleted" => "* ".to_string(),
|
||||
"todo" => if checked { "- [x] " } else { "- [ ] " }.to_string(),
|
||||
_ => format!("{}. ", order.unwrap_or(1)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn list_indent(depth: usize) -> String {
|
||||
" ".repeat(depth)
|
||||
}
|
||||
|
||||
pub(crate) struct MarkdownRenderer<'a> {
|
||||
options: &'a DeltaToMdOptions,
|
||||
}
|
||||
|
||||
impl<'a> MarkdownRenderer<'a> {
|
||||
pub(crate) fn new(options: &'a DeltaToMdOptions) -> Self {
|
||||
Self { options }
|
||||
}
|
||||
|
||||
pub(crate) fn write_block(&self, output: &mut String, spec: &BlockSpec, list_depth: usize) {
|
||||
match spec.flavour {
|
||||
BlockFlavour::Paragraph => {
|
||||
let prefix = paragraph_prefix(spec.block_type_str().unwrap_or_default());
|
||||
let text_md = delta_ops_to_markdown(&spec.text, self.options);
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_paragraph(prefix, &text_md);
|
||||
}
|
||||
BlockFlavour::List => {
|
||||
let type_ = spec.block_type_str().unwrap_or("bulleted");
|
||||
let checked = spec.checked.unwrap_or(false);
|
||||
let prefix = list_prefix(type_, checked, spec.order);
|
||||
let indent = list_indent(list_depth);
|
||||
let text_md = delta_ops_to_markdown(&spec.text, self.options);
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_list_item(&indent, &prefix, &text_md);
|
||||
}
|
||||
BlockFlavour::Code => {
|
||||
let text = delta_ops_to_plain_text(&spec.text);
|
||||
let lang = spec.language.as_deref().unwrap_or_default();
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_code_block(lang, &text);
|
||||
}
|
||||
BlockFlavour::Divider => {
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_divider();
|
||||
}
|
||||
BlockFlavour::Image => {
|
||||
if let Some(image) = spec.image.as_ref() {
|
||||
output.push_str(&image.render_markdown());
|
||||
}
|
||||
}
|
||||
BlockFlavour::Bookmark => {
|
||||
if let Some(bookmark) = spec.bookmark.as_ref() {
|
||||
output.push_str(&format!("\n[](Bookmark,{})\n\n", bookmark.url));
|
||||
}
|
||||
}
|
||||
BlockFlavour::EmbedYoutube => {
|
||||
if let Some(embed) = spec.embed_youtube.as_ref() {
|
||||
output.push_str(
|
||||
&format!(
|
||||
"\n <iframe\n type=\"text/html\"\n width=\"100%\"\n height=\"410px\"\n src=\"https://www.youtube.com/embed/{}\"\n frameborder=\"0\"\n allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\"\n allowfullscreen\n credentialless>\n </iframe>\n\n",
|
||||
embed.video_id
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
BlockFlavour::EmbedIframe => {
|
||||
if let Some(embed) = spec.embed_iframe.as_ref() {
|
||||
output.push_str(&format!("\n<iframe src=\"{}\"></iframe>\n\n", embed.url));
|
||||
}
|
||||
}
|
||||
BlockFlavour::Callout => {}
|
||||
BlockFlavour::Table => {
|
||||
if let Some(table) = spec.table.as_ref()
|
||||
&& let Some(table_md) = table.render_markdown()
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_table(&table_md);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_paragraph_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_paragraph("# ", "Title\n");
|
||||
}
|
||||
assert_eq!(markdown, "# Title\n\n");
|
||||
|
||||
markdown.clear();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_paragraph("", "Plain");
|
||||
}
|
||||
assert_eq!(markdown, "Plain\n\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_list_item(" ", "* ", "Item\n");
|
||||
}
|
||||
assert_eq!(markdown, " * Item\n");
|
||||
|
||||
markdown.clear();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_list_item("", "- [ ] ", "Task");
|
||||
}
|
||||
assert_eq!(markdown, "- [ ] Task\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_code_block_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_code_block("rs", "fn main() {}");
|
||||
}
|
||||
assert_eq!(markdown, "```rs\nfn main() {}\n```\n\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_table_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_table("|a|b|\n|---|---|\n|1|2|\n");
|
||||
}
|
||||
assert_eq!(markdown, "|a|b|\n|---|---|\n|1|2|\n\n");
|
||||
|
||||
markdown.clear();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_table("|a|b|");
|
||||
}
|
||||
assert_eq!(markdown, "|a|b|\n\n");
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
mod block_spec;
|
||||
mod blocksuite;
|
||||
mod doc_loader;
|
||||
mod error;
|
||||
mod markdown;
|
||||
mod read;
|
||||
#[cfg(test)]
|
||||
mod roundtrip_tests;
|
||||
mod schema;
|
||||
mod table;
|
||||
mod value;
|
||||
mod write;
|
||||
|
||||
pub use error::ParseError;
|
||||
pub use read::{
|
||||
BlockInfo, CrawlResult, MarkdownResult, PageDocContent, WorkspaceDocContent, get_doc_ids_from_binary,
|
||||
parse_doc_from_binary, parse_doc_to_markdown, parse_page_doc, parse_workspace_doc,
|
||||
};
|
||||
pub use write::{
|
||||
add_doc_to_root_doc, build_full_doc, build_public_root_doc, update_doc, update_doc_properties, update_doc_title,
|
||||
update_root_doc_meta_title,
|
||||
};
|
||||
@@ -1,286 +0,0 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use y_octo::{Any, Map, Value};
|
||||
|
||||
use super::{text_content, text_content_for_summary};
|
||||
use crate::doc_parser::{
|
||||
blocksuite::{DocContext, collect_child_ids, get_string},
|
||||
markdown::{
|
||||
DeltaToMdOptions, InlineReferencePayload, delta_value_to_inline_markdown, extract_inline_references_from_value,
|
||||
text_to_inline_markdown,
|
||||
},
|
||||
table::{MarkdownTableOptions, render_markdown_table},
|
||||
value::{any_as_string, value_to_string},
|
||||
};
|
||||
|
||||
pub(super) struct DatabaseTable {
|
||||
pub(super) columns: Vec<DatabaseColumn>,
|
||||
pub(super) rows: Vec<Vec<String>>,
|
||||
}
|
||||
|
||||
pub(super) struct DatabaseOption {
|
||||
id: Option<String>,
|
||||
value: Option<String>,
|
||||
color: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) struct DatabaseColumn {
|
||||
pub(super) id: String,
|
||||
pub(super) name: Option<String>,
|
||||
pub(super) col_type: String,
|
||||
pub(super) options: Vec<DatabaseOption>,
|
||||
}
|
||||
|
||||
pub(super) fn build_database_table(
|
||||
block: &Map,
|
||||
context: &DocContext,
|
||||
md_options: &DeltaToMdOptions,
|
||||
) -> Option<DatabaseTable> {
|
||||
let columns = parse_database_columns(block)?;
|
||||
let cells_map = block.get("prop:cells").and_then(|v| v.to_map())?;
|
||||
let child_ids = collect_child_ids(block);
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for child_id in child_ids {
|
||||
let row_cells = cells_map.get(&child_id).and_then(|v| v.to_map());
|
||||
let mut row = Vec::new();
|
||||
|
||||
for column in columns.iter() {
|
||||
let mut cell_text = String::new();
|
||||
if column.col_type == "title" {
|
||||
if let Some(child_block) = context.block_pool.get(&child_id) {
|
||||
if let Some(text_md) = text_to_inline_markdown(child_block, "prop:text", md_options) {
|
||||
cell_text = text_md;
|
||||
} else if let Some((text, _)) = text_content(child_block, "prop:text") {
|
||||
cell_text = text;
|
||||
} else if let Some((text, _)) = text_content_for_summary(child_block, "prop:text") {
|
||||
cell_text = text;
|
||||
}
|
||||
}
|
||||
} else if let Some(row_cells) = &row_cells
|
||||
&& let Some(cell_val) = row_cells.get(&column.id).and_then(|v| v.to_map())
|
||||
&& let Some(value) = cell_val.get("value")
|
||||
{
|
||||
if let Some(text_md) = delta_value_to_inline_markdown(&value, md_options) {
|
||||
cell_text = text_md;
|
||||
} else {
|
||||
cell_text = format_cell_value(&value, column);
|
||||
}
|
||||
}
|
||||
|
||||
row.push(cell_text);
|
||||
}
|
||||
if row.iter().any(|cell| !cell.is_empty()) {
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
Some(DatabaseTable { columns, rows })
|
||||
}
|
||||
|
||||
pub(super) fn database_table_markdown(table: DatabaseTable) -> Option<String> {
|
||||
let mut rows = Vec::with_capacity(table.rows.len() + 1);
|
||||
let header = table
|
||||
.columns
|
||||
.iter()
|
||||
.map(|column| column.name.as_deref().unwrap_or_default().to_string())
|
||||
.collect::<Vec<_>>();
|
||||
rows.push(header);
|
||||
rows.extend(table.rows);
|
||||
|
||||
let options = MarkdownTableOptions::new(false, "<br />", true);
|
||||
render_markdown_table(&rows, options)
|
||||
}
|
||||
|
||||
pub(super) fn database_summary_text(block: &Map, context: &DocContext) -> Option<String> {
|
||||
let md_options = DeltaToMdOptions::new(None);
|
||||
let table = build_database_table(block, context, &md_options)?;
|
||||
let mut summary = String::new();
|
||||
|
||||
if let Some(title) = get_string(block, "prop:title")
|
||||
&& !title.is_empty()
|
||||
{
|
||||
summary.push_str(&title);
|
||||
summary.push('|');
|
||||
}
|
||||
|
||||
for column in table.columns.iter() {
|
||||
if let Some(name) = column.name.as_ref()
|
||||
&& !name.is_empty()
|
||||
{
|
||||
summary.push_str(name);
|
||||
summary.push('|');
|
||||
}
|
||||
for option in column.options.iter() {
|
||||
if let Some(value) = option.value.as_ref()
|
||||
&& !value.is_empty()
|
||||
{
|
||||
summary.push_str(value);
|
||||
summary.push('|');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for row in table.rows.iter() {
|
||||
for cell_text in row.iter() {
|
||||
if !cell_text.is_empty() {
|
||||
summary.push_str(cell_text);
|
||||
summary.push('|');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if summary.is_empty() { None } else { Some(summary) }
|
||||
}
|
||||
|
||||
pub(super) fn gather_database_texts(block: &Map) -> (Vec<String>, Option<String>) {
|
||||
let mut texts = Vec::new();
|
||||
let database_title = get_string(block, "prop:title");
|
||||
if let Some(title) = &database_title {
|
||||
texts.push(title.clone());
|
||||
}
|
||||
|
||||
if let Some(columns) = parse_database_columns(block) {
|
||||
for column in columns.iter() {
|
||||
if let Some(name) = column.name.as_ref() {
|
||||
texts.push(name.clone());
|
||||
}
|
||||
for option in column.options.iter() {
|
||||
if let Some(value) = option.value.as_ref() {
|
||||
texts.push(value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(texts, database_title)
|
||||
}
|
||||
|
||||
pub(super) fn collect_database_cell_references(block: &Map) -> Vec<InlineReferencePayload> {
|
||||
let cells_map = match block.get("prop:cells").and_then(|value| value.to_map()) {
|
||||
Some(map) => map,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
|
||||
let mut refs = Vec::new();
|
||||
let mut seen: HashSet<(String, String)> = HashSet::new();
|
||||
|
||||
for row in cells_map.values() {
|
||||
let Some(row_map) = row.to_map() else {
|
||||
continue;
|
||||
};
|
||||
for cell in row_map.values() {
|
||||
let Some(cell_map) = cell.to_map() else {
|
||||
continue;
|
||||
};
|
||||
let Some(value) = cell_map.get("value") else {
|
||||
continue;
|
||||
};
|
||||
for reference in extract_inline_references_from_value(&value) {
|
||||
let key = (reference.doc_id.clone(), reference.payload.clone());
|
||||
if seen.insert(key) {
|
||||
refs.push(reference);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
refs
|
||||
}
|
||||
|
||||
fn parse_database_columns(block: &Map) -> Option<Vec<DatabaseColumn>> {
|
||||
let columns = block.get("prop:columns").and_then(|value| value.to_array())?;
|
||||
let mut parsed = Vec::new();
|
||||
for column_value in columns.iter() {
|
||||
if let Some(column) = column_value.to_map() {
|
||||
let id = column
|
||||
.get("id")
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
let name = column.get("name").and_then(|value| value_to_string(&value));
|
||||
let col_type = column
|
||||
.get("type")
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
let options = parse_database_options(&column);
|
||||
parsed.push(DatabaseColumn {
|
||||
id,
|
||||
name,
|
||||
col_type,
|
||||
options,
|
||||
});
|
||||
}
|
||||
}
|
||||
Some(parsed)
|
||||
}
|
||||
|
||||
fn parse_database_options(column: &Map) -> Vec<DatabaseOption> {
|
||||
let Some(data) = column.get("data").and_then(|value| value.to_map()) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Some(options) = data.get("options").and_then(|value| value.to_array()) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut parsed = Vec::new();
|
||||
for option_value in options.iter() {
|
||||
if let Some(option) = option_value.to_map() {
|
||||
parsed.push(DatabaseOption {
|
||||
id: option.get("id").and_then(|value| value_to_string(&value)),
|
||||
value: option.get("value").and_then(|value| value_to_string(&value)),
|
||||
color: option.get("color").and_then(|value| value_to_string(&value)),
|
||||
});
|
||||
}
|
||||
}
|
||||
parsed
|
||||
}
|
||||
|
||||
fn format_option_tag(option: &DatabaseOption) -> String {
|
||||
let id = option.id.as_deref().unwrap_or_default();
|
||||
let value = option.value.as_deref().unwrap_or_default();
|
||||
let color = option.color.as_deref().unwrap_or_default();
|
||||
|
||||
format!("<span data-affine-option data-value=\"{id}\" data-option-color=\"{color}\">{value}</span>")
|
||||
}
|
||||
|
||||
fn format_cell_value(value: &Value, column: &DatabaseColumn) -> String {
|
||||
match column.col_type.as_str() {
|
||||
"select" => {
|
||||
let id = match value {
|
||||
Value::Any(any) => any_as_string(any).map(str::to_string),
|
||||
Value::Text(text) => Some(text.to_string()),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(id) = id {
|
||||
for option in column.options.iter() {
|
||||
if option.id.as_deref() == Some(id.as_str()) {
|
||||
return format_option_tag(option);
|
||||
}
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
"multi-select" => {
|
||||
let ids: Vec<String> = match value {
|
||||
Value::Any(Any::Array(ids)) => ids.iter().filter_map(any_as_string).map(str::to_string).collect(),
|
||||
Value::Array(array) => array.iter().filter_map(|id_val| value_to_string(&id_val)).collect(),
|
||||
_ => Vec::new(),
|
||||
};
|
||||
|
||||
if ids.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let mut selected = Vec::new();
|
||||
for id in ids.iter() {
|
||||
for option in column.options.iter() {
|
||||
if option.id.as_deref() == Some(id.as_str()) {
|
||||
selected.push(format_option_tag(option));
|
||||
}
|
||||
}
|
||||
}
|
||||
selected.join("")
|
||||
}
|
||||
_ => value_to_string(value).unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
@@ -1,985 +0,0 @@
|
||||
mod database;
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map as JsonMap, Value as JsonValue};
|
||||
use y_octo::{Any, Map};
|
||||
|
||||
use self::database::{
|
||||
build_database_table, collect_database_cell_references, database_summary_text, database_table_markdown,
|
||||
gather_database_texts,
|
||||
};
|
||||
use super::{
|
||||
ParseError,
|
||||
block_spec::{BlockFlavour, BlockSpec, ImageSpec},
|
||||
blocksuite::{DocContext, get_block_id, get_flavour, get_list_depth, get_string, nearest_by_flavour},
|
||||
doc_loader::load_doc,
|
||||
markdown::{DeltaToMdOptions, MarkdownRenderer, MarkdownWriter, extract_inline_references},
|
||||
schema::{NOTE_FLAVOUR, PAGE_FLAVOUR},
|
||||
value::{any_as_string, any_truthy, build_reference_payload, params_value_to_json, value_to_string},
|
||||
};
|
||||
|
||||
const SUMMARY_LIMIT: usize = 1000;
|
||||
const DEFAULT_PAGE_TITLE: &str = "Untitled";
|
||||
const KNOWN_UNSUPPORTED_MARKDOWN_FLAVOURS: [&str; 10] = [
|
||||
"affine:attachment",
|
||||
"affine:callout",
|
||||
"affine:note",
|
||||
"affine:edgeless-text",
|
||||
"affine:embed-linked-doc",
|
||||
"affine:embed-synced-doc",
|
||||
"affine:frame",
|
||||
"affine:latex",
|
||||
"affine:surface",
|
||||
"affine:surface-ref",
|
||||
];
|
||||
|
||||
const BOOKMARK_FLAVOURS: [&str; 6] = [
|
||||
"affine:bookmark",
|
||||
"affine:embed-youtube",
|
||||
"affine:embed-iframe",
|
||||
"affine:embed-figma",
|
||||
"affine:embed-github",
|
||||
"affine:embed-loom",
|
||||
];
|
||||
|
||||
struct SummaryBuilder {
|
||||
summary: String,
|
||||
remaining: Option<isize>,
|
||||
}
|
||||
|
||||
impl SummaryBuilder {
|
||||
fn new(limit: isize) -> Self {
|
||||
let remaining = if limit < 0 { None } else { Some(limit) };
|
||||
Self {
|
||||
summary: String::new(),
|
||||
remaining,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_unlimited(&self) -> bool {
|
||||
self.remaining.is_none()
|
||||
}
|
||||
|
||||
fn push_text(&mut self, text: &str, len: usize) {
|
||||
match self.remaining {
|
||||
None => self.summary.push_str(text),
|
||||
Some(remaining) if remaining > 0 => {
|
||||
self.summary.push_str(text);
|
||||
self.remaining = Some(remaining - len as isize);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn push_raw(&mut self, text: &str) {
|
||||
self.summary.push_str(text);
|
||||
}
|
||||
|
||||
fn into_string(self) -> String {
|
||||
self.summary
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BlockInfo {
|
||||
pub block_id: String,
|
||||
pub flavour: String,
|
||||
pub content: Option<Vec<String>>,
|
||||
pub blob: Option<Vec<String>>,
|
||||
pub ref_doc_id: Option<Vec<String>>,
|
||||
pub ref_info: Option<Vec<String>>,
|
||||
pub parent_flavour: Option<String>,
|
||||
pub parent_block_id: Option<String>,
|
||||
pub additional: Option<String>,
|
||||
}
|
||||
|
||||
impl BlockInfo {
|
||||
fn base(
|
||||
block_id: &str,
|
||||
flavour: &str,
|
||||
parent_flavour: Option<&String>,
|
||||
parent_block_id: Option<&String>,
|
||||
additional: Option<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
block_id: block_id.to_string(),
|
||||
flavour: flavour.to_string(),
|
||||
content: None,
|
||||
blob: None,
|
||||
ref_doc_id: None,
|
||||
ref_info: None,
|
||||
parent_flavour: parent_flavour.cloned(),
|
||||
parent_block_id: parent_block_id.cloned(),
|
||||
additional,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CrawlResult {
|
||||
pub blocks: Vec<BlockInfo>,
|
||||
pub title: String,
|
||||
pub summary: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PageDocContent {
|
||||
pub title: String,
|
||||
pub summary: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WorkspaceDocContent {
|
||||
pub name: String,
|
||||
#[serde(rename = "avatarKey")]
|
||||
pub avatar_key: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MarkdownResult {
|
||||
pub title: String,
|
||||
pub markdown: String,
|
||||
pub known_unsupported_blocks: Vec<String>,
|
||||
pub unknown_blocks: Vec<String>,
|
||||
}
|
||||
|
||||
fn is_known_unsupported_markdown_flavour(flavour: &str) -> bool {
|
||||
KNOWN_UNSUPPORTED_MARKDOWN_FLAVOURS.contains(&flavour) || flavour.starts_with("affine:edgeless-")
|
||||
}
|
||||
|
||||
fn is_edgeless_markdown_flavour(flavour: &str) -> bool {
|
||||
matches!(flavour, "affine:surface" | "affine:frame" | "affine:surface-ref") || flavour.starts_with("affine:edgeless-")
|
||||
}
|
||||
|
||||
fn has_skipped_markdown_ancestor(
|
||||
block_id: &str,
|
||||
parent_lookup: &HashMap<String, String>,
|
||||
skipped_subtrees: &HashSet<String>,
|
||||
) -> bool {
|
||||
let mut cursor = parent_lookup.get(block_id).cloned();
|
||||
while let Some(parent_id) = cursor {
|
||||
if skipped_subtrees.contains(&parent_id) {
|
||||
return true;
|
||||
}
|
||||
cursor = parent_lookup.get(&parent_id).cloned();
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub fn parse_workspace_doc(doc_bin: Vec<u8>) -> Result<Option<WorkspaceDocContent>, ParseError> {
|
||||
let doc = load_doc(&doc_bin, None)?;
|
||||
|
||||
let meta = match doc.get_map("meta") {
|
||||
Ok(meta) => meta,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
|
||||
let name = get_string(&meta, "name").unwrap_or_default();
|
||||
let avatar_key = get_string(&meta, "avatar").unwrap_or_default();
|
||||
|
||||
Ok(Some(WorkspaceDocContent { name, avatar_key }))
|
||||
}
|
||||
|
||||
pub fn parse_page_doc(
|
||||
doc_bin: Vec<u8>,
|
||||
max_summary_length: Option<isize>,
|
||||
) -> Result<Option<PageDocContent>, ParseError> {
|
||||
let doc = load_doc(&doc_bin, None)?;
|
||||
|
||||
let blocks_map = match doc.get_map("blocks") {
|
||||
Ok(map) => map,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
|
||||
if blocks_map.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let Some(context) = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let mut walker = context.walker();
|
||||
let mut content = PageDocContent {
|
||||
title: context
|
||||
.block_pool
|
||||
.get(&context.root_block_id)
|
||||
.and_then(|block| get_string(block, "prop:title"))
|
||||
.unwrap_or_default(),
|
||||
summary: String::new(),
|
||||
};
|
||||
let mut summary = SummaryBuilder::new(max_summary_length.unwrap_or(150));
|
||||
|
||||
while let Some((_parent_block_id, block_id)) = walker.next() {
|
||||
let Some(block) = context.block_pool.get(&block_id) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let Some(flavour) = get_flavour(block) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match flavour.as_str() {
|
||||
"affine:page" | "affine:note" => {
|
||||
walker.enqueue_children(&block_id, block);
|
||||
}
|
||||
"affine:attachment" | "affine:transcription" | "affine:callout" => {
|
||||
if summary.is_unlimited() {
|
||||
walker.enqueue_children(&block_id, block);
|
||||
}
|
||||
}
|
||||
"affine:database" => {
|
||||
if summary.is_unlimited()
|
||||
&& let Some(text) = database_summary_text(block, &context)
|
||||
{
|
||||
summary.push_raw(&text);
|
||||
}
|
||||
}
|
||||
"affine:table" => {
|
||||
if summary.is_unlimited() {
|
||||
let contents = table_cell_texts(block);
|
||||
if !contents.is_empty() {
|
||||
summary.push_raw(&contents.join("|"));
|
||||
}
|
||||
}
|
||||
}
|
||||
"affine:paragraph" | "affine:list" | "affine:code" => {
|
||||
walker.enqueue_children(&block_id, block);
|
||||
if let Some((text, len)) = text_content_for_summary(block, "prop:text") {
|
||||
summary.push_text(&text, len);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
content.summary = summary.into_string();
|
||||
Ok(Some(content))
|
||||
}
|
||||
|
||||
pub fn parse_doc_to_markdown(
|
||||
doc_bin: Vec<u8>,
|
||||
doc_id: String,
|
||||
ai_editable: bool,
|
||||
doc_url_prefix: Option<String>,
|
||||
) -> Result<MarkdownResult, ParseError> {
|
||||
let doc = load_doc(&doc_bin, Some(doc_id.as_str()))?;
|
||||
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Ok(MarkdownResult {
|
||||
title: "".into(),
|
||||
markdown: "".into(),
|
||||
known_unsupported_blocks: vec![],
|
||||
unknown_blocks: vec![],
|
||||
});
|
||||
}
|
||||
|
||||
let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("root block not found".into()))?;
|
||||
let root_block_id = context.root_block_id.clone();
|
||||
let mut walker = context.walker();
|
||||
let mut doc_title = String::from(DEFAULT_PAGE_TITLE);
|
||||
let mut markdown = String::new();
|
||||
let mut known_unsupported_blocks = Vec::new();
|
||||
let mut unknown_blocks = Vec::new();
|
||||
let mut skipped_subtrees = HashSet::new();
|
||||
let md_options = DeltaToMdOptions::new(doc_url_prefix);
|
||||
let renderer = MarkdownRenderer::new(&md_options);
|
||||
|
||||
while let Some((_parent_block_id, block_id)) = walker.next() {
|
||||
let block = match context.block_pool.get(&block_id) {
|
||||
Some(block) => block,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flavour = match get_flavour(block) {
|
||||
Some(flavour) => flavour,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
if flavour == PAGE_FLAVOUR {
|
||||
// enqueue children first to keep traversal order similar to JS implementation
|
||||
walker.enqueue_children(&block_id, block);
|
||||
let title = get_string(block, "prop:title").unwrap_or_default();
|
||||
doc_title = title.clone();
|
||||
continue;
|
||||
}
|
||||
|
||||
let parent_id = context.parent_lookup.get(&block_id);
|
||||
let parent_flavour = parent_id
|
||||
.and_then(|id| context.block_pool.get(id))
|
||||
.and_then(get_flavour);
|
||||
|
||||
if parent_flavour.as_deref() == Some("affine:database") {
|
||||
continue;
|
||||
}
|
||||
|
||||
// enqueue children first to keep traversal order similar to JS implementation
|
||||
walker.enqueue_children(&block_id, block);
|
||||
|
||||
if is_known_unsupported_markdown_flavour(flavour.as_str()) {
|
||||
known_unsupported_blocks.push(format!("{block_id}:{flavour}"));
|
||||
if is_edgeless_markdown_flavour(flavour.as_str()) {
|
||||
skipped_subtrees.insert(block_id.clone());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if BlockFlavour::from_str(flavour.as_str()).is_none() && flavour.as_str() != "affine:database" {
|
||||
unknown_blocks.push(format!("{block_id}:{flavour}"));
|
||||
skipped_subtrees.insert(block_id.clone());
|
||||
continue;
|
||||
}
|
||||
|
||||
if has_skipped_markdown_ancestor(&block_id, &context.parent_lookup, &skipped_subtrees) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let block_level = if ai_editable {
|
||||
block_level(&block_id, &root_block_id, &context.parent_lookup)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let ai_block = ai_editable && block_level == 2;
|
||||
|
||||
let mut block_markdown = String::new();
|
||||
|
||||
match flavour.as_str() {
|
||||
"affine:database" => {
|
||||
let title = get_string(block, "prop:title").unwrap_or_default();
|
||||
block_markdown.push_str(&format!("\n### {title}\n"));
|
||||
|
||||
if let Some(table) = build_database_table(block, &context, &md_options)
|
||||
&& let Some(table_md) = database_table_markdown(table)
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut block_markdown);
|
||||
writer.push_table(&table_md);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let Some(block_flavour) = BlockFlavour::from_str(flavour.as_str()) else {
|
||||
continue;
|
||||
};
|
||||
let spec = BlockSpec::from_block_map_with_flavour(block, block_flavour);
|
||||
let list_depth = if block_flavour == BlockFlavour::List {
|
||||
get_list_depth(&block_id, &context.parent_lookup, &context.block_pool)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
renderer.write_block(&mut block_markdown, &spec, list_depth);
|
||||
}
|
||||
}
|
||||
|
||||
if ai_block {
|
||||
markdown.push_str(&format!("<!-- block_id={block_id} flavour={flavour} -->\n"));
|
||||
}
|
||||
markdown.push_str(&block_markdown);
|
||||
}
|
||||
|
||||
Ok(MarkdownResult {
|
||||
title: doc_title,
|
||||
markdown,
|
||||
known_unsupported_blocks,
|
||||
unknown_blocks,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlResult, ParseError> {
|
||||
let doc = load_doc(&doc_bin, Some(doc_id.as_str()))?;
|
||||
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Err(ParseError::ParserError("blocks map is empty".into()));
|
||||
}
|
||||
|
||||
let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("root block not found".into()))?;
|
||||
let mut walker = context.walker();
|
||||
let mut blocks: Vec<BlockInfo> = Vec::with_capacity(context.block_pool.len());
|
||||
let mut doc_title = String::new();
|
||||
let mut summary = SummaryBuilder::new(SUMMARY_LIMIT as isize);
|
||||
|
||||
while let Some((parent_block_id, block_id)) = walker.next() {
|
||||
let block = match context.block_pool.get(&block_id) {
|
||||
Some(block) => block,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flavour = match get_flavour(block) {
|
||||
Some(flavour) => flavour,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let parent_block = parent_block_id.as_ref().and_then(|id| context.block_pool.get(id));
|
||||
let parent_flavour = parent_block.and_then(get_flavour);
|
||||
|
||||
let note_block = nearest_by_flavour(&block_id, NOTE_FLAVOUR, &context.parent_lookup, &context.block_pool);
|
||||
let note_block_id = note_block.as_ref().and_then(get_block_id);
|
||||
let display_mode = determine_display_mode(note_block.as_ref());
|
||||
|
||||
// enqueue children first to keep traversal order similar to JS implementation
|
||||
walker.enqueue_children(&block_id, block);
|
||||
|
||||
let build_block = |database_name: Option<&String>| {
|
||||
BlockInfo::base(
|
||||
&block_id,
|
||||
&flavour,
|
||||
parent_flavour.as_ref(),
|
||||
parent_block_id.as_ref(),
|
||||
compose_additional(&display_mode, note_block_id.as_ref(), database_name),
|
||||
)
|
||||
};
|
||||
|
||||
if flavour == PAGE_FLAVOUR {
|
||||
let title = get_string(block, "prop:title").unwrap_or_default();
|
||||
doc_title = title.clone();
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(vec![title]);
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if matches!(flavour.as_str(), "affine:paragraph" | "affine:list" | "affine:code") {
|
||||
if let Some(text) = block.get("prop:text").and_then(|value| value.to_text()) {
|
||||
let database_name = if flavour == "affine:paragraph" && parent_flavour.as_deref() == Some("affine:database") {
|
||||
parent_block.and_then(|map| get_string(map, "prop:title"))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let content = text.to_string();
|
||||
let text_len = text.len() as usize;
|
||||
let refs = extract_inline_references(&text.to_delta());
|
||||
|
||||
let mut info = build_block(database_name.as_ref());
|
||||
info.content = Some(vec![content.clone()]);
|
||||
if !refs.is_empty() {
|
||||
info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect());
|
||||
info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect());
|
||||
}
|
||||
blocks.push(info);
|
||||
summary.push_text(&content, text_len);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if matches!(flavour.as_str(), "affine:embed-linked-doc" | "affine:embed-synced-doc") {
|
||||
if let Some(page_id) = get_string(block, "prop:pageId") {
|
||||
let mut info = build_block(None);
|
||||
let payload = embed_ref_payload(block, &page_id);
|
||||
apply_doc_ref(&mut info, page_id, payload);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:attachment" {
|
||||
if let Some(blob_id) = get_string(block, "prop:sourceId") {
|
||||
let mut info = build_block(None);
|
||||
let name = get_string(block, "prop:name").unwrap_or_default();
|
||||
apply_blob_info(&mut info, blob_id, name);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:image" {
|
||||
let image = ImageSpec::from_block_map(block);
|
||||
if !image.source_id.is_empty() {
|
||||
let mut info = build_block(None);
|
||||
let caption = image.caption.unwrap_or_default();
|
||||
apply_blob_info(&mut info, image.source_id, caption);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:surface" {
|
||||
let texts = gather_surface_texts(block);
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(texts);
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:database" {
|
||||
let (texts, database_name) = gather_database_texts(block);
|
||||
let mut info = BlockInfo::base(
|
||||
&block_id,
|
||||
&flavour,
|
||||
parent_flavour.as_ref(),
|
||||
parent_block_id.as_ref(),
|
||||
compose_additional(&display_mode, note_block_id.as_ref(), database_name.as_ref()),
|
||||
);
|
||||
info.content = Some(texts);
|
||||
let refs = collect_database_cell_references(block);
|
||||
if !refs.is_empty() {
|
||||
info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect());
|
||||
info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect());
|
||||
}
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:latex" {
|
||||
if let Some(content) = get_string(block, "prop:latex") {
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(vec![content]);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:table" {
|
||||
let contents = table_cell_texts(block);
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(contents);
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if BOOKMARK_FLAVOURS.contains(&flavour.as_str()) {
|
||||
blocks.push(build_block(None));
|
||||
}
|
||||
}
|
||||
|
||||
if doc_title.is_empty() {
|
||||
doc_title = DEFAULT_PAGE_TITLE.into();
|
||||
}
|
||||
|
||||
Ok(CrawlResult {
|
||||
blocks,
|
||||
title: doc_title,
|
||||
summary: summary.into_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_doc_ids_from_binary(doc_bin: Vec<u8>, include_trash: bool) -> Result<Vec<String>, ParseError> {
|
||||
let doc = load_doc(&doc_bin, None)?;
|
||||
|
||||
let mut doc_ids = Vec::new();
|
||||
let meta = doc.get_map("meta")?;
|
||||
let pages_value = meta.get("pages");
|
||||
if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) {
|
||||
for page_val in pages.iter() {
|
||||
if let Some(page) = page_val.to_map() {
|
||||
let id = get_string(&page, "id");
|
||||
if let Some(id) = id {
|
||||
let trash = page
|
||||
.get("trash")
|
||||
.and_then(|v| match v.to_any() {
|
||||
Some(Any::True) => Some(true),
|
||||
Some(Any::False) => Some(false),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or(false);
|
||||
|
||||
if include_trash || !trash {
|
||||
doc_ids.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Ok(doc_ids);
|
||||
}
|
||||
|
||||
if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) {
|
||||
for entry in entries {
|
||||
let Any::Object(map) = entry else {
|
||||
continue;
|
||||
};
|
||||
let id = map.get("id").and_then(any_as_string).map(str::to_string);
|
||||
if let Some(id) = id {
|
||||
let trash = map.get("trash").map(any_truthy).unwrap_or(false);
|
||||
if include_trash || !trash {
|
||||
doc_ids.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(doc_ids)
|
||||
}
|
||||
|
||||
fn block_level(block_id: &str, root_id: &str, parent_lookup: &HashMap<String, String>) -> usize {
|
||||
let mut level = 0;
|
||||
let mut cursor = block_id;
|
||||
while let Some(parent) = parent_lookup.get(cursor) {
|
||||
level += 1;
|
||||
if parent == root_id {
|
||||
break;
|
||||
}
|
||||
cursor = parent;
|
||||
}
|
||||
level
|
||||
}
|
||||
|
||||
pub(super) fn text_content(block: &Map, key: &str) -> Option<(String, usize)> {
|
||||
block.get(key).and_then(|value| {
|
||||
value.to_text().map(|text| {
|
||||
let content = text.to_string();
|
||||
let len = text.len() as usize;
|
||||
(content, len)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn determine_display_mode(note_block: Option<&Map>) -> String {
|
||||
match note_block.and_then(|block| get_string(block, "prop:displayMode")) {
|
||||
Some(mode) if mode == "both" => "page".into(),
|
||||
Some(mode) => mode,
|
||||
None => "edgeless".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn compose_additional(
|
||||
display_mode: &str,
|
||||
note_block_id: Option<&String>,
|
||||
database_name: Option<&String>,
|
||||
) -> Option<String> {
|
||||
let mut payload = JsonMap::new();
|
||||
payload.insert("displayMode".into(), JsonValue::String(display_mode.to_string()));
|
||||
if let Some(note_id) = note_block_id {
|
||||
payload.insert("noteBlockId".into(), JsonValue::String(note_id.clone()));
|
||||
}
|
||||
if let Some(name) = database_name {
|
||||
payload.insert("databaseName".into(), JsonValue::String(name.clone()));
|
||||
}
|
||||
Some(JsonValue::Object(payload).to_string())
|
||||
}
|
||||
|
||||
fn apply_blob_info(info: &mut BlockInfo, blob_id: String, content: String) {
|
||||
info.blob = Some(vec![blob_id]);
|
||||
info.content = Some(vec![content]);
|
||||
}
|
||||
|
||||
fn apply_doc_ref(info: &mut BlockInfo, page_id: String, payload: Option<String>) {
|
||||
info.ref_doc_id = Some(vec![page_id]);
|
||||
if let Some(payload) = payload {
|
||||
info.ref_info = Some(vec![payload]);
|
||||
}
|
||||
}
|
||||
|
||||
fn embed_ref_payload(block: &Map, page_id: &str) -> Option<String> {
|
||||
let params = block.get("prop:params").as_ref().and_then(params_value_to_json);
|
||||
Some(build_reference_payload(page_id, params))
|
||||
}
|
||||
|
||||
fn gather_surface_texts(block: &Map) -> Vec<String> {
|
||||
let mut texts = Vec::new();
|
||||
let elements = match block.get("prop:elements").and_then(|value| value.to_map()) {
|
||||
Some(map) => map,
|
||||
None => return texts,
|
||||
};
|
||||
|
||||
if elements
|
||||
.get("type")
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.as_deref()
|
||||
!= Some("$blocksuite:internal:native$")
|
||||
{
|
||||
return texts;
|
||||
}
|
||||
|
||||
if let Some(value_map) = elements.get("value").and_then(|value| value.to_map()) {
|
||||
for value in value_map.values() {
|
||||
if let Some(element) = value.to_map()
|
||||
&& let Some(text) = element.get("text").and_then(|value| value.to_text())
|
||||
{
|
||||
texts.push(text.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
texts.sort();
|
||||
texts
|
||||
}
|
||||
|
||||
fn table_cell_texts(block: &Map) -> Vec<String> {
|
||||
let mut contents = Vec::new();
|
||||
for key in block.keys() {
|
||||
if key.starts_with("prop:cells.")
|
||||
&& key.ends_with(".text")
|
||||
&& let Some(value) = block.get(key).and_then(|value| value_to_string(&value))
|
||||
&& !value.is_empty()
|
||||
{
|
||||
contents.push(value);
|
||||
}
|
||||
}
|
||||
contents
|
||||
}
|
||||
|
||||
pub(super) fn text_content_for_summary(block: &Map, key: &str) -> Option<(String, usize)> {
|
||||
if let Some((text, len)) = text_content(block, key) {
|
||||
return Some((text, len));
|
||||
}
|
||||
|
||||
block.get(key).and_then(|value| {
|
||||
value_to_string(&value).map(|text| {
|
||||
let len = text.chars().count();
|
||||
(text, len)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
use y_octo::{AHashMap, Any, DocOptions, TextAttributes, TextDeltaOp, TextInsert, Value};
|
||||
|
||||
use super::*;
|
||||
use crate::doc_parser::build_full_doc;
|
||||
|
||||
#[test]
|
||||
fn test_parse_doc_from_binary() {
|
||||
let json = include_bytes!("../../../fixtures/demo.ydoc.json");
|
||||
let doc_bin = include_bytes!("../../../fixtures/demo.ydoc").to_vec();
|
||||
let doc_id = "dYpV7PPhk8amRkY5IAcVO".to_string();
|
||||
|
||||
let result = parse_doc_from_binary(doc_bin, doc_id).unwrap();
|
||||
let config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict)
|
||||
.numeric_mode(assert_json_diff::NumericMode::AssumeFloat);
|
||||
assert_json_diff::assert_json_matches!(
|
||||
serde_json::from_slice::<serde_json::Value>(json).unwrap(),
|
||||
serde_json::json!(result),
|
||||
config
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_database_cell_references() {
|
||||
let doc_id = "doc-with-db".to_string();
|
||||
let doc = DocOptions::new().with_guid(doc_id.clone()).build();
|
||||
let mut blocks = doc.get_or_create_map("blocks").unwrap();
|
||||
|
||||
let mut page = doc.create_map().unwrap();
|
||||
page.insert("sys:id".into(), "page").unwrap();
|
||||
page.insert("sys:flavour".into(), "affine:page").unwrap();
|
||||
let mut page_children = doc.create_array().unwrap();
|
||||
page_children.push("note").unwrap();
|
||||
page.insert("sys:children".into(), Value::Array(page_children)).unwrap();
|
||||
let mut page_title = doc.create_text().unwrap();
|
||||
page_title.insert(0, "Page").unwrap();
|
||||
page.insert("prop:title".into(), Value::Text(page_title)).unwrap();
|
||||
blocks.insert("page".into(), Value::Map(page)).unwrap();
|
||||
|
||||
let mut note = doc.create_map().unwrap();
|
||||
note.insert("sys:id".into(), "note").unwrap();
|
||||
note.insert("sys:flavour".into(), "affine:note").unwrap();
|
||||
let mut note_children = doc.create_array().unwrap();
|
||||
note_children.push("db").unwrap();
|
||||
note.insert("sys:children".into(), Value::Array(note_children)).unwrap();
|
||||
note.insert("prop:displayMode".into(), "page").unwrap();
|
||||
blocks.insert("note".into(), Value::Map(note)).unwrap();
|
||||
|
||||
let mut db = doc.create_map().unwrap();
|
||||
db.insert("sys:id".into(), "db").unwrap();
|
||||
db.insert("sys:flavour".into(), "affine:database").unwrap();
|
||||
db.insert("sys:children".into(), Value::Array(doc.create_array().unwrap()))
|
||||
.unwrap();
|
||||
let mut db_title = doc.create_text().unwrap();
|
||||
db_title.insert(0, "Database").unwrap();
|
||||
db.insert("prop:title".into(), Value::Text(db_title)).unwrap();
|
||||
|
||||
let mut columns = doc.create_array().unwrap();
|
||||
let mut column = doc.create_map().unwrap();
|
||||
column.insert("id".into(), "col1").unwrap();
|
||||
column.insert("name".into(), "Text").unwrap();
|
||||
column.insert("type".into(), "rich-text").unwrap();
|
||||
column
|
||||
.insert("data".into(), Value::Map(doc.create_map().unwrap()))
|
||||
.unwrap();
|
||||
columns.push(Value::Map(column)).unwrap();
|
||||
db.insert("prop:columns".into(), Value::Array(columns)).unwrap();
|
||||
|
||||
let mut cell_text = doc.create_text().unwrap();
|
||||
let mut reference = AHashMap::default();
|
||||
reference.insert("pageId".into(), Any::String("target-doc".into()));
|
||||
let mut params = AHashMap::default();
|
||||
params.insert("mode".into(), Any::String("page".into()));
|
||||
reference.insert("params".into(), Any::Object(params));
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert("reference".into(), Any::Object(reference));
|
||||
cell_text
|
||||
.apply_delta(&[
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("See ".into()),
|
||||
format: None,
|
||||
},
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Target".into()),
|
||||
format: Some(attrs),
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let mut cell = doc.create_map().unwrap();
|
||||
cell.insert("columnId".into(), "col1").unwrap();
|
||||
cell.insert("value".into(), Value::Text(cell_text)).unwrap();
|
||||
let mut row = doc.create_map().unwrap();
|
||||
row.insert("col1".into(), Value::Map(cell)).unwrap();
|
||||
let mut cells = doc.create_map().unwrap();
|
||||
cells.insert("row1".into(), Value::Map(row)).unwrap();
|
||||
db.insert("prop:cells".into(), Value::Map(cells)).unwrap();
|
||||
|
||||
blocks.insert("db".into(), Value::Map(db)).unwrap();
|
||||
|
||||
let doc_bin = doc.encode_update_v1().unwrap();
|
||||
let result = parse_doc_from_binary(doc_bin, doc_id).unwrap();
|
||||
let db_block = result.blocks.iter().find(|block| block.block_id == "db").unwrap();
|
||||
assert_eq!(db_block.ref_doc_id, Some(vec!["target-doc".to_string()]));
|
||||
assert_eq!(
|
||||
db_block.ref_info,
|
||||
Some(vec![build_reference_payload(
|
||||
"target-doc",
|
||||
Some(json!({"mode": "page"}))
|
||||
)])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_doc_to_markdown_ai_editable_image_table() {
|
||||
let doc_id = "ai-editable-doc";
|
||||
let markdown = "\n\n| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let doc_bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let result = parse_doc_to_markdown(doc_bin, doc_id.to_string(), true, None).expect("parse doc");
|
||||
let md = result.markdown;
|
||||
|
||||
assert!(md.contains("flavour=affine:image"));
|
||||
assert!(md.contains("blob://image-id"));
|
||||
assert!(md.contains("|A|B|"));
|
||||
assert!(md.contains("|---|---|"));
|
||||
assert!(
|
||||
result
|
||||
.known_unsupported_blocks
|
||||
.iter()
|
||||
.any(|block| block.ends_with(":affine:note"))
|
||||
);
|
||||
assert!(result.unknown_blocks.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_doc_to_markdown_with_edgeless_text_container() {
|
||||
let doc_id = "edgeless-text-doc".to_string();
|
||||
let doc = DocOptions::new().with_guid(doc_id.clone()).build();
|
||||
let mut blocks = doc.get_or_create_map("blocks").unwrap();
|
||||
|
||||
let mut page = doc.create_map().unwrap();
|
||||
page.insert("sys:id".into(), "page").unwrap();
|
||||
page.insert("sys:flavour".into(), "affine:page").unwrap();
|
||||
let mut page_children = doc.create_array().unwrap();
|
||||
page_children.push("surface").unwrap();
|
||||
page.insert("sys:children".into(), Value::Array(page_children)).unwrap();
|
||||
let mut page_title = doc.create_text().unwrap();
|
||||
page_title.insert(0, "Page").unwrap();
|
||||
page.insert("prop:title".into(), Value::Text(page_title)).unwrap();
|
||||
blocks.insert("page".into(), Value::Map(page)).unwrap();
|
||||
|
||||
let mut surface = doc.create_map().unwrap();
|
||||
surface.insert("sys:id".into(), "surface").unwrap();
|
||||
surface.insert("sys:flavour".into(), "affine:surface").unwrap();
|
||||
let mut surface_children = doc.create_array().unwrap();
|
||||
surface_children.push("edgeless-text").unwrap();
|
||||
surface
|
||||
.insert("sys:children".into(), Value::Array(surface_children))
|
||||
.unwrap();
|
||||
blocks.insert("surface".into(), Value::Map(surface)).unwrap();
|
||||
|
||||
let mut edgeless_text = doc.create_map().unwrap();
|
||||
edgeless_text.insert("sys:id".into(), "edgeless-text").unwrap();
|
||||
edgeless_text
|
||||
.insert("sys:flavour".into(), "affine:edgeless-text")
|
||||
.unwrap();
|
||||
let mut edgeless_text_children = doc.create_array().unwrap();
|
||||
edgeless_text_children.push("paragraph").unwrap();
|
||||
edgeless_text
|
||||
.insert("sys:children".into(), Value::Array(edgeless_text_children))
|
||||
.unwrap();
|
||||
blocks
|
||||
.insert("edgeless-text".into(), Value::Map(edgeless_text))
|
||||
.unwrap();
|
||||
|
||||
let mut paragraph = doc.create_map().unwrap();
|
||||
paragraph.insert("sys:id".into(), "paragraph").unwrap();
|
||||
paragraph.insert("sys:flavour".into(), "affine:paragraph").unwrap();
|
||||
paragraph
|
||||
.insert("sys:children".into(), Value::Array(doc.create_array().unwrap()))
|
||||
.unwrap();
|
||||
paragraph.insert("prop:type".into(), "text").unwrap();
|
||||
let mut paragraph_text = doc.create_text().unwrap();
|
||||
paragraph_text.insert(0, "hello from edgeless").unwrap();
|
||||
paragraph
|
||||
.insert("prop:text".into(), Value::Text(paragraph_text))
|
||||
.unwrap();
|
||||
blocks.insert("paragraph".into(), Value::Map(paragraph)).unwrap();
|
||||
|
||||
let doc_bin = doc.encode_update_v1().unwrap();
|
||||
let result = parse_doc_to_markdown(doc_bin, doc_id, false, None).expect("parse doc");
|
||||
|
||||
assert!(result.markdown.is_empty());
|
||||
assert!(
|
||||
result
|
||||
.known_unsupported_blocks
|
||||
.contains(&"surface:affine:surface".to_string())
|
||||
);
|
||||
assert!(
|
||||
result
|
||||
.known_unsupported_blocks
|
||||
.contains(&"edgeless-text:affine:edgeless-text".to_string())
|
||||
);
|
||||
assert!(result.unknown_blocks.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_doc_to_markdown_collects_unknown_blocks() {
|
||||
let doc_id = "unknown-block-doc".to_string();
|
||||
let doc = DocOptions::new().with_guid(doc_id.clone()).build();
|
||||
let mut blocks = doc.get_or_create_map("blocks").unwrap();
|
||||
|
||||
let mut page = doc.create_map().unwrap();
|
||||
page.insert("sys:id".into(), "page").unwrap();
|
||||
page.insert("sys:flavour".into(), "affine:page").unwrap();
|
||||
let mut page_children = doc.create_array().unwrap();
|
||||
page_children.push("mystery").unwrap();
|
||||
page.insert("sys:children".into(), Value::Array(page_children)).unwrap();
|
||||
let mut page_title = doc.create_text().unwrap();
|
||||
page_title.insert(0, "Page").unwrap();
|
||||
page.insert("prop:title".into(), Value::Text(page_title)).unwrap();
|
||||
blocks.insert("page".into(), Value::Map(page)).unwrap();
|
||||
|
||||
let mut mystery = doc.create_map().unwrap();
|
||||
mystery.insert("sys:id".into(), "mystery").unwrap();
|
||||
mystery.insert("sys:flavour".into(), "affine:custom-unknown").unwrap();
|
||||
let mut mystery_children = doc.create_array().unwrap();
|
||||
mystery_children.push("paragraph").unwrap();
|
||||
mystery
|
||||
.insert("sys:children".into(), Value::Array(mystery_children))
|
||||
.unwrap();
|
||||
blocks.insert("mystery".into(), Value::Map(mystery)).unwrap();
|
||||
|
||||
let mut paragraph = doc.create_map().unwrap();
|
||||
paragraph.insert("sys:id".into(), "paragraph").unwrap();
|
||||
paragraph.insert("sys:flavour".into(), "affine:paragraph").unwrap();
|
||||
paragraph
|
||||
.insert("sys:children".into(), Value::Array(doc.create_array().unwrap()))
|
||||
.unwrap();
|
||||
paragraph.insert("prop:type".into(), "text").unwrap();
|
||||
let mut paragraph_text = doc.create_text().unwrap();
|
||||
paragraph_text.insert(0, "child of unknown").unwrap();
|
||||
paragraph
|
||||
.insert("prop:text".into(), Value::Text(paragraph_text))
|
||||
.unwrap();
|
||||
blocks.insert("paragraph".into(), Value::Map(paragraph)).unwrap();
|
||||
|
||||
let doc_bin = doc.encode_update_v1().unwrap();
|
||||
let result = parse_doc_to_markdown(doc_bin, doc_id, false, None).expect("parse doc");
|
||||
|
||||
assert!(result.markdown.is_empty());
|
||||
assert!(result.known_unsupported_blocks.is_empty());
|
||||
assert_eq!(result.unknown_blocks, vec!["mystery:affine:custom-unknown".to_string()]);
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
use super::{build_full_doc, parse_doc_to_markdown};
|
||||
|
||||
fn assert_markdown_roundtrip(markdown: &str, expected: &str) {
|
||||
let doc_id = "roundtrip-doc";
|
||||
let title = "Roundtrip Title";
|
||||
let bin = build_full_doc(title, markdown, doc_id).expect("create doc");
|
||||
let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc");
|
||||
assert_eq!(result.title, title);
|
||||
assert_eq!(result.markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_inline_styles() {
|
||||
let markdown = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com).";
|
||||
let expected = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com).\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_list_items() {
|
||||
let markdown = "- Item 1\n- Item 2\n- [ ] Task\n- [x] Done";
|
||||
let expected = "* Item 1\n* Item 2\n- [ ] Task\n- [x] Done\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_code_block() {
|
||||
let markdown = "```rust\nfn main() {}\n```";
|
||||
let expected = "```rust\nfn main() {}\n\n```\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_code_block_indentation() {
|
||||
let markdown = "```python\n def indented():\n return \"ok\"\n```";
|
||||
let doc_id = "roundtrip-indent";
|
||||
let title = "Roundtrip Title";
|
||||
let bin = build_full_doc(title, markdown, doc_id).expect("create doc");
|
||||
let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc");
|
||||
assert!(result.markdown.contains("\n def indented():"));
|
||||
assert!(result.markdown.contains("\n return \"ok\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_table() {
|
||||
let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let expected = "|A|B|\n|---|---|\n|1|2|\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_image_with_caption() {
|
||||
let markdown = "";
|
||||
let expected = "<img\n src=\"blob://image-id\"\n alt=\"Alt\"\n width=\"auto\"\n height=\"auto\"\n/>\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
pub(super) const PAGE_FLAVOUR: &str = "affine:page";
|
||||
pub(super) const NOTE_FLAVOUR: &str = "affine:note";
|
||||
pub(super) const SURFACE_FLAVOUR: &str = "affine:surface";
|
||||
|
||||
pub(super) const SYS_ID: &str = "sys:id";
|
||||
pub(super) const SYS_FLAVOUR: &str = "sys:flavour";
|
||||
pub(super) const SYS_VERSION: &str = "sys:version";
|
||||
pub(super) const SYS_CHILDREN: &str = "sys:children";
|
||||
|
||||
pub(super) const PROP_TITLE: &str = "prop:title";
|
||||
pub(super) const PROP_TEXT: &str = "prop:text";
|
||||
pub(super) const PROP_TYPE: &str = "prop:type";
|
||||
pub(super) const PROP_CHECKED: &str = "prop:checked";
|
||||
pub(super) const PROP_LANGUAGE: &str = "prop:language";
|
||||
pub(super) const PROP_ORDER: &str = "prop:order";
|
||||
|
||||
pub(super) const PROP_ELEMENTS: &str = "prop:elements";
|
||||
pub(super) const PROP_BACKGROUND: &str = "prop:background";
|
||||
pub(super) const PROP_XYWH: &str = "prop:xywh";
|
||||
pub(super) const PROP_INDEX: &str = "prop:index";
|
||||
pub(super) const PROP_HIDDEN: &str = "prop:hidden";
|
||||
pub(super) const PROP_DISPLAY_MODE: &str = "prop:displayMode";
|
||||
|
||||
pub(super) const PROP_SOURCE_ID: &str = "prop:sourceId";
|
||||
pub(super) const PROP_CAPTION: &str = "prop:caption";
|
||||
pub(super) const PROP_WIDTH: &str = "prop:width";
|
||||
pub(super) const PROP_HEIGHT: &str = "prop:height";
|
||||
pub(super) const PROP_URL: &str = "prop:url";
|
||||
pub(super) const PROP_VIDEO_ID: &str = "prop:videoId";
|
||||
|
||||
pub(super) const PROP_ROWS_PREFIX: &str = "prop:rows.";
|
||||
pub(super) const PROP_COLUMNS_PREFIX: &str = "prop:columns.";
|
||||
pub(super) const PROP_CELLS_PREFIX: &str = "prop:cells.";
|
||||
pub(super) const PROP_ROW_ID_SUFFIX: &str = ".rowId";
|
||||
pub(super) const PROP_COLUMN_ID_SUFFIX: &str = ".columnId";
|
||||
pub(super) const PROP_ORDER_SUFFIX: &str = ".order";
|
||||
pub(super) const PROP_TEXT_SUFFIX: &str = ".text";
|
||||
|
||||
pub(super) fn table_row_id_key(row_id: &str) -> String {
|
||||
format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ROW_ID_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_row_order_key(row_id: &str) -> String {
|
||||
format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ORDER_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_column_id_key(column_id: &str) -> String {
|
||||
format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_COLUMN_ID_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_column_order_key(column_id: &str) -> String {
|
||||
format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_ORDER_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_cell_text_key(row_id: &str, column_id: &str) -> String {
|
||||
format!("{PROP_CELLS_PREFIX}{row_id}:{column_id}{PROP_TEXT_SUFFIX}")
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct MarkdownTableOptions {
|
||||
pub(super) escape_pipes: bool,
|
||||
pub(super) newline_replacement: &'static str,
|
||||
pub(super) trim: bool,
|
||||
}
|
||||
|
||||
impl MarkdownTableOptions {
|
||||
pub(super) const fn new(escape_pipes: bool, newline_replacement: &'static str, trim: bool) -> Self {
|
||||
Self {
|
||||
escape_pipes,
|
||||
newline_replacement,
|
||||
trim,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn render_markdown_table(rows: &[Vec<String>], options: MarkdownTableOptions) -> Option<String> {
|
||||
let (header, body) = rows.split_first()?;
|
||||
let header_line = format_table_row(header, options);
|
||||
let separator_line = format_table_row(&vec!["---".to_string(); header.len()], options);
|
||||
let mut lines = vec![header_line, separator_line];
|
||||
for row in body {
|
||||
lines.push(format_table_row(row, options));
|
||||
}
|
||||
Some(lines.join("\n"))
|
||||
}
|
||||
|
||||
fn format_table_row(row: &[String], options: MarkdownTableOptions) -> String {
|
||||
let cells = row
|
||||
.iter()
|
||||
.map(|cell| format_table_cell(cell, options))
|
||||
.collect::<Vec<_>>();
|
||||
format!("|{}|", cells.join("|"))
|
||||
}
|
||||
|
||||
fn format_table_cell(cell: &str, options: MarkdownTableOptions) -> String {
|
||||
let mut value = if options.trim {
|
||||
cell.trim().to_string()
|
||||
} else {
|
||||
cell.to_string()
|
||||
};
|
||||
|
||||
if options.escape_pipes {
|
||||
value = value.replace('|', "\\|");
|
||||
}
|
||||
if !options.newline_replacement.is_empty() {
|
||||
value = collapse_newlines(&value, options.newline_replacement);
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
fn collapse_newlines(value: &str, replacement: &str) -> String {
|
||||
if replacement.is_empty() {
|
||||
return value.to_string();
|
||||
}
|
||||
let mut out = String::with_capacity(value.len());
|
||||
let mut in_newline = false;
|
||||
for ch in value.chars() {
|
||||
if ch == '\n' {
|
||||
if !in_newline {
|
||||
out.push_str(replacement);
|
||||
in_newline = true;
|
||||
}
|
||||
} else {
|
||||
in_newline = false;
|
||||
out.push(ch);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
use serde_json::{Map as JsonMap, Value as JsonValue};
|
||||
use y_octo::{AHashMap, Any, Value};
|
||||
|
||||
pub(super) fn any_truthy(value: &Any) -> bool {
|
||||
match value {
|
||||
Any::True => true,
|
||||
Any::False | Any::Null | Any::Undefined => false,
|
||||
Any::String(value) => !value.is_empty(),
|
||||
Any::Integer(value) => *value != 0,
|
||||
Any::Float32(value) => value.0 != 0.0,
|
||||
Any::Float64(value) => value.0 != 0.0,
|
||||
Any::BigInt64(value) => *value != 0,
|
||||
Any::Object(_) | Any::Array(_) | Any::Binary(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn any_as_string(value: &Any) -> Option<&str> {
|
||||
match value {
|
||||
Any::String(value) => Some(value),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn any_as_u64(value: &Any) -> Option<u64> {
|
||||
match value {
|
||||
Any::Integer(value) if *value >= 0 => Some(*value as u64),
|
||||
Any::Float32(value) if value.0 >= 0.0 => Some(value.0 as u64),
|
||||
Any::Float64(value) if value.0 >= 0.0 => Some(value.0 as u64),
|
||||
Any::BigInt64(value) if *value >= 0 => Some(*value as u64),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn value_to_string(value: &Value) -> Option<String> {
|
||||
if let Some(text) = value.to_text() {
|
||||
return Some(text.to_string());
|
||||
}
|
||||
|
||||
if let Some(any) = value.to_any() {
|
||||
return any_to_string(&any);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn value_to_any(value: &Value) -> Option<Any> {
|
||||
if let Some(any) = value.to_any() {
|
||||
return Some(any);
|
||||
}
|
||||
|
||||
if let Some(text) = value.to_text() {
|
||||
return Some(Any::String(text.to_string()));
|
||||
}
|
||||
|
||||
if let Some(array) = value.to_array() {
|
||||
let mut values = Vec::new();
|
||||
for item in array.iter() {
|
||||
if let Some(any) = value_to_any(&item) {
|
||||
values.push(any);
|
||||
} else if let Some(text) = value_to_string(&item) {
|
||||
values.push(Any::String(text));
|
||||
}
|
||||
}
|
||||
return Some(Any::Array(values));
|
||||
}
|
||||
|
||||
if let Some(map) = value.to_map() {
|
||||
let mut values = AHashMap::default();
|
||||
for key in map.keys() {
|
||||
if let Some(entry) = map.get(key) {
|
||||
if let Some(any) = value_to_any(&entry) {
|
||||
values.insert(key.to_string(), any);
|
||||
} else if let Some(text) = value_to_string(&entry) {
|
||||
values.insert(key.to_string(), Any::String(text));
|
||||
}
|
||||
}
|
||||
}
|
||||
return Some(Any::Object(values));
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn value_to_f64(value: Value) -> Option<f64> {
|
||||
value.to_any().and_then(|any| match any {
|
||||
Any::Integer(v) => Some(v as f64),
|
||||
Any::BigInt64(v) => Some(v as f64),
|
||||
Any::Float32(v) => Some(v.0 as f64),
|
||||
Any::Float64(v) => Some(v.0),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn any_to_string(any: &Any) -> Option<String> {
|
||||
match any {
|
||||
Any::String(value) => Some(value.to_string()),
|
||||
Any::Integer(value) => Some(value.to_string()),
|
||||
Any::Float32(value) => Some(value.0.to_string()),
|
||||
Any::Float64(value) => Some(value.0.to_string()),
|
||||
Any::BigInt64(value) => Some(value.to_string()),
|
||||
Any::True => Some("true".into()),
|
||||
Any::False => Some("false".into()),
|
||||
Any::Null | Any::Undefined => None,
|
||||
Any::Array(_) | Any::Object(_) | Any::Binary(_) => serde_json::to_string(any).ok(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn params_any_map_to_json(params: &AHashMap<String, Any>) -> JsonValue {
|
||||
let mut values = JsonMap::new();
|
||||
for (key, value) in params.iter() {
|
||||
if let Ok(value) = serde_json::to_value(value) {
|
||||
values.insert(key.clone(), value);
|
||||
}
|
||||
}
|
||||
JsonValue::Object(values)
|
||||
}
|
||||
|
||||
pub(super) fn params_value_to_json(params: &Value) -> Option<JsonValue> {
|
||||
serde_json::to_value(params).ok()
|
||||
}
|
||||
|
||||
pub(super) fn build_reference_payload(doc_id: &str, params: Option<JsonValue>) -> String {
|
||||
let mut payload = JsonMap::new();
|
||||
payload.insert("docId".into(), JsonValue::String(doc_id.to_string()));
|
||||
if let Some(JsonValue::Object(params)) = params {
|
||||
for (key, value) in params.into_iter() {
|
||||
payload.insert(key, value);
|
||||
}
|
||||
}
|
||||
JsonValue::Object(payload).to_string()
|
||||
}
|
||||
@@ -1,426 +0,0 @@
|
||||
use y_octo::{TextDeltaOp, TextInsert};
|
||||
|
||||
use super::{
|
||||
super::schema::{
|
||||
PROP_CAPTION, PROP_CELLS_PREFIX, PROP_CHECKED, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER,
|
||||
PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL, PROP_VIDEO_ID, PROP_WIDTH, SYS_CHILDREN,
|
||||
SYS_FLAVOUR, SYS_ID, SYS_VERSION, table_cell_text_key, table_column_id_key, table_column_order_key,
|
||||
table_row_id_key, table_row_order_key,
|
||||
},
|
||||
*,
|
||||
};
|
||||
|
||||
pub(super) const BOXED_NATIVE_TYPE: &str = "$blocksuite:internal:native$";
|
||||
pub(super) const NOTE_BG_LIGHT: &str = "#ffffff";
|
||||
pub(super) const NOTE_BG_DARK: &str = "#252525";
|
||||
const TABLE_ORDER_WIDTH: usize = 6;
|
||||
|
||||
pub(super) fn block_version(flavour: &str) -> i32 {
|
||||
match flavour {
|
||||
"affine:page" => 2,
|
||||
"affine:surface" => 5,
|
||||
"affine:note" => 1,
|
||||
"affine:paragraph" => 1,
|
||||
"affine:list" => 1,
|
||||
"affine:code" => 1,
|
||||
"affine:divider" => 1,
|
||||
"affine:image" => 1,
|
||||
"affine:table" => 1,
|
||||
"affine:bookmark" => 1,
|
||||
"affine:embed-youtube" => 1,
|
||||
"affine:embed-iframe" => 1,
|
||||
"affine:callout" => 1,
|
||||
_ => 1,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct TextBlockProps<'a> {
|
||||
pub block_type: Option<&'a str>,
|
||||
pub checked: Option<bool>,
|
||||
pub language: Option<&'a str>,
|
||||
pub order: Option<i64>,
|
||||
pub text: &'a [TextDeltaOp],
|
||||
}
|
||||
|
||||
pub(super) struct ImageBlockProps<'a> {
|
||||
pub source_id: &'a str,
|
||||
pub caption: Option<&'a str>,
|
||||
pub width: Option<f64>,
|
||||
pub height: Option<f64>,
|
||||
}
|
||||
|
||||
pub(super) struct BookmarkBlockProps<'a> {
|
||||
pub url: &'a str,
|
||||
pub caption: Option<&'a str>,
|
||||
}
|
||||
|
||||
pub(super) struct EmbedYoutubeBlockProps<'a> {
|
||||
pub video_id: &'a str,
|
||||
}
|
||||
|
||||
pub(super) struct EmbedIframeBlockProps<'a> {
|
||||
pub url: &'a str,
|
||||
}
|
||||
|
||||
pub(super) fn insert_text(doc: &Doc, block: &mut Map, key: &str, ops: &[TextDeltaOp]) -> Result<(), ParseError> {
|
||||
let mut text = doc.create_text()?;
|
||||
// Attach first so updates encode parent types before their contents.
|
||||
block.insert(key.to_string(), Value::Text(text.clone()))?;
|
||||
if !ops.is_empty() {
|
||||
text.apply_delta(ops)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn text_ops_from_plain(text: &str) -> Vec<TextDeltaOp> {
|
||||
if text.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text.to_string()),
|
||||
format: None,
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn insert_children(doc: &Doc, block: &mut Map, children: &[String]) -> Result<(), ParseError> {
|
||||
let mut array = doc.create_array()?;
|
||||
// Attach first so updates encode parent types before their contents.
|
||||
block.insert(SYS_CHILDREN.to_string(), Value::Array(array.clone()))?;
|
||||
for child_id in children {
|
||||
array.push(child_id.to_string())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn insert_block_map(doc: &Doc, blocks_map: &mut Map, block_id: &str) -> Result<Map, ParseError> {
|
||||
let empty_map = doc.create_map()?;
|
||||
blocks_map.insert(block_id.to_string(), Value::Map(empty_map))?;
|
||||
|
||||
blocks_map
|
||||
.get(block_id)
|
||||
.and_then(|value| value.to_map())
|
||||
.ok_or_else(|| ParseError::ParserError("Failed to retrieve inserted block map".into()))
|
||||
}
|
||||
|
||||
pub(super) fn insert_sys_fields(block: &mut Map, block_id: &str, flavour: &str) -> Result<(), ParseError> {
|
||||
block.insert(SYS_ID.to_string(), Any::String(block_id.to_string()))?;
|
||||
block.insert(SYS_FLAVOUR.to_string(), Any::String(flavour.to_string()))?;
|
||||
block.insert(SYS_VERSION.to_string(), Any::Integer(block_version(flavour)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_text_block_props(
|
||||
doc: &Doc,
|
||||
block: &mut Map,
|
||||
props: &TextBlockProps<'_>,
|
||||
preserve_text: bool,
|
||||
clear_missing: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
match props.block_type {
|
||||
Some(block_type) => {
|
||||
block.insert(PROP_TYPE.to_string(), Any::String(block_type.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_TYPE).is_some() {
|
||||
block.remove(PROP_TYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !preserve_text && !props.text.is_empty() {
|
||||
insert_text(doc, block, PROP_TEXT, props.text)?;
|
||||
} else if !preserve_text && clear_missing && block.get(PROP_TEXT).is_some() {
|
||||
block.remove(PROP_TEXT);
|
||||
}
|
||||
|
||||
match props.checked {
|
||||
Some(checked) => {
|
||||
block.insert(PROP_CHECKED.to_string(), if checked { Any::True } else { Any::False })?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_CHECKED).is_some() {
|
||||
block.remove(PROP_CHECKED);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.language {
|
||||
Some(language) => {
|
||||
block.insert(PROP_LANGUAGE.to_string(), Any::String(language.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_LANGUAGE).is_some() {
|
||||
block.remove(PROP_LANGUAGE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.order {
|
||||
Some(order) => {
|
||||
block.insert(PROP_ORDER.to_string(), Any::Float64((order as f64).into()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_ORDER).is_some() {
|
||||
block.remove(PROP_ORDER);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_image_block_props(
|
||||
block: &mut Map,
|
||||
props: &ImageBlockProps<'_>,
|
||||
clear_missing: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_SOURCE_ID.to_string(), Any::String(props.source_id.to_string()))?;
|
||||
|
||||
match props.caption {
|
||||
Some(caption) => {
|
||||
block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_CAPTION).is_some() {
|
||||
block.remove(PROP_CAPTION);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.width {
|
||||
Some(width) => {
|
||||
block.insert(PROP_WIDTH.to_string(), Any::Float64(width.into()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_WIDTH).is_some() {
|
||||
block.remove(PROP_WIDTH);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.height {
|
||||
Some(height) => {
|
||||
block.insert(PROP_HEIGHT.to_string(), Any::Float64(height.into()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_HEIGHT).is_some() {
|
||||
block.remove(PROP_HEIGHT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_bookmark_block_props(
|
||||
block: &mut Map,
|
||||
props: &BookmarkBlockProps<'_>,
|
||||
clear_missing: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?;
|
||||
|
||||
match props.caption {
|
||||
Some(caption) => {
|
||||
block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_CAPTION).is_some() {
|
||||
block.remove(PROP_CAPTION);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_embed_youtube_block_props(
|
||||
block: &mut Map,
|
||||
props: &EmbedYoutubeBlockProps<'_>,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_VIDEO_ID.to_string(), Any::String(props.video_id.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_embed_iframe_block_props(
|
||||
block: &mut Map,
|
||||
props: &EmbedIframeBlockProps<'_>,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_table_block_props(block: &mut Map, rows: &[Vec<String>]) -> Result<(), ParseError> {
|
||||
clear_table_props(block);
|
||||
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let column_count = rows.iter().map(|row| row.len()).max().unwrap_or(0);
|
||||
let column_ids: Vec<String> = (0..column_count).map(|_| nanoid::nanoid!()).collect();
|
||||
|
||||
for (col_idx, column_id) in column_ids.iter().enumerate() {
|
||||
let order = format_table_order(col_idx);
|
||||
block.insert(table_column_id_key(column_id), Any::String(column_id.to_string()))?;
|
||||
block.insert(table_column_order_key(column_id), Any::String(order))?;
|
||||
}
|
||||
|
||||
for (row_idx, row) in rows.iter().enumerate() {
|
||||
let row_id = nanoid::nanoid!();
|
||||
let order = format_table_order(row_idx);
|
||||
block.insert(table_row_id_key(&row_id), Any::String(row_id.to_string()))?;
|
||||
block.insert(table_row_order_key(&row_id), Any::String(order))?;
|
||||
|
||||
for (col_idx, column_id) in column_ids.iter().enumerate() {
|
||||
let cell_text = row.get(col_idx).cloned().unwrap_or_default();
|
||||
block.insert(table_cell_text_key(&row_id, column_id), Any::String(cell_text))?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) struct ApplyBlockOptions {
|
||||
pub preserve_text: bool,
|
||||
pub clear_missing: bool,
|
||||
}
|
||||
|
||||
pub(super) fn apply_block_spec(
|
||||
doc: &Doc,
|
||||
block: &mut Map,
|
||||
spec: &BlockSpec,
|
||||
options: ApplyBlockOptions,
|
||||
) -> Result<(), ParseError> {
|
||||
match spec.flavour {
|
||||
BlockFlavour::Image => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let image = spec
|
||||
.image
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("image spec missing".into()))?;
|
||||
let props = ImageBlockProps {
|
||||
source_id: &image.source_id,
|
||||
caption: image.caption.as_deref(),
|
||||
width: image.width,
|
||||
height: image.height,
|
||||
};
|
||||
apply_image_block_props(block, &props, options.clear_missing)?;
|
||||
}
|
||||
BlockFlavour::Bookmark => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let bookmark = spec
|
||||
.bookmark
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("bookmark spec missing".into()))?;
|
||||
let props = BookmarkBlockProps {
|
||||
url: &bookmark.url,
|
||||
caption: bookmark.caption.as_deref(),
|
||||
};
|
||||
apply_bookmark_block_props(block, &props, options.clear_missing)?;
|
||||
}
|
||||
BlockFlavour::EmbedYoutube => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let embed = spec
|
||||
.embed_youtube
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?;
|
||||
let props = EmbedYoutubeBlockProps {
|
||||
video_id: &embed.video_id,
|
||||
};
|
||||
apply_embed_youtube_block_props(block, &props)?;
|
||||
}
|
||||
BlockFlavour::EmbedIframe => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let embed = spec
|
||||
.embed_iframe
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?;
|
||||
let props = EmbedIframeBlockProps { url: &embed.url };
|
||||
apply_embed_iframe_block_props(block, &props)?;
|
||||
}
|
||||
BlockFlavour::Callout => {
|
||||
return Ok(());
|
||||
}
|
||||
BlockFlavour::Table => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let table = spec
|
||||
.table
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("table spec missing".into()))?;
|
||||
apply_table_block_props(block, &table.rows)?;
|
||||
}
|
||||
_ => {
|
||||
let props = TextBlockProps {
|
||||
block_type: spec.block_type_str(),
|
||||
checked: spec.checked,
|
||||
language: spec.language.as_deref(),
|
||||
order: spec.order,
|
||||
text: &spec.text,
|
||||
};
|
||||
apply_text_block_props(doc, block, &props, options.preserve_text, options.clear_missing)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn insert_block_tree(doc: &Doc, blocks_map: &mut Map, node: &BlockNode) -> Result<String, ParseError> {
|
||||
let block_id = nanoid::nanoid!();
|
||||
let mut block_map = insert_block_map(doc, blocks_map, &block_id)?;
|
||||
|
||||
insert_sys_fields(&mut block_map, &block_id, node.spec.flavour.as_str())?;
|
||||
apply_block_spec(
|
||||
doc,
|
||||
&mut block_map,
|
||||
&node.spec,
|
||||
ApplyBlockOptions {
|
||||
preserve_text: false,
|
||||
clear_missing: false,
|
||||
},
|
||||
)?;
|
||||
|
||||
let child_ids = node
|
||||
.children
|
||||
.iter()
|
||||
.map(|child| insert_block_tree(doc, blocks_map, child))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
insert_children(doc, &mut block_map, &child_ids)?;
|
||||
|
||||
Ok(block_id)
|
||||
}
|
||||
|
||||
fn clear_table_props(block: &mut Map) {
|
||||
let keys = block
|
||||
.keys()
|
||||
.filter(|key| {
|
||||
key.starts_with(PROP_ROWS_PREFIX) || key.starts_with(PROP_COLUMNS_PREFIX) || key.starts_with(PROP_CELLS_PREFIX)
|
||||
})
|
||||
.map(|s| s.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
for key in keys {
|
||||
block.remove(&key);
|
||||
}
|
||||
}
|
||||
|
||||
fn format_table_order(index: usize) -> String {
|
||||
format!("{index:0width$}", width = TABLE_ORDER_WIDTH)
|
||||
}
|
||||
|
||||
pub(super) fn boxed_empty_map(doc: &Doc) -> Result<Map, ParseError> {
|
||||
doc.create_map().map_err(ParseError::from)
|
||||
}
|
||||
|
||||
pub(super) fn note_background_map(doc: &Doc) -> Result<Map, ParseError> {
|
||||
doc.create_map().map_err(ParseError::from)
|
||||
}
|
||||
@@ -1,281 +0,0 @@
|
||||
//! Markdown to YDoc conversion module
|
||||
//!
|
||||
//! Converts markdown content into AFFiNE-compatible y-octo document binary
|
||||
//! format.
|
||||
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::{
|
||||
super::{
|
||||
markdown::parse_markdown_blocks,
|
||||
schema::{PROP_BACKGROUND, PROP_DISPLAY_MODE, PROP_ELEMENTS, PROP_HIDDEN, PROP_INDEX, PROP_XYWH, SURFACE_FLAVOUR},
|
||||
},
|
||||
builder::{
|
||||
BOXED_NATIVE_TYPE, NOTE_BG_DARK, NOTE_BG_LIGHT, boxed_empty_map, insert_block_map, insert_block_tree,
|
||||
insert_children, insert_sys_fields, insert_text, note_background_map, text_ops_from_plain,
|
||||
},
|
||||
*,
|
||||
};
|
||||
|
||||
/// Converts markdown into an AFFiNE-compatible y-octo document binary.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `title` - The document title
|
||||
/// * `markdown` - The markdown content to convert
|
||||
/// * `doc_id` - The document ID to use
|
||||
///
|
||||
/// # Returns
|
||||
/// A binary vector containing the y-octo encoded document update
|
||||
pub fn build_full_doc(title: &str, markdown: &str, doc_id: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let nodes = parse_markdown_blocks(markdown)?;
|
||||
build_doc_update(doc_id, title, &nodes)
|
||||
}
|
||||
|
||||
fn build_doc_update(doc_id: &str, title: &str, blocks: &[BlockNode]) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
let mut blocks_map = doc.get_or_create_map("blocks")?;
|
||||
|
||||
let page_id = nanoid::nanoid!();
|
||||
let surface_id = nanoid::nanoid!();
|
||||
let note_id = nanoid::nanoid!();
|
||||
|
||||
// Insert root blocks first to establish stable IDs.
|
||||
let mut page_map = insert_block_map(&doc, &mut blocks_map, &page_id)?;
|
||||
let mut surface_map = insert_block_map(&doc, &mut blocks_map, &surface_id)?;
|
||||
let mut note_map = insert_block_map(&doc, &mut blocks_map, ¬e_id)?;
|
||||
|
||||
// Create content blocks under note.
|
||||
let content_ids = insert_block_trees(&doc, &mut blocks_map, blocks)?;
|
||||
|
||||
// Page block
|
||||
insert_sys_fields(&mut page_map, &page_id, PAGE_FLAVOUR)?;
|
||||
insert_children(&doc, &mut page_map, &[surface_id.clone(), note_id.clone()])?;
|
||||
insert_text(&doc, &mut page_map, PROP_TITLE, &text_ops_from_plain(title))?;
|
||||
|
||||
// Surface block
|
||||
insert_sys_fields(&mut surface_map, &surface_id, SURFACE_FLAVOUR)?;
|
||||
insert_children(&doc, &mut surface_map, &[])?;
|
||||
let mut boxed = boxed_empty_map(&doc)?;
|
||||
surface_map.insert(PROP_ELEMENTS.to_string(), Value::Map(boxed.clone()))?;
|
||||
boxed.insert("type".to_string(), Any::String(BOXED_NATIVE_TYPE.to_string()))?;
|
||||
let value = doc.create_map()?;
|
||||
boxed.insert("value".to_string(), Value::Map(value))?;
|
||||
|
||||
// Note block
|
||||
insert_sys_fields(&mut note_map, ¬e_id, NOTE_FLAVOUR)?;
|
||||
insert_children(&doc, &mut note_map, &content_ids)?;
|
||||
let mut background = note_background_map(&doc)?;
|
||||
note_map.insert(PROP_BACKGROUND.to_string(), Value::Map(background.clone()))?;
|
||||
background.insert("light".to_string(), Any::String(NOTE_BG_LIGHT.to_string()))?;
|
||||
background.insert("dark".to_string(), Any::String(NOTE_BG_DARK.to_string()))?;
|
||||
note_map.insert(PROP_XYWH.to_string(), Any::String("[0,0,800,95]".to_string()))?;
|
||||
note_map.insert(PROP_INDEX.to_string(), Any::String("a0".to_string()))?;
|
||||
note_map.insert(PROP_HIDDEN.to_string(), Any::False)?;
|
||||
note_map.insert(PROP_DISPLAY_MODE.to_string(), Any::String("both".to_string()))?;
|
||||
|
||||
Ok(doc.encode_update_v1()?)
|
||||
}
|
||||
|
||||
fn insert_block_trees(doc: &Doc, blocks_map: &mut Map, blocks: &[BlockNode]) -> Result<Vec<String>, ParseError> {
|
||||
let mut ids = Vec::with_capacity(blocks.len());
|
||||
for block in blocks {
|
||||
let id = insert_block_tree(doc, blocks_map, block)?;
|
||||
ids.push(id);
|
||||
}
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::{Any, DocOptions};
|
||||
|
||||
use super::{
|
||||
super::super::{
|
||||
blocksuite::get_string,
|
||||
markdown::{MAX_BLOCKS, MAX_MARKDOWN_CHARS},
|
||||
schema::PAGE_FLAVOUR,
|
||||
},
|
||||
*,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn test_simple_markdown() {
|
||||
let markdown = "# Hello World\n\nThis is a test paragraph.";
|
||||
let result = build_full_doc("Hello World", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_title_from_param() {
|
||||
let markdown = "# Markdown Title\n\nContent.";
|
||||
let doc_id = "title-param-test";
|
||||
let bin = build_full_doc("External Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut title = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
|
||||
{
|
||||
title = get_string(&block_map, "prop:title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(title.as_deref(), Some("External Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_list() {
|
||||
let markdown = "# Test List\n\n- Item 1\n- Item 2\n- Item 3";
|
||||
let result = build_full_doc("Test List", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_code() {
|
||||
let markdown = "# Code Example\n\n```rust\nfn main() {\n println!(\"Hello\");\n}\n```";
|
||||
let result = build_full_doc("Code Example", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_headings() {
|
||||
let markdown = "# H1\n\n## H2\n\n### H3\n\nParagraph text.";
|
||||
let result = build_full_doc("H1", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_markdown() {
|
||||
let result = build_full_doc("Untitled", "", "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_whitespace_only_markdown() {
|
||||
let result = build_full_doc("Untitled", " \n\n\t\n ", "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_without_h1() {
|
||||
let markdown = "## Secondary Heading\n\nSome content without H1.";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nested_lists() {
|
||||
let markdown = "# Nested Lists\n\n- Item 1\n - Nested 1.1\n - Nested 1.2\n- Item 2\n - Nested 2.1";
|
||||
let result = build_full_doc("Nested Lists", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blockquote() {
|
||||
let markdown = "# Title\n\n> A blockquote";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_divider() {
|
||||
let markdown = "# Title\n\nBefore divider\n\n---\n\nAfter divider";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_numbered_list() {
|
||||
let markdown = "# Title\n\n1. First item\n2. Second item";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_too_large() {
|
||||
let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1);
|
||||
let result = build_full_doc("Title", &markdown, "test-doc-id");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_block_limit() {
|
||||
let mut markdown = String::from("# Title\n\n");
|
||||
for i in 0..=MAX_BLOCKS {
|
||||
markdown.push_str(&format!("Paragraph {i}\n\n"));
|
||||
}
|
||||
let result = build_full_doc("Title", &markdown, "test-doc-id");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_image() {
|
||||
let markdown = "";
|
||||
let doc_id = "image-doc";
|
||||
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut found = false;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image")
|
||||
{
|
||||
let source_id = get_string(&block_map, "prop:sourceId");
|
||||
assert_eq!(source_id.as_deref(), Some("image-id"));
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_table() {
|
||||
let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let doc_id = "table-doc";
|
||||
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut found_cell = false;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table")
|
||||
{
|
||||
for key in block_map.keys() {
|
||||
if key.starts_with("prop:cells.") && key.ends_with(".text") {
|
||||
let value = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::String(value) => Some(value),
|
||||
_ => None,
|
||||
});
|
||||
if let Some(value) = value
|
||||
&& (value == "A" || value == "1")
|
||||
{
|
||||
found_cell = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found_cell);
|
||||
}
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
use super::{
|
||||
builder::{insert_text, text_ops_from_plain},
|
||||
root_doc::ensure_pages_array,
|
||||
*,
|
||||
};
|
||||
|
||||
pub fn update_doc_title(existing_binary: &[u8], doc_id: &str, title: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = load_doc(existing_binary, Some(doc_id))?;
|
||||
|
||||
let state_before = doc.get_state_vector();
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Err(ParseError::ParserError("blocks map is empty".into()));
|
||||
}
|
||||
|
||||
let mut page_block = find_page_block(&blocks_map)?;
|
||||
let current = get_string(&page_block, PROP_TITLE).unwrap_or_default();
|
||||
if current != title {
|
||||
insert_text(&doc, &mut page_block, PROP_TITLE, &text_ops_from_plain(title))?;
|
||||
}
|
||||
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
pub fn update_root_doc_meta_title(root_doc_bin: &[u8], doc_id: &str, title: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = load_doc_or_new(root_doc_bin)?;
|
||||
|
||||
let state_before = doc.get_state_vector();
|
||||
let mut meta = doc.get_or_create_map("meta")?;
|
||||
let mut pages = ensure_pages_array(&doc, &mut meta)?;
|
||||
|
||||
let mut found = false;
|
||||
for idx in 0..pages.len() {
|
||||
let Some(mut page) = pages.get(idx).and_then(|v| v.to_map()) else {
|
||||
continue;
|
||||
};
|
||||
if get_string(&page, "id").as_deref() == Some(doc_id) {
|
||||
page.insert("title".to_string(), Any::String(title.to_string()))?;
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
let page_map = doc.create_map()?;
|
||||
|
||||
let idx = pages.len();
|
||||
pages.insert(idx, Value::Map(page_map))?;
|
||||
|
||||
if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) {
|
||||
inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?;
|
||||
inserted_page.insert("title".to_string(), Any::String(title.to_string()))?;
|
||||
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as i64)
|
||||
.unwrap_or(0);
|
||||
inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?;
|
||||
|
||||
let tags = doc.create_array()?;
|
||||
inserted_page.insert("tags".to_string(), Value::Array(tags))?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
fn find_page_block(blocks_map: &Map) -> Result<Map, ParseError> {
|
||||
let index = build_block_index(blocks_map);
|
||||
let page_id = find_block_id_by_flavour(&index.block_pool, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
|
||||
blocks_map
|
||||
.get(&page_id)
|
||||
.and_then(|value| value.to_map())
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::*;
|
||||
use crate::doc_parser::{add_doc_to_root_doc, build_full_doc};
|
||||
|
||||
#[test]
|
||||
fn test_update_doc_title() {
|
||||
let doc_id = "doc-meta-title-test";
|
||||
let initial = build_full_doc("Old Title", "Content.", doc_id).expect("create doc");
|
||||
let delta = update_doc_title(&initial, doc_id, "New Title").expect("update title");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&initial).expect("apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut title = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
|
||||
{
|
||||
title = get_string(&block_map, "prop:title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(title.as_deref(), Some("New Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_root_doc_meta_title() {
|
||||
let doc_id = "root-meta-title-test";
|
||||
let root_bin = add_doc_to_root_doc(Vec::new(), doc_id, Some("Old Title")).expect("create root meta");
|
||||
let delta = update_root_doc_meta_title(&root_bin, doc_id, "New Title").expect("update meta");
|
||||
|
||||
let mut doc = DocOptions::new().build();
|
||||
doc.apply_update_from_binary_v1(&root_bin).expect("apply root");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let meta = doc.get_map("meta").expect("meta map");
|
||||
let pages = meta.get("pages").and_then(|v| v.to_array()).expect("pages array");
|
||||
let mut title = None;
|
||||
for page in pages.iter() {
|
||||
if let Some(page_map) = page.to_map()
|
||||
&& get_string(&page_map, "id").as_deref() == Some(doc_id)
|
||||
{
|
||||
title = get_string(&page_map, "title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(title.as_deref(), Some("New Title"));
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
use y_octo::{Any, DocOptions, Map};
|
||||
|
||||
use super::{
|
||||
super::{doc_loader::is_empty_doc, value::value_to_string},
|
||||
ParseError,
|
||||
};
|
||||
|
||||
pub fn update_doc_properties(
|
||||
existing_binary: &[u8],
|
||||
properties_doc_id: &str,
|
||||
target_doc_id: &str,
|
||||
created_by: Option<&str>,
|
||||
updated_by: Option<&str>,
|
||||
) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = if is_empty_doc(existing_binary) {
|
||||
DocOptions::new().with_guid(properties_doc_id.to_string()).build()
|
||||
} else {
|
||||
super::load_doc(existing_binary, Some(properties_doc_id))?
|
||||
};
|
||||
|
||||
let state_before = doc.get_state_vector();
|
||||
let mut record = doc.get_or_create_map(target_doc_id)?;
|
||||
let mut changed = false;
|
||||
|
||||
if record.get("id").is_none() {
|
||||
record.insert("id".to_string(), Any::String(target_doc_id.to_string()))?;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if let Some(created_by) = created_by
|
||||
&& get_record_string(&record, "createdBy").as_deref() != Some(created_by)
|
||||
{
|
||||
record.insert("createdBy".to_string(), Any::String(created_by.to_string()))?;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if let Some(updated_by) = updated_by
|
||||
&& get_record_string(&record, "updatedBy").as_deref() != Some(updated_by)
|
||||
{
|
||||
record.insert("updatedBy".to_string(), Any::String(updated_by.to_string()))?;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if !changed {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
fn get_record_string(record: &Map, key: &str) -> Option<String> {
|
||||
record.get(key).and_then(|value| value_to_string(&value))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_update_doc_properties_creates_record() {
|
||||
let properties_doc_id = "doc-properties";
|
||||
let target_doc_id = "doc-1";
|
||||
let update = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-1"))
|
||||
.expect("update properties");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(properties_doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&update).expect("apply");
|
||||
|
||||
let record = doc.get_map(target_doc_id).expect("record map");
|
||||
assert_eq!(get_record_string(&record, "id").as_deref(), Some(target_doc_id));
|
||||
assert_eq!(get_record_string(&record, "createdBy").as_deref(), Some("user-1"));
|
||||
assert_eq!(get_record_string(&record, "updatedBy").as_deref(), Some("user-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_doc_properties_no_change() {
|
||||
let properties_doc_id = "doc-properties-no-change";
|
||||
let target_doc_id = "doc-2";
|
||||
let initial = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-2"))
|
||||
.expect("initial update");
|
||||
|
||||
let delta = update_doc_properties(
|
||||
&initial,
|
||||
properties_doc_id,
|
||||
target_doc_id,
|
||||
Some("user-1"),
|
||||
Some("user-2"),
|
||||
)
|
||||
.expect("no change update");
|
||||
|
||||
assert!(delta.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
pub mod builder;
|
||||
mod create;
|
||||
mod doc_meta;
|
||||
mod doc_properties;
|
||||
mod root_doc;
|
||||
mod update;
|
||||
|
||||
pub use create::build_full_doc;
|
||||
pub use doc_meta::{update_doc_title, update_root_doc_meta_title};
|
||||
pub use doc_properties::update_doc_properties;
|
||||
pub use root_doc::{add_doc_to_root_doc, build_public_root_doc};
|
||||
pub use update::update_doc;
|
||||
use y_octo::{Any, Doc, Map, Value};
|
||||
|
||||
use super::{
|
||||
ParseError,
|
||||
block_spec::{BlockFlavour, BlockNode, BlockSpec},
|
||||
blocksuite::{build_block_index, find_block_id_by_flavour, get_string},
|
||||
doc_loader::{load_doc, load_doc_or_new},
|
||||
schema::{NOTE_FLAVOUR, PAGE_FLAVOUR, PROP_TITLE},
|
||||
};
|
||||
@@ -1,291 +0,0 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use y_octo::Array;
|
||||
|
||||
use super::*;
|
||||
|
||||
const DEFAULT_DOC_TITLE: &str = "Untitled";
|
||||
const SIMPLE_PAGE_META_KEYS: &[&str] = &[
|
||||
"id",
|
||||
"title",
|
||||
"createDate",
|
||||
"updatedDate",
|
||||
"trash",
|
||||
"trashDate",
|
||||
"headerImage",
|
||||
];
|
||||
|
||||
fn any_to_value(doc: &Doc, any: Any) -> Result<Value, ParseError> {
|
||||
match any {
|
||||
Any::Array(values) => {
|
||||
let mut array = doc.create_array()?;
|
||||
for value in values {
|
||||
let item = any_to_value(doc, value)?;
|
||||
array.push(item)?;
|
||||
}
|
||||
Ok(Value::Array(array))
|
||||
}
|
||||
Any::Object(values) => {
|
||||
let mut map = doc.create_map()?;
|
||||
for (key, value) in values {
|
||||
let item = any_to_value(doc, value)?;
|
||||
map.insert(key, item)?;
|
||||
}
|
||||
Ok(Value::Map(map))
|
||||
}
|
||||
_ => Ok(Value::Any(any)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn ensure_pages_array(doc: &Doc, meta: &mut Map) -> Result<Array, ParseError> {
|
||||
let pages_value = meta.get("pages");
|
||||
if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) {
|
||||
return Ok(pages);
|
||||
}
|
||||
|
||||
if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) {
|
||||
let mut pages = doc.create_array()?;
|
||||
for entry in entries {
|
||||
let value = any_to_value(doc, entry)?;
|
||||
pages.push(value)?;
|
||||
}
|
||||
meta.insert("pages".to_string(), Value::Array(pages.clone()))?;
|
||||
return Ok(pages);
|
||||
}
|
||||
|
||||
let pages = doc.create_array()?;
|
||||
meta.insert("pages".to_string(), Value::Array(pages.clone()))?;
|
||||
Ok(pages)
|
||||
}
|
||||
|
||||
/// Adds a document ID to the root doc's meta.pages array.
|
||||
/// Returns a binary update that can be applied to the root doc.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `root_doc_bin` - The current root doc binary
|
||||
/// * `doc_id` - The document ID to add
|
||||
/// * `title` - Optional title for the document
|
||||
///
|
||||
/// # Returns
|
||||
/// A Vec<u8> containing the y-octo update binary to add the doc
|
||||
pub fn add_doc_to_root_doc(root_doc_bin: Vec<u8>, doc_id: &str, title: Option<&str>) -> Result<Vec<u8>, ParseError> {
|
||||
// Handle empty or minimal root doc - create a new one
|
||||
let doc = load_doc_or_new(&root_doc_bin)?;
|
||||
|
||||
// Capture state before modifications to encode only the delta
|
||||
let state_before = doc.get_state_vector();
|
||||
|
||||
// Get or create the meta map
|
||||
let mut meta = doc.get_or_create_map("meta")?;
|
||||
|
||||
let mut pages = ensure_pages_array(&doc, &mut meta)?;
|
||||
|
||||
// Check if doc already exists
|
||||
let doc_exists = pages.iter().any(|page_val| {
|
||||
page_val
|
||||
.to_map()
|
||||
.and_then(|page| get_string(&page, "id"))
|
||||
.map(|id| id == doc_id)
|
||||
.unwrap_or(false)
|
||||
});
|
||||
|
||||
if !doc_exists {
|
||||
let page_map = doc.create_map()?;
|
||||
|
||||
let idx = pages.len();
|
||||
pages.insert(idx, Value::Map(page_map))?;
|
||||
|
||||
if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) {
|
||||
inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?;
|
||||
|
||||
let page_title = title.unwrap_or(DEFAULT_DOC_TITLE);
|
||||
inserted_page.insert("title".to_string(), Any::String(page_title.to_string()))?;
|
||||
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as i64)
|
||||
.unwrap_or(0);
|
||||
inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?;
|
||||
|
||||
let tags = doc.create_array()?;
|
||||
inserted_page.insert("tags".to_string(), Value::Array(tags))?;
|
||||
}
|
||||
}
|
||||
|
||||
// Encode only the changes (delta) since state_before
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
fn insert_page_stub(doc: &Doc, pages: &mut Array, doc_id: &str, title: Option<&str>) -> Result<(), ParseError> {
|
||||
let page_map = doc.create_map()?;
|
||||
let idx = pages.len();
|
||||
pages.insert(idx, Value::Map(page_map))?;
|
||||
|
||||
if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) {
|
||||
inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?;
|
||||
inserted_page.insert(
|
||||
"title".to_string(),
|
||||
Any::String(title.unwrap_or(DEFAULT_DOC_TITLE).to_string()),
|
||||
)?;
|
||||
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as i64)
|
||||
.unwrap_or(0);
|
||||
inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?;
|
||||
|
||||
let tags = doc.create_array()?;
|
||||
inserted_page.insert("tags".to_string(), Value::Array(tags))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn insert_simple_array(doc: &Doc, values: &[Any]) -> Result<Value, ParseError> {
|
||||
let mut array = doc.create_array()?;
|
||||
for value in values {
|
||||
match value {
|
||||
Any::Array(_) | Any::Object(_) => continue,
|
||||
_ => array.push(Value::Any(value.clone()))?,
|
||||
}
|
||||
}
|
||||
Ok(Value::Array(array))
|
||||
}
|
||||
|
||||
fn insert_page_from_any(doc: &Doc, pages: &mut Array, page: Any) -> Result<Option<String>, ParseError> {
|
||||
let Any::Object(page) = page else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let Some(page_id) = page.get("id").and_then(|value| match value {
|
||||
Any::String(value) => Some(value.clone()),
|
||||
_ => None,
|
||||
}) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let page_map = doc.create_map()?;
|
||||
let idx = pages.len();
|
||||
pages.insert(idx, Value::Map(page_map))?;
|
||||
|
||||
let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
for key in SIMPLE_PAGE_META_KEYS {
|
||||
let Some(value) = page.get(*key) else {
|
||||
continue;
|
||||
};
|
||||
match value {
|
||||
Any::Array(values) => {
|
||||
inserted_page.insert((*key).to_string(), insert_simple_array(doc, values)?)?;
|
||||
}
|
||||
Any::Object(_) => continue,
|
||||
_ => {
|
||||
inserted_page.insert((*key).to_string(), Value::Any(value.clone()))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(Any::Array(tags)) = page.get("tags") {
|
||||
inserted_page.insert("tags".to_string(), insert_simple_array(doc, tags)?)?;
|
||||
}
|
||||
|
||||
Ok(Some(page_id))
|
||||
}
|
||||
|
||||
pub fn build_public_root_doc(root_doc_bin: &[u8], doc_metas: &[(&str, Option<&str>)]) -> Result<Vec<u8>, ParseError> {
|
||||
let source = load_doc_or_new(root_doc_bin)?;
|
||||
let public_doc_ids = doc_metas
|
||||
.iter()
|
||||
.map(|(doc_id, _title)| (*doc_id).to_string())
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
let doc = Doc::default();
|
||||
let mut meta = doc.get_or_create_map("meta")?;
|
||||
let mut pages = ensure_pages_array(&doc, &mut meta)?;
|
||||
let mut copied = HashSet::new();
|
||||
|
||||
if let Ok(source_meta) = source.get_map("meta") {
|
||||
let source_pages_value = source_meta.get("pages");
|
||||
|
||||
if let Some(source_pages) = source_pages_value.as_ref().and_then(|value| value.to_array()) {
|
||||
for page_val in source_pages.iter() {
|
||||
let Some(page) = page_val.to_map() else {
|
||||
continue;
|
||||
};
|
||||
let Some(page_id) = get_string(&page, "id") else {
|
||||
continue;
|
||||
};
|
||||
if !public_doc_ids.contains(&page_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let page_object = Any::Object(
|
||||
page
|
||||
.iter()
|
||||
.filter_map(|(key, value)| value.to_any().map(|any| (key.to_string(), any)))
|
||||
.collect(),
|
||||
);
|
||||
if let Some(inserted_page_id) = insert_page_from_any(&doc, &mut pages, page_object)? {
|
||||
copied.insert(inserted_page_id);
|
||||
}
|
||||
}
|
||||
} else if let Some(Any::Array(entries)) = source_pages_value.and_then(|value| value.to_any()) {
|
||||
for entry in entries {
|
||||
let Any::Object(page) = entry.clone() else {
|
||||
continue;
|
||||
};
|
||||
let Some(Any::String(page_id)) = page.get("id") else {
|
||||
continue;
|
||||
};
|
||||
if !public_doc_ids.contains(page_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(inserted_page_id) = insert_page_from_any(&doc, &mut pages, entry)? {
|
||||
copied.insert(inserted_page_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (doc_id, title) in doc_metas {
|
||||
if copied.contains(*doc_id) {
|
||||
continue;
|
||||
}
|
||||
insert_page_stub(&doc, &mut pages, doc_id, *title)?;
|
||||
}
|
||||
|
||||
Ok(doc.encode_update_v1()?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::doc_parser::get_doc_ids_from_binary;
|
||||
|
||||
#[test]
|
||||
fn test_build_public_root_doc_filters_private_pages() {
|
||||
let root = add_doc_to_root_doc(Vec::new(), "public-doc", Some("Public")).expect("create public entry");
|
||||
let update = add_doc_to_root_doc(root.clone(), "private-doc", Some("Private")).expect("create private entry");
|
||||
|
||||
let mut merged = load_doc_or_new(&root).expect("load root");
|
||||
merged
|
||||
.apply_update_from_binary_v1(&update)
|
||||
.expect("apply second update");
|
||||
let merged_bin = merged.encode_update_v1().expect("encode merged");
|
||||
|
||||
let public_root = build_public_root_doc(
|
||||
&merged_bin,
|
||||
&[("public-doc", Some("Public")), ("missing-public-doc", Some("Fallback"))],
|
||||
)
|
||||
.expect("build public root");
|
||||
|
||||
let doc_ids = get_doc_ids_from_binary(public_root, false).expect("read public root doc ids");
|
||||
assert_eq!(
|
||||
doc_ids,
|
||||
vec!["public-doc".to_string(), "missing-public-doc".to_string()]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,671 +0,0 @@
|
||||
//! Update YDoc module
|
||||
//!
|
||||
//! Provides functionality to update existing AFFiNE documents by applying
|
||||
//! surgical y-octo operations based on content differences.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use super::{
|
||||
super::{
|
||||
block_spec::{TreeNode, count_tree_nodes, text_delta_eq},
|
||||
blocksuite::{collect_child_ids, find_child_id_by_flavour},
|
||||
markdown::{MAX_BLOCKS, parse_markdown_blocks},
|
||||
},
|
||||
builder::{ApplyBlockOptions, apply_block_spec, insert_block_tree, insert_children},
|
||||
*,
|
||||
};
|
||||
|
||||
const MAX_LCS_CELLS: usize = 2_000_000;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct StoredNode {
|
||||
id: String,
|
||||
spec: BlockSpec,
|
||||
children: Vec<StoredNode>,
|
||||
}
|
||||
|
||||
impl TreeNode for StoredNode {
|
||||
fn children(&self) -> &[StoredNode] {
|
||||
&self.children
|
||||
}
|
||||
}
|
||||
|
||||
struct DocState {
|
||||
doc: Doc,
|
||||
note_id: String,
|
||||
blocks: Vec<StoredNode>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum PatchOp {
|
||||
Keep(usize, usize),
|
||||
Delete(usize),
|
||||
Insert(usize),
|
||||
Update(usize, usize),
|
||||
}
|
||||
|
||||
/// Updates an existing document with new markdown content.
|
||||
///
|
||||
/// This function performs structural diffing between the existing document
|
||||
/// and the new markdown content, then applies block-level replacements
|
||||
/// for changed blocks. This enables proper CRDT merging with concurrent
|
||||
/// edits from other clients.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `existing_binary` - The current document binary
|
||||
/// * `new_markdown` - The new markdown content (document title is not updated)
|
||||
/// * `doc_id` - The document ID
|
||||
///
|
||||
/// # Returns
|
||||
/// A binary vector representing only the delta (changes) to apply
|
||||
pub fn update_doc(existing_binary: &[u8], new_markdown: &str, doc_id: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let mut new_nodes = parse_markdown_blocks(new_markdown)?;
|
||||
let state = load_doc_state(existing_binary, doc_id)?;
|
||||
|
||||
check_limits(&state.blocks, &new_nodes)?;
|
||||
|
||||
let state_before = state.doc.get_state_vector();
|
||||
|
||||
let mut blocks_map = state.doc.get_map("blocks")?;
|
||||
|
||||
let new_children = sync_nodes(&state.doc, &mut blocks_map, &state.blocks, &mut new_nodes)?;
|
||||
sync_children(&state.doc, &mut blocks_map, &state.note_id, &new_children)?;
|
||||
|
||||
Ok(state.doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
fn load_doc_state(binary: &[u8], doc_id: &str) -> Result<DocState, ParseError> {
|
||||
let doc = load_doc(binary, Some(doc_id))?;
|
||||
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Err(ParseError::ParserError("blocks map is empty".into()));
|
||||
}
|
||||
|
||||
let block_index = build_block_index(&blocks_map);
|
||||
let page_id = find_block_id_by_flavour(&block_index.block_pool, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
|
||||
let page_block = block_index
|
||||
.block_pool
|
||||
.get(&page_id)
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
|
||||
let note_id = find_child_id_by_flavour(page_block, &block_index.block_pool, NOTE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("note block not found".into()))?;
|
||||
let note_block = block_index
|
||||
.block_pool
|
||||
.get(¬e_id)
|
||||
.ok_or_else(|| ParseError::ParserError("note block not found".into()))?;
|
||||
let content_ids = collect_child_ids(note_block);
|
||||
|
||||
let mut blocks = Vec::new();
|
||||
for block_id in content_ids {
|
||||
let block = block_index
|
||||
.block_pool
|
||||
.get(&block_id)
|
||||
.ok_or_else(|| ParseError::ParserError("content block not found".into()))?;
|
||||
blocks.push(build_stored_tree(&block_id, block, &block_index.block_pool)?);
|
||||
}
|
||||
|
||||
Ok(DocState { doc, note_id, blocks })
|
||||
}
|
||||
|
||||
fn build_stored_tree(block_id: &str, block: &Map, pool: &HashMap<String, Map>) -> Result<StoredNode, ParseError> {
|
||||
let spec = BlockSpec::from_block_map(block)?;
|
||||
|
||||
let child_ids = collect_child_ids(block);
|
||||
if !child_ids.is_empty() && !matches!(spec.flavour, BlockFlavour::List | BlockFlavour::Callout) {
|
||||
return Err(ParseError::ParserError(format!(
|
||||
"unsupported children on block: {block_id}"
|
||||
)));
|
||||
}
|
||||
let mut children = Vec::new();
|
||||
for child_id in child_ids {
|
||||
let child_block = pool
|
||||
.get(&child_id)
|
||||
.ok_or_else(|| ParseError::ParserError("child block not found".into()))?;
|
||||
children.push(build_stored_tree(&child_id, child_block, pool)?);
|
||||
}
|
||||
|
||||
Ok(StoredNode {
|
||||
id: block_id.to_string(),
|
||||
spec,
|
||||
children,
|
||||
})
|
||||
}
|
||||
|
||||
fn sync_nodes(
|
||||
doc: &Doc,
|
||||
blocks_map: &mut Map,
|
||||
current: &[StoredNode],
|
||||
target: &mut [BlockNode],
|
||||
) -> Result<Vec<String>, ParseError> {
|
||||
let ops = diff_blocks(current, target);
|
||||
let mut new_children = Vec::new();
|
||||
let mut to_remove = Vec::new();
|
||||
|
||||
for op in ops {
|
||||
match op {
|
||||
PatchOp::Keep(old_idx, new_idx) => {
|
||||
let old_node = ¤t[old_idx];
|
||||
let new_node = &target[new_idx];
|
||||
update_block_props(doc, blocks_map, old_node, &new_node.spec, true)?;
|
||||
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
|
||||
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
|
||||
new_children.push(old_node.id.clone());
|
||||
}
|
||||
PatchOp::Update(old_idx, new_idx) => {
|
||||
let old_node = ¤t[old_idx];
|
||||
let new_node = &target[new_idx];
|
||||
update_block_props(doc, blocks_map, old_node, &new_node.spec, false)?;
|
||||
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
|
||||
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
|
||||
new_children.push(old_node.id.clone());
|
||||
}
|
||||
PatchOp::Insert(new_idx) => {
|
||||
let new_id = insert_block_tree(doc, blocks_map, &target[new_idx])?;
|
||||
new_children.push(new_id);
|
||||
}
|
||||
PatchOp::Delete(old_idx) => {
|
||||
let node = ¤t[old_idx];
|
||||
if node.spec.flavour == BlockFlavour::Callout {
|
||||
new_children.push(node.id.clone());
|
||||
} else {
|
||||
collect_tree_ids(node, &mut to_remove);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for id in to_remove {
|
||||
blocks_map.remove(&id);
|
||||
}
|
||||
|
||||
Ok(new_children)
|
||||
}
|
||||
|
||||
fn diff_blocks(current: &[StoredNode], target: &[BlockNode]) -> Vec<PatchOp> {
|
||||
let old_len = current.len();
|
||||
let new_len = target.len();
|
||||
|
||||
if old_len == 0 {
|
||||
return (0..new_len).map(PatchOp::Insert).collect();
|
||||
}
|
||||
if new_len == 0 {
|
||||
return (0..old_len).map(PatchOp::Delete).collect();
|
||||
}
|
||||
|
||||
let mut lcs = vec![vec![0usize; new_len + 1]; old_len + 1];
|
||||
|
||||
for i in 1..=old_len {
|
||||
for j in 1..=new_len {
|
||||
let old_spec = ¤t[i - 1].spec;
|
||||
let new_spec = &target[j - 1].spec;
|
||||
|
||||
if old_spec.is_exact(new_spec) {
|
||||
lcs[i][j] = lcs[i - 1][j - 1] + 1;
|
||||
} else {
|
||||
lcs[i][j] = std::cmp::max(lcs[i - 1][j], lcs[i][j - 1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut ops = Vec::new();
|
||||
let mut i = old_len;
|
||||
let mut j = new_len;
|
||||
|
||||
while i > 0 || j > 0 {
|
||||
if i > 0 && j > 0 {
|
||||
let old_spec = ¤t[i - 1].spec;
|
||||
let new_spec = &target[j - 1].spec;
|
||||
|
||||
if old_spec.is_exact(new_spec) {
|
||||
ops.push(PatchOp::Keep(i - 1, j - 1));
|
||||
i -= 1;
|
||||
j -= 1;
|
||||
} else if old_spec.is_similar(new_spec)
|
||||
&& lcs[i - 1][j - 1] >= lcs[i - 1][j]
|
||||
&& lcs[i - 1][j - 1] >= lcs[i][j - 1]
|
||||
{
|
||||
ops.push(PatchOp::Update(i - 1, j - 1));
|
||||
i -= 1;
|
||||
j -= 1;
|
||||
} else if lcs[i][j - 1] >= lcs[i - 1][j] {
|
||||
ops.push(PatchOp::Insert(j - 1));
|
||||
j -= 1;
|
||||
} else {
|
||||
ops.push(PatchOp::Delete(i - 1));
|
||||
i -= 1;
|
||||
}
|
||||
} else if j > 0 {
|
||||
ops.push(PatchOp::Insert(j - 1));
|
||||
j -= 1;
|
||||
} else {
|
||||
ops.push(PatchOp::Delete(i - 1));
|
||||
i -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
ops.reverse();
|
||||
ops
|
||||
}
|
||||
|
||||
fn update_block_props(
|
||||
doc: &Doc,
|
||||
blocks_map: &mut Map,
|
||||
node: &StoredNode,
|
||||
target: &BlockSpec,
|
||||
preserve_text: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
let Some(mut block) = blocks_map.get(&node.id).and_then(|v| v.to_map()) else {
|
||||
return Err(ParseError::ParserError(format!("Block {} not found", node.id)));
|
||||
};
|
||||
|
||||
let preserve = match target.flavour {
|
||||
BlockFlavour::Image
|
||||
| BlockFlavour::Table
|
||||
| BlockFlavour::Bookmark
|
||||
| BlockFlavour::EmbedYoutube
|
||||
| BlockFlavour::EmbedIframe => preserve_text,
|
||||
_ => preserve_text || text_delta_eq(&node.spec.text, &target.text),
|
||||
};
|
||||
|
||||
apply_block_spec(
|
||||
doc,
|
||||
&mut block,
|
||||
target,
|
||||
ApplyBlockOptions {
|
||||
preserve_text: preserve,
|
||||
clear_missing: true,
|
||||
},
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sync_children(doc: &Doc, blocks_map: &mut Map, block_id: &str, children: &[String]) -> Result<(), ParseError> {
|
||||
let Some(mut block) = blocks_map.get(block_id).and_then(|v| v.to_map()) else {
|
||||
return Err(ParseError::ParserError("Block not found".into()));
|
||||
};
|
||||
|
||||
let current_children = collect_child_ids(&block);
|
||||
if current_children != children {
|
||||
insert_children(doc, &mut block, children)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_tree_ids(node: &StoredNode, output: &mut Vec<String>) {
|
||||
output.push(node.id.clone());
|
||||
for child in &node.children {
|
||||
collect_tree_ids(child, output);
|
||||
}
|
||||
}
|
||||
|
||||
fn check_limits(current: &[StoredNode], target: &[BlockNode]) -> Result<(), ParseError> {
|
||||
let current_count = count_tree_nodes(current);
|
||||
let target_count = count_tree_nodes(target);
|
||||
|
||||
if current_count > MAX_BLOCKS || target_count > MAX_BLOCKS {
|
||||
return Err(ParseError::ParserError("block_count_too_large".into()));
|
||||
}
|
||||
|
||||
if current_count.saturating_mul(target_count) > MAX_LCS_CELLS {
|
||||
return Err(ParseError::ParserError("diff_matrix_too_large".into()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::{Any, DocOptions, TextDeltaOp, TextInsert};
|
||||
|
||||
use super::{super::builder::text_ops_from_plain, *};
|
||||
use crate::doc_parser::{
|
||||
block_spec::BlockType, blocksuite::get_string, build_full_doc, markdown::MAX_MARKDOWN_CHARS, parse_doc_to_markdown,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn test_compute_text_diff_simple() {
|
||||
let ops = text_ops_from_plain("hello world");
|
||||
assert_eq!(ops.len(), 1);
|
||||
match &ops[0] {
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text),
|
||||
format: None,
|
||||
} => {
|
||||
assert_eq!(text, "hello world");
|
||||
}
|
||||
_ => panic!("unexpected delta op"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_content_block_similarity() {
|
||||
let b1 = BlockSpec {
|
||||
flavour: BlockFlavour::Paragraph,
|
||||
block_type: Some(BlockType::H1),
|
||||
text: text_ops_from_plain("Hello"),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
let b2 = BlockSpec {
|
||||
flavour: BlockFlavour::Paragraph,
|
||||
block_type: Some(BlockType::H1),
|
||||
text: text_ops_from_plain("World"),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
let b3 = BlockSpec {
|
||||
flavour: BlockFlavour::Paragraph,
|
||||
block_type: Some(BlockType::H2),
|
||||
text: text_ops_from_plain("Hello"),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
|
||||
assert!(b1.is_similar(&b2));
|
||||
assert!(!b1.is_similar(&b3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_roundtrip() {
|
||||
let initial_md = "# Test Document\n\nFirst paragraph.\n\nSecond paragraph.";
|
||||
let doc_id = "update-test";
|
||||
|
||||
let initial_bin = build_full_doc("Test Document", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let updated_md = "# Test Document\n\nFirst paragraph.\n\nModified second paragraph.\n\nNew third paragraph.";
|
||||
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
assert!(!delta.is_empty(), "Delta should contain changes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_does_not_update_page_title() {
|
||||
let initial_md = "# Original Title\n\nContent here.";
|
||||
let doc_id = "title-test";
|
||||
|
||||
let initial_bin = build_full_doc("Original Title", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let updated_md = "# New Title\n\nContent here.";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("Should apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map exists");
|
||||
let mut title = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
|
||||
{
|
||||
title = get_string(&block_map, "prop:title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(title.as_deref(), Some("Original Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_no_changes() {
|
||||
let markdown = "# Same Title\n\nSame content.";
|
||||
let doc_id = "no-change-test";
|
||||
|
||||
let initial_bin = build_full_doc("Same Title", markdown, doc_id).expect("Should create initial doc");
|
||||
let delta = update_doc(&initial_bin, markdown, doc_id).expect("Should compute delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
doc
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("Should apply delta even with no changes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_ignores_ai_editable_comments() {
|
||||
let markdown = "Plain paragraph.";
|
||||
let doc_id = "ai-comment-test";
|
||||
|
||||
let initial_bin = build_full_doc("Title", markdown, doc_id).expect("Should create initial doc");
|
||||
|
||||
let ai_markdown = parse_doc_to_markdown(initial_bin.clone(), doc_id.to_string(), true, None)
|
||||
.expect("parse doc")
|
||||
.markdown;
|
||||
assert!(ai_markdown.contains("block_id="));
|
||||
|
||||
let delta = update_doc(&initial_bin, &ai_markdown, doc_id).expect("Should compute delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("Should apply delta");
|
||||
|
||||
let before = parse_doc_to_markdown(initial_bin, doc_id.to_string(), false, None)
|
||||
.expect("parse before")
|
||||
.markdown;
|
||||
let after = parse_doc_to_markdown(doc.encode_update_v1().unwrap(), doc_id.to_string(), false, None)
|
||||
.expect("parse after")
|
||||
.markdown;
|
||||
|
||||
assert_eq!(after, before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_add_block() {
|
||||
let initial_md = "# Add Block Test\n\nOriginal paragraph.";
|
||||
let doc_id = "add-block-test";
|
||||
|
||||
let initial_bin = build_full_doc("Add Block Test", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
initial_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
|
||||
let updated_md = "# Add Block Test\n\nOriginal paragraph.\n\nNew paragraph added.";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
assert!(!delta.is_empty(), "Delta should contain changes");
|
||||
|
||||
let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("Should apply delta with new block");
|
||||
|
||||
let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
assert!(
|
||||
updated_count > initial_count,
|
||||
"Expected more blocks after insert, got {updated_count} vs {initial_count}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_delete_block() {
|
||||
let initial_md = "# Delete Block Test\n\nFirst paragraph.\n\nSecond paragraph to delete.";
|
||||
let doc_id = "delete-block-test";
|
||||
|
||||
let initial_bin = build_full_doc("Delete Block Test", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
initial_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
|
||||
let updated_md = "# Delete Block Test\n\nFirst paragraph.";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
assert!(!delta.is_empty(), "Delta should contain changes");
|
||||
|
||||
let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("Should apply delta with block deletion");
|
||||
|
||||
let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
assert!(
|
||||
updated_count < initial_count,
|
||||
"Expected fewer blocks after deletion, got {updated_count} vs {initial_count}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_update_image_caption() {
|
||||
let initial_md = "";
|
||||
let doc_id = "image-update-test";
|
||||
let initial_bin = build_full_doc("Image", initial_md, doc_id).expect("create doc");
|
||||
|
||||
let updated_md = "";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut caption = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image")
|
||||
{
|
||||
caption = get_string(&block_map, "prop:caption");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(caption.as_deref(), Some("New Caption"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_update_table_cell() {
|
||||
let initial_md = "| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let doc_id = "table-update-test";
|
||||
let initial_bin = build_full_doc("Table", initial_md, doc_id).expect("create doc");
|
||||
|
||||
let updated_md = "| A | B |\n| --- | --- |\n| 1 | 9 |";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut found = false;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table")
|
||||
{
|
||||
for key in block_map.keys() {
|
||||
if key.starts_with("prop:cells.")
|
||||
&& key.ends_with(".text")
|
||||
&& let Some(value) = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::String(value) => Some(value),
|
||||
_ => None,
|
||||
})
|
||||
&& value == "9"
|
||||
{
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_concurrent_merge_simulation() {
|
||||
let base_md = "# Concurrent Test\n\nBase paragraph.";
|
||||
let doc_id = "concurrent-test";
|
||||
|
||||
let base_bin = build_full_doc("Concurrent Test", base_md, doc_id).expect("Should create base doc");
|
||||
|
||||
let mut base_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
base_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base");
|
||||
let base_count = base_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
|
||||
let client_a_md = "# Concurrent Test\n\nModified by client A.";
|
||||
let delta_a = update_doc(&base_bin, client_a_md, doc_id).expect("Delta A");
|
||||
|
||||
let client_b_md = "# Concurrent Test\n\nBase paragraph.\n\nAdded by client B.";
|
||||
let delta_b = update_doc(&base_bin, client_b_md, doc_id).expect("Delta B");
|
||||
|
||||
let mut final_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
final_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base");
|
||||
final_doc.apply_update_from_binary_v1(&delta_a).expect("Apply delta A");
|
||||
final_doc.apply_update_from_binary_v1(&delta_b).expect("Apply delta B");
|
||||
|
||||
let final_count = final_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
assert!(
|
||||
final_count > base_count,
|
||||
"Expected merged blocks after concurrent updates, got {final_count} vs {base_count}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_empty_binary_errors() {
|
||||
let markdown = "# New Document\n\nCreated from empty binary.";
|
||||
let doc_id = "empty-fallback-test";
|
||||
|
||||
let result = update_doc(&[], markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
|
||||
let result = update_doc(&[0, 0], markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_markdown_too_large() {
|
||||
let initial_md = "# Title\n\nContent.";
|
||||
let doc_id = "size-limit-test";
|
||||
let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1);
|
||||
let result = update_doc(&initial_bin, &markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_rejects_unsupported_markdown() {
|
||||
let initial_md = "# Title\n\nContent.";
|
||||
let doc_id = "unsupported-test";
|
||||
let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let markdown = "# Title\n\n<div>HTML</div>";
|
||||
let result = update_doc(&initial_bin, markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
#[cfg(feature = "ydoc-loader")]
|
||||
pub mod doc_parser;
|
||||
#[cfg(feature = "hashcash")]
|
||||
pub mod hashcash;
|
||||
#[cfg(feature = "napi")]
|
||||
|
||||
Reference in New Issue
Block a user