mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-13 08:06:24 +08:00
feat: refactor doc write in native (#14272)
This commit is contained in:
Vendored
+51
-8
@@ -27,23 +27,24 @@ export interface Chunk {
|
||||
content: string
|
||||
}
|
||||
|
||||
export declare function fromModelName(modelName: string): Tokenizer | null
|
||||
|
||||
export declare function getMime(input: Uint8Array): string
|
||||
|
||||
export declare function htmlSanitize(input: string): string
|
||||
|
||||
/**
|
||||
* Converts markdown content to AFFiNE-compatible y-octo document binary.
|
||||
*
|
||||
* # Arguments
|
||||
* * `title` - The document title
|
||||
* * `markdown` - The markdown content to convert
|
||||
* * `doc_id` - The document ID to use for the y-octo doc
|
||||
*
|
||||
* # Returns
|
||||
* A Buffer containing the y-octo document update binary
|
||||
*/
|
||||
export declare function markdownToDocBinary(markdown: string, docId: string): Buffer
|
||||
export declare function createDocWithMarkdown(title: string, markdown: string, docId: string): Buffer
|
||||
|
||||
export declare function fromModelName(modelName: string): Tokenizer | null
|
||||
|
||||
export declare function getMime(input: Uint8Array): string
|
||||
|
||||
export declare function htmlSanitize(input: string): string
|
||||
|
||||
/**
|
||||
* Merge updates in form like `Y.applyUpdate(doc, update)` way and return the
|
||||
@@ -103,9 +104,38 @@ export declare function parseWorkspaceDoc(docBin: Buffer): NativeWorkspaceDocCon
|
||||
|
||||
export declare function readAllDocIdsFromRootDoc(docBin: Buffer, includeTrash?: boolean | undefined | null): Array<string>
|
||||
|
||||
/**
|
||||
* Updates or creates the docProperties record for a document.
|
||||
*
|
||||
* # Arguments
|
||||
* * `existing_binary` - The current docProperties document binary
|
||||
* * `properties_doc_id` - The docProperties document ID
|
||||
* (db$${workspaceId}$docProperties)
|
||||
* * `target_doc_id` - The document ID to update in docProperties
|
||||
* * `created_by` - Optional creator user ID
|
||||
* * `updated_by` - Optional updater user ID
|
||||
*
|
||||
* # Returns
|
||||
* A Buffer containing only the delta (changes) as a y-octo update binary
|
||||
*/
|
||||
export declare function updateDocProperties(existingBinary: Buffer, propertiesDocId: string, targetDocId: string, createdBy?: string | undefined | null, updatedBy?: string | undefined | null): Buffer
|
||||
|
||||
/**
|
||||
* Updates a document's title without touching content blocks.
|
||||
*
|
||||
* # Arguments
|
||||
* * `existing_binary` - The current document binary
|
||||
* * `title` - The new title
|
||||
* * `doc_id` - The document ID
|
||||
*
|
||||
* # Returns
|
||||
* A Buffer containing only the delta (changes) as a y-octo update binary
|
||||
*/
|
||||
export declare function updateDocTitle(existingBinary: Buffer, title: string, docId: string): Buffer
|
||||
|
||||
/**
|
||||
* Updates an existing document with new markdown content.
|
||||
* Uses structural and text-level diffing to apply minimal changes.
|
||||
* Uses structural diffing to apply block-level replacements for changes.
|
||||
*
|
||||
* # Arguments
|
||||
* * `existing_binary` - The current document binary
|
||||
@@ -117,4 +147,17 @@ export declare function readAllDocIdsFromRootDoc(docBin: Buffer, includeTrash?:
|
||||
*/
|
||||
export declare function updateDocWithMarkdown(existingBinary: Buffer, newMarkdown: string, docId: string): Buffer
|
||||
|
||||
/**
|
||||
* Updates a document title in the workspace root doc's meta.pages array.
|
||||
*
|
||||
* # Arguments
|
||||
* * `root_doc_bin` - The current root doc binary (workspaceId doc)
|
||||
* * `doc_id` - The document ID to update
|
||||
* * `title` - The new title for the document
|
||||
*
|
||||
* # Returns
|
||||
* A Buffer containing the y-octo update binary to apply to the root doc
|
||||
*/
|
||||
export declare function updateRootDocMetaTitle(rootDocBin: Buffer, docId: string, title: string): Buffer
|
||||
|
||||
export declare function verifyChallengeResponse(response: string, bits: number, resource: string): Promise<boolean>
|
||||
|
||||
@@ -136,20 +136,21 @@ pub fn read_all_doc_ids_from_root_doc(doc_bin: Buffer, include_trash: Option<boo
|
||||
/// Converts markdown content to AFFiNE-compatible y-octo document binary.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `title` - The document title
|
||||
/// * `markdown` - The markdown content to convert
|
||||
/// * `doc_id` - The document ID to use for the y-octo doc
|
||||
///
|
||||
/// # Returns
|
||||
/// A Buffer containing the y-octo document update binary
|
||||
#[napi]
|
||||
pub fn markdown_to_doc_binary(markdown: String, doc_id: String) -> Result<Buffer> {
|
||||
let result =
|
||||
doc_parser::markdown_to_ydoc(&markdown, &doc_id).map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
|
||||
pub fn create_doc_with_markdown(title: String, markdown: String, doc_id: String) -> Result<Buffer> {
|
||||
let result = doc_parser::build_full_doc(&title, &markdown, &doc_id)
|
||||
.map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
|
||||
Ok(Buffer::from(result))
|
||||
}
|
||||
|
||||
/// Updates an existing document with new markdown content.
|
||||
/// Uses structural and text-level diffing to apply minimal changes.
|
||||
/// Uses structural diffing to apply block-level replacements for changes.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `existing_binary` - The current document binary
|
||||
@@ -160,11 +161,58 @@ pub fn markdown_to_doc_binary(markdown: String, doc_id: String) -> Result<Buffer
|
||||
/// A Buffer containing only the delta (changes) as a y-octo update binary
|
||||
#[napi]
|
||||
pub fn update_doc_with_markdown(existing_binary: Buffer, new_markdown: String, doc_id: String) -> Result<Buffer> {
|
||||
let result = doc_parser::update_ydoc(&existing_binary, &new_markdown, &doc_id)
|
||||
let result = doc_parser::update_doc(&existing_binary, &new_markdown, &doc_id)
|
||||
.map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
|
||||
Ok(Buffer::from(result))
|
||||
}
|
||||
|
||||
/// Updates a document's title without touching content blocks.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `existing_binary` - The current document binary
|
||||
/// * `title` - The new title
|
||||
/// * `doc_id` - The document ID
|
||||
///
|
||||
/// # Returns
|
||||
/// A Buffer containing only the delta (changes) as a y-octo update binary
|
||||
#[napi]
|
||||
pub fn update_doc_title(existing_binary: Buffer, title: String, doc_id: String) -> Result<Buffer> {
|
||||
let result = doc_parser::update_doc_title(&existing_binary, &doc_id, &title)
|
||||
.map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
|
||||
Ok(Buffer::from(result))
|
||||
}
|
||||
|
||||
/// Updates or creates the docProperties record for a document.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `existing_binary` - The current docProperties document binary
|
||||
/// * `properties_doc_id` - The docProperties document ID
|
||||
/// (db$${workspaceId}$docProperties)
|
||||
/// * `target_doc_id` - The document ID to update in docProperties
|
||||
/// * `created_by` - Optional creator user ID
|
||||
/// * `updated_by` - Optional updater user ID
|
||||
///
|
||||
/// # Returns
|
||||
/// A Buffer containing only the delta (changes) as a y-octo update binary
|
||||
#[napi]
|
||||
pub fn update_doc_properties(
|
||||
existing_binary: Buffer,
|
||||
properties_doc_id: String,
|
||||
target_doc_id: String,
|
||||
created_by: Option<String>,
|
||||
updated_by: Option<String>,
|
||||
) -> Result<Buffer> {
|
||||
let result = doc_parser::update_doc_properties(
|
||||
&existing_binary,
|
||||
&properties_doc_id,
|
||||
&target_doc_id,
|
||||
created_by.as_deref(),
|
||||
updated_by.as_deref(),
|
||||
)
|
||||
.map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
|
||||
Ok(Buffer::from(result))
|
||||
}
|
||||
|
||||
/// Adds a document ID to the workspace root doc's meta.pages array.
|
||||
/// This registers the document in the workspace so it appears in the UI.
|
||||
///
|
||||
@@ -181,3 +229,19 @@ pub fn add_doc_to_root_doc(root_doc_bin: Buffer, doc_id: String, title: Option<S
|
||||
.map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
|
||||
Ok(Buffer::from(result))
|
||||
}
|
||||
|
||||
/// Updates a document title in the workspace root doc's meta.pages array.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `root_doc_bin` - The current root doc binary (workspaceId doc)
|
||||
/// * `doc_id` - The document ID to update
|
||||
/// * `title` - The new title for the document
|
||||
///
|
||||
/// # Returns
|
||||
/// A Buffer containing the y-octo update binary to apply to the root doc
|
||||
#[napi]
|
||||
pub fn update_root_doc_meta_title(root_doc_bin: Buffer, doc_id: String, title: String) -> Result<Buffer> {
|
||||
let result = doc_parser::update_root_doc_meta_title(&root_doc_bin, &doc_id, &title)
|
||||
.map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
|
||||
Ok(Buffer::from(result))
|
||||
}
|
||||
|
||||
+27
@@ -70,6 +70,33 @@ Generated by [AVA](https://avajs.dev).
|
||||
␊
|
||||
␊
|
||||
␊
|
||||
␊
|
||||
[](Bookmark,https://affine.pro/)␊
|
||||
␊
|
||||
␊
|
||||
[](Bookmark,https://www.youtube.com/@affinepro)␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
|
||||
alt=""␊
|
||||
width="1302"␊
|
||||
height="728"␊
|
||||
/>␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
|
||||
alt=""␊
|
||||
width="1463"␊
|
||||
height="374"␊
|
||||
/>␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
|
||||
alt=""␊
|
||||
width="862"␊
|
||||
height="1388"␊
|
||||
/>␊
|
||||
␊
|
||||
`,
|
||||
title: 'Write, Draw, Plan all at Once.',
|
||||
}
|
||||
|
||||
BIN
Binary file not shown.
+27
@@ -70,6 +70,33 @@ Generated by [AVA](https://avajs.dev).
|
||||
␊
|
||||
␊
|
||||
␊
|
||||
␊
|
||||
[](Bookmark,https://affine.pro/)␊
|
||||
␊
|
||||
␊
|
||||
[](Bookmark,https://www.youtube.com/@affinepro)␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
|
||||
alt=""␊
|
||||
width="1302"␊
|
||||
height="728"␊
|
||||
/>␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
|
||||
alt=""␊
|
||||
width="1463"␊
|
||||
height="374"␊
|
||||
/>␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
|
||||
alt=""␊
|
||||
width="862"␊
|
||||
height="1388"␊
|
||||
/>␊
|
||||
␊
|
||||
`,
|
||||
title: 'Write, Draw, Plan all at Once.',
|
||||
}
|
||||
|
||||
BIN
Binary file not shown.
+27
@@ -70,6 +70,33 @@ Generated by [AVA](https://avajs.dev).
|
||||
␊
|
||||
␊
|
||||
␊
|
||||
␊
|
||||
[](Bookmark,https://affine.pro/)␊
|
||||
␊
|
||||
␊
|
||||
[](Bookmark,https://www.youtube.com/@affinepro)␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
|
||||
alt=""␊
|
||||
width="1302"␊
|
||||
height="728"␊
|
||||
/>␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
|
||||
alt=""␊
|
||||
width="1463"␊
|
||||
height="374"␊
|
||||
/>␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
|
||||
alt=""␊
|
||||
width="862"␊
|
||||
height="1388"␊
|
||||
/>␊
|
||||
␊
|
||||
`,
|
||||
title: 'Write, Draw, Plan all at Once.',
|
||||
}
|
||||
|
||||
BIN
Binary file not shown.
@@ -1,10 +1,14 @@
|
||||
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { nanoid } from 'nanoid';
|
||||
|
||||
import { EventBus } from '../../base';
|
||||
import {
|
||||
addDocToRootDoc,
|
||||
markdownToDocBinary,
|
||||
createDocWithMarkdown,
|
||||
updateDocProperties,
|
||||
updateDocTitle,
|
||||
updateDocWithMarkdown,
|
||||
updateRootDocMetaTitle,
|
||||
} from '../../native';
|
||||
import { PgWorkspaceDocStorageAdapter } from './adapters/workspace';
|
||||
|
||||
@@ -16,22 +20,40 @@ export interface UpdateDocResult {
|
||||
success: boolean;
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface Events {
|
||||
'doc.updates.pushed': {
|
||||
spaceType: 'workspace' | 'userspace';
|
||||
spaceId: string;
|
||||
docId: string;
|
||||
updates: Uint8Array[];
|
||||
timestamp: number;
|
||||
editor?: string;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class DocWriter {
|
||||
private readonly logger = new Logger(DocWriter.name);
|
||||
|
||||
constructor(private readonly storage: PgWorkspaceDocStorageAdapter) {}
|
||||
constructor(
|
||||
private readonly storage: PgWorkspaceDocStorageAdapter,
|
||||
private readonly event: EventBus
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Creates a new document from markdown content.
|
||||
*
|
||||
* @param workspaceId - The workspace ID
|
||||
* @param markdown - The markdown content
|
||||
* @param title - The document title
|
||||
* @param markdown - The markdown content (body only)
|
||||
* @param editorId - Optional editor ID for tracking
|
||||
* @returns The created document ID
|
||||
*/
|
||||
async createDoc(
|
||||
workspaceId: string,
|
||||
title: string,
|
||||
markdown: string,
|
||||
editorId?: string
|
||||
): Promise<CreateDocResult> {
|
||||
@@ -58,24 +80,50 @@ export class DocWriter {
|
||||
`Creating doc ${docId} in workspace ${workspaceId} from markdown`
|
||||
);
|
||||
|
||||
// Convert markdown to y-octo binary
|
||||
const binary = markdownToDocBinary(markdown, docId);
|
||||
|
||||
// Extract title from markdown (first H1 heading)
|
||||
const titleMatch = markdown.match(/^#\s+(.+?)(?:\s*#+)?\s*$/m);
|
||||
const title = titleMatch ? titleMatch[1].trim() : undefined;
|
||||
// Convert markdown to y-octo binary using the provided title
|
||||
const binary = createDocWithMarkdown(title, markdown, docId);
|
||||
|
||||
// Prepare root doc update to register the new document
|
||||
const rootDocUpdate = addDocToRootDoc(rootDocBin, docId, title);
|
||||
|
||||
// Push both updates together - root doc first, then the new doc
|
||||
await this.storage.pushDocUpdates(
|
||||
const rootTimestamp = await this.storage.pushDocUpdates(
|
||||
workspaceId,
|
||||
workspaceId,
|
||||
[rootDocUpdate],
|
||||
editorId
|
||||
);
|
||||
await this.storage.pushDocUpdates(workspaceId, docId, [binary], editorId);
|
||||
this.emitDocUpdatesPushed({
|
||||
spaceId: workspaceId,
|
||||
docId: workspaceId,
|
||||
updates: [rootDocUpdate],
|
||||
timestamp: rootTimestamp,
|
||||
editor: editorId,
|
||||
});
|
||||
|
||||
const docTimestamp = await this.storage.pushDocUpdates(
|
||||
workspaceId,
|
||||
docId,
|
||||
[binary],
|
||||
editorId
|
||||
);
|
||||
this.emitDocUpdatesPushed({
|
||||
spaceId: workspaceId,
|
||||
docId,
|
||||
updates: [binary],
|
||||
timestamp: docTimestamp,
|
||||
editor: editorId,
|
||||
});
|
||||
|
||||
await this.updateDocProperties(
|
||||
workspaceId,
|
||||
docId,
|
||||
{
|
||||
createdBy: editorId,
|
||||
updatedBy: editorId,
|
||||
},
|
||||
editorId
|
||||
);
|
||||
|
||||
this.logger.debug(
|
||||
`Created and registered doc ${docId} in workspace ${workspaceId}`
|
||||
@@ -88,8 +136,10 @@ export class DocWriter {
|
||||
* Updates an existing document with new markdown content.
|
||||
*
|
||||
* Uses structural diffing to compute minimal changes between the existing
|
||||
* document and new markdown, then applies only the delta. This preserves
|
||||
* document history and enables proper CRDT merging with concurrent edits.
|
||||
* document and new markdown, then applies block-level replacements for
|
||||
* changed blocks. This preserves document history and enables proper CRDT
|
||||
* merging with concurrent edits.
|
||||
* Note: this does not update the document title.
|
||||
*
|
||||
* @param workspaceId - The workspace ID
|
||||
* @param docId - The document ID to update
|
||||
@@ -124,8 +174,194 @@ export class DocWriter {
|
||||
const delta = updateDocWithMarkdown(existingBinary, markdown, docId);
|
||||
|
||||
// Push only the delta changes
|
||||
await this.storage.pushDocUpdates(workspaceId, docId, [delta], editorId);
|
||||
const timestamp = await this.storage.pushDocUpdates(
|
||||
workspaceId,
|
||||
docId,
|
||||
[delta],
|
||||
editorId
|
||||
);
|
||||
this.emitDocUpdatesPushed({
|
||||
spaceId: workspaceId,
|
||||
docId,
|
||||
updates: [delta],
|
||||
timestamp,
|
||||
editor: editorId,
|
||||
});
|
||||
|
||||
await this.updateDocProperties(
|
||||
workspaceId,
|
||||
docId,
|
||||
{ updatedBy: editorId },
|
||||
editorId
|
||||
);
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates document metadata (currently title only).
|
||||
*
|
||||
* @param workspaceId - The workspace ID
|
||||
* @param docId - The document ID to update
|
||||
* @param meta - Metadata updates
|
||||
* @param editorId - Optional editor ID for tracking
|
||||
*/
|
||||
async updateDocMeta(
|
||||
workspaceId: string,
|
||||
docId: string,
|
||||
meta: { title?: string },
|
||||
editorId?: string
|
||||
): Promise<UpdateDocResult> {
|
||||
if (meta.title === undefined) {
|
||||
throw new Error('No metadata provided');
|
||||
}
|
||||
|
||||
this.logger.debug(`Updating doc meta ${docId} in workspace ${workspaceId}`);
|
||||
|
||||
const existingDoc = await this.storage.getDoc(workspaceId, docId);
|
||||
if (!existingDoc?.bin) {
|
||||
throw new NotFoundException(`Document ${docId} not found`);
|
||||
}
|
||||
|
||||
const rootDoc = await this.storage.getDoc(workspaceId, workspaceId);
|
||||
if (!rootDoc?.bin) {
|
||||
throw new NotFoundException(
|
||||
`Workspace ${workspaceId} not found or has no root document`
|
||||
);
|
||||
}
|
||||
|
||||
const existingBinary = Buffer.isBuffer(existingDoc.bin)
|
||||
? existingDoc.bin
|
||||
: Buffer.from(
|
||||
existingDoc.bin.buffer,
|
||||
existingDoc.bin.byteOffset,
|
||||
existingDoc.bin.byteLength
|
||||
);
|
||||
const rootDocBin = Buffer.isBuffer(rootDoc.bin)
|
||||
? rootDoc.bin
|
||||
: Buffer.from(
|
||||
rootDoc.bin.buffer,
|
||||
rootDoc.bin.byteOffset,
|
||||
rootDoc.bin.byteLength
|
||||
);
|
||||
|
||||
const titleUpdate = updateDocTitle(existingBinary, meta.title, docId);
|
||||
const rootMetaUpdate = updateRootDocMetaTitle(
|
||||
rootDocBin,
|
||||
docId,
|
||||
meta.title
|
||||
);
|
||||
|
||||
const rootTimestamp = await this.storage.pushDocUpdates(
|
||||
workspaceId,
|
||||
workspaceId,
|
||||
[rootMetaUpdate],
|
||||
editorId
|
||||
);
|
||||
this.emitDocUpdatesPushed({
|
||||
spaceId: workspaceId,
|
||||
docId: workspaceId,
|
||||
updates: [rootMetaUpdate],
|
||||
timestamp: rootTimestamp,
|
||||
editor: editorId,
|
||||
});
|
||||
|
||||
const docTimestamp = await this.storage.pushDocUpdates(
|
||||
workspaceId,
|
||||
docId,
|
||||
[titleUpdate],
|
||||
editorId
|
||||
);
|
||||
this.emitDocUpdatesPushed({
|
||||
spaceId: workspaceId,
|
||||
docId,
|
||||
updates: [titleUpdate],
|
||||
timestamp: docTimestamp,
|
||||
editor: editorId,
|
||||
});
|
||||
|
||||
await this.updateDocProperties(
|
||||
workspaceId,
|
||||
docId,
|
||||
{ updatedBy: editorId },
|
||||
editorId
|
||||
);
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
private emitDocUpdatesPushed(payload: {
|
||||
spaceId: string;
|
||||
docId: string;
|
||||
updates: Uint8Array[];
|
||||
timestamp: number;
|
||||
editor?: string;
|
||||
}) {
|
||||
this.event.emit('doc.updates.pushed', {
|
||||
spaceType: 'workspace',
|
||||
spaceId: payload.spaceId,
|
||||
docId: payload.docId,
|
||||
updates: payload.updates,
|
||||
timestamp: payload.timestamp,
|
||||
editor: payload.editor,
|
||||
});
|
||||
}
|
||||
|
||||
private async updateDocProperties(
|
||||
workspaceId: string,
|
||||
docId: string,
|
||||
props: { createdBy?: string; updatedBy?: string },
|
||||
editorId?: string
|
||||
) {
|
||||
if (!editorId) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
workspaceId === docId ||
|
||||
docId.startsWith('db$') ||
|
||||
docId.startsWith('userdata$')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if (!props.createdBy && !props.updatedBy) {
|
||||
return;
|
||||
}
|
||||
|
||||
const propertiesDocId = `db$${workspaceId}$docProperties`;
|
||||
const existingDoc = await this.storage.getDoc(workspaceId, propertiesDocId);
|
||||
const existingBinary = existingDoc?.bin
|
||||
? Buffer.isBuffer(existingDoc.bin)
|
||||
? existingDoc.bin
|
||||
: Buffer.from(
|
||||
existingDoc.bin.buffer,
|
||||
existingDoc.bin.byteOffset,
|
||||
existingDoc.bin.byteLength
|
||||
)
|
||||
: Buffer.alloc(0);
|
||||
|
||||
const update = updateDocProperties(
|
||||
existingBinary,
|
||||
propertiesDocId,
|
||||
docId,
|
||||
props.createdBy,
|
||||
props.updatedBy
|
||||
);
|
||||
if (this.storage.isEmptyBin(update)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const timestamp = await this.storage.pushDocUpdates(
|
||||
workspaceId,
|
||||
propertiesDocId,
|
||||
[update],
|
||||
editorId
|
||||
);
|
||||
this.emitDocUpdatesPushed({
|
||||
spaceId: workspaceId,
|
||||
docId: propertiesDocId,
|
||||
updates: [update],
|
||||
timestamp,
|
||||
editor: editorId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,9 +6,10 @@ import {
|
||||
OnGatewayDisconnect,
|
||||
SubscribeMessage as RawSubscribeMessage,
|
||||
WebSocketGateway,
|
||||
WebSocketServer,
|
||||
} from '@nestjs/websockets';
|
||||
import { ClsInterceptor } from 'nestjs-cls';
|
||||
import { Socket } from 'socket.io';
|
||||
import { type Server, Socket } from 'socket.io';
|
||||
|
||||
import {
|
||||
CallMetric,
|
||||
@@ -18,6 +19,7 @@ import {
|
||||
GatewayErrorWrapper,
|
||||
metrics,
|
||||
NotInSpace,
|
||||
OnEvent,
|
||||
SpaceAccessDenied,
|
||||
} from '../../base';
|
||||
import { Models } from '../../models';
|
||||
@@ -141,6 +143,9 @@ export class SpaceSyncGateway
|
||||
{
|
||||
protected logger = new Logger(SpaceSyncGateway.name);
|
||||
|
||||
@WebSocketServer()
|
||||
private readonly server!: Server;
|
||||
|
||||
private connectionCount = 0;
|
||||
|
||||
constructor(
|
||||
@@ -166,6 +171,46 @@ export class SpaceSyncGateway
|
||||
metrics.socketio.gauge('connections').record(this.connectionCount);
|
||||
}
|
||||
|
||||
@OnEvent('doc.updates.pushed')
|
||||
onDocUpdatesPushed({
|
||||
spaceType,
|
||||
spaceId,
|
||||
docId,
|
||||
updates,
|
||||
timestamp,
|
||||
editor,
|
||||
}: Events['doc.updates.pushed']) {
|
||||
if (!this.server || updates.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const encodedUpdates = updates.map(update =>
|
||||
Buffer.from(update).toString('base64')
|
||||
);
|
||||
|
||||
this.server
|
||||
.to(Room(spaceId, 'sync-019'))
|
||||
.emit('space:broadcast-doc-updates', {
|
||||
spaceType,
|
||||
spaceId,
|
||||
docId,
|
||||
updates: encodedUpdates,
|
||||
timestamp,
|
||||
});
|
||||
|
||||
const room = `${spaceType}:${Room(spaceId)}`;
|
||||
encodedUpdates.forEach(update => {
|
||||
this.server.to(room).emit('space:broadcast-doc-update', {
|
||||
spaceType,
|
||||
spaceId,
|
||||
docId,
|
||||
update,
|
||||
timestamp,
|
||||
editor,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
selectAdapter(client: Socket, spaceType: SpaceType): SyncSocketAdapter {
|
||||
let adapters: Record<SpaceType, SyncSocketAdapter> = (client as any)
|
||||
.affineSyncAdapters;
|
||||
|
||||
@@ -133,14 +133,24 @@ export class TelemetryService {
|
||||
};
|
||||
} catch (error) {
|
||||
const err = error as Error;
|
||||
this.logger.error('Telemetry forwarding failed', err);
|
||||
return {
|
||||
ok: false,
|
||||
error: {
|
||||
name: err?.name ?? 'TelemetryForwardingError',
|
||||
message: err?.message ?? 'Telemetry forwarding failed',
|
||||
},
|
||||
};
|
||||
if (env.dev) {
|
||||
this.logger.error('Telemetry forwarding failed', err);
|
||||
return {
|
||||
ok: false,
|
||||
error: {
|
||||
name: err?.name ?? 'TelemetryForwardingError',
|
||||
message: err?.message ?? 'Telemetry forwarding failed',
|
||||
},
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
ok: false,
|
||||
error: {
|
||||
name: 'TelemetryForwardingError',
|
||||
message: 'Telemetry forwarding failed',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+87
-8
@@ -1440,6 +1440,33 @@ Generated by [AVA](https://avajs.dev).
|
||||
␊
|
||||
␊
|
||||
␊
|
||||
␊
|
||||
[](Bookmark,https://affine.pro/)␊
|
||||
␊
|
||||
␊
|
||||
[](Bookmark,https://www.youtube.com/@affinepro)␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
|
||||
alt=""␊
|
||||
width="1302"␊
|
||||
height="728"␊
|
||||
/>␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
|
||||
alt=""␊
|
||||
width="1463"␊
|
||||
height="374"␊
|
||||
/>␊
|
||||
␊
|
||||
<img␊
|
||||
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
|
||||
alt=""␊
|
||||
width="862"␊
|
||||
height="1388"␊
|
||||
/>␊
|
||||
␊
|
||||
`,
|
||||
title: 'Write, Draw, Plan all at Once.',
|
||||
}
|
||||
@@ -1449,55 +1476,74 @@ Generated by [AVA](https://avajs.dev).
|
||||
> Snapshot 1
|
||||
|
||||
{
|
||||
markdown: `<!-- block_id=RX4CG2zsBk flavour=affine:note -->␊
|
||||
markdown: `<!-- block_id=FoPQcAyV_m flavour=affine:paragraph -->␊
|
||||
AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊
|
||||
␊
|
||||
<!-- block_id=oz48nn_zp8 flavour=affine:paragraph -->␊
|
||||
␊
|
||||
␊
|
||||
<!-- block_id=g8a-D9-jXS flavour=affine:paragraph -->␊
|
||||
# You own your data, with no compromises␊
|
||||
␊
|
||||
<!-- block_id=J8lHN1GR_5 flavour=affine:paragraph -->␊
|
||||
## Local-first & Real-time collaborative␊
|
||||
␊
|
||||
<!-- block_id=xCuWdM0VLz flavour=affine:paragraph -->␊
|
||||
We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊
|
||||
␊
|
||||
<!-- block_id=zElMi0tViK flavour=affine:paragraph -->␊
|
||||
AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊
|
||||
␊
|
||||
<!-- block_id=Z4rK0OF9Wk flavour=affine:paragraph -->␊
|
||||
␊
|
||||
␊
|
||||
<!-- block_id=S1mkc8zUoU flavour=affine:note -->␊
|
||||
<!-- block_id=DQ0Ryb-SpW flavour=affine:paragraph -->␊
|
||||
### Blocks that assemble your next docs, tasks kanban or whiteboard␊
|
||||
␊
|
||||
<!-- block_id=yGlBdshAqN flavour=affine:note -->␊
|
||||
<!-- block_id=HAZC3URZp_ flavour=affine:paragraph -->␊
|
||||
There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊
|
||||
␊
|
||||
<!-- block_id=0H87ypiuv8 flavour=affine:paragraph -->␊
|
||||
We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊
|
||||
␊
|
||||
<!-- block_id=Sp4G1KD0Wn flavour=affine:paragraph -->␊
|
||||
If you want to learn more about the product design of AFFiNE, here goes the concepts:␊
|
||||
␊
|
||||
<!-- block_id=RsUhDuEqXa flavour=affine:paragraph -->␊
|
||||
To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊
|
||||
␊
|
||||
<!-- block_id=6lDiuDqZGL flavour=affine:note -->␊
|
||||
<!-- block_id=Z2HibKzAr- flavour=affine:paragraph -->␊
|
||||
## A true canvas for blocks in any form␊
|
||||
␊
|
||||
<!-- block_id=UwvWddamzM flavour=affine:paragraph -->␊
|
||||
[Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊
|
||||
␊
|
||||
<!-- block_id=g9xKUjhJj1 flavour=affine:paragraph -->␊
|
||||
␊
|
||||
␊
|
||||
<!-- block_id=wDTn4YJ4pm flavour=affine:paragraph -->␊
|
||||
"We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊
|
||||
␊
|
||||
<!-- block_id=xFrrdiP3-V flavour=affine:list -->␊
|
||||
* Quip & Notion with their great concept of "everything is a block"␊
|
||||
<!-- block_id=Tp9xyN4Okl flavour=affine:list -->␊
|
||||
* Trello with their Kanban␊
|
||||
<!-- block_id=K_4hUzKZFQ flavour=affine:list -->␊
|
||||
* Airtable & Miro with their no-code programable datasheets␊
|
||||
<!-- block_id=QwMzON2s7x flavour=affine:list -->␊
|
||||
* Miro & Whimiscal with their edgeless visual whiteboard␊
|
||||
<!-- block_id=FFVmit6u1T flavour=affine:list -->␊
|
||||
* Remnote & Capacities with their object-based tag system␊
|
||||
<!-- block_id=cauvaHOQmh flavour=affine:note -->␊
|
||||
<!-- block_id=YqnG5O6AE6 flavour=affine:paragraph -->␊
|
||||
For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊
|
||||
␊
|
||||
<!-- block_id=sbDTmZMZcq flavour=affine:paragraph -->␊
|
||||
## Self Host␊
|
||||
␊
|
||||
<!-- block_id=QVvitesfbj flavour=affine:paragraph -->␊
|
||||
Self host AFFiNE␊
|
||||
␊
|
||||
<!-- block_id=2jwCeO8Yot flavour=affine:note -->␊
|
||||
<!-- block_id=U_GoHFD9At flavour=affine:database -->␊
|
||||
␊
|
||||
### Learning From␊
|
||||
||Title|Tag|␊
|
||||
@@ -1510,14 +1556,47 @@ Generated by [AVA](https://avajs.dev).
|
||||
|Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|<span data-affine-option data-value="HgHsKOUINZ" data-option-color="var(--affine-tag-blue)">Reference</span>|␊
|
||||
|Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊
|
||||
␊
|
||||
<!-- block_id=c9MF_JiRgx flavour=affine:note -->␊
|
||||
<!-- block_id=NyHXrMX3R1 flavour=affine:paragraph -->␊
|
||||
## Affine Development␊
|
||||
␊
|
||||
<!-- block_id=9-K49otbCv flavour=affine:paragraph -->␊
|
||||
For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊
|
||||
␊
|
||||
<!-- block_id=faFteK9eG- flavour=affine:paragraph -->␊
|
||||
␊
|
||||
␊
|
||||
<!-- block_id=6x7ALjUDjj flavour=affine:surface -->␊
|
||||
<!-- block_id=ECrtbvW6xx flavour=affine:bookmark -->␊
|
||||
␊
|
||||
[](Bookmark,https://affine.pro/)␊
|
||||
␊
|
||||
<!-- block_id=5W--UQLN11 flavour=affine:bookmark -->␊
|
||||
␊
|
||||
[](Bookmark,https://www.youtube.com/@affinepro)␊
|
||||
␊
|
||||
<!-- block_id=lcZphIJe63 flavour=affine:image -->␊
|
||||
<img␊
|
||||
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
|
||||
alt=""␊
|
||||
width="1302"␊
|
||||
height="728"␊
|
||||
/>␊
|
||||
␊
|
||||
<!-- block_id=JlgVJdWU12 flavour=affine:image -->␊
|
||||
<img␊
|
||||
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
|
||||
alt=""␊
|
||||
width="1463"␊
|
||||
height="374"␊
|
||||
/>␊
|
||||
␊
|
||||
<!-- block_id=lht7AqBqnF flavour=affine:image -->␊
|
||||
<img␊
|
||||
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
|
||||
alt=""␊
|
||||
width="862"␊
|
||||
height="1388"␊
|
||||
/>␊
|
||||
␊
|
||||
`,
|
||||
title: 'Write, Draw, Plan all at Once.',
|
||||
}
|
||||
|
||||
Binary file not shown.
@@ -51,6 +51,9 @@ export const AFFINE_PRO_LICENSE_AES_KEY =
|
||||
serverNativeModule.AFFINE_PRO_LICENSE_AES_KEY;
|
||||
|
||||
// MCP write tools exports
|
||||
export const markdownToDocBinary = serverNativeModule.markdownToDocBinary;
|
||||
export const createDocWithMarkdown = serverNativeModule.createDocWithMarkdown;
|
||||
export const updateDocWithMarkdown = serverNativeModule.updateDocWithMarkdown;
|
||||
export const addDocToRootDoc = serverNativeModule.addDocToRootDoc;
|
||||
export const updateDocTitle = serverNativeModule.updateDocTitle;
|
||||
export const updateDocProperties = serverNativeModule.updateDocProperties;
|
||||
export const updateRootDocMetaTitle = serverNativeModule.updateRootDocMetaTitle;
|
||||
|
||||
@@ -166,146 +166,217 @@ export class WorkspaceMcpProvider {
|
||||
}
|
||||
);
|
||||
|
||||
// Write tools - create and update documents
|
||||
server.registerTool(
|
||||
'create_document',
|
||||
{
|
||||
title: 'Create Document',
|
||||
description:
|
||||
'Create a new document in the workspace with the given title and markdown content. Returns the ID of the created document.',
|
||||
inputSchema: z.object({
|
||||
title: z.string().min(1).describe('The title of the new document'),
|
||||
content: z
|
||||
.string()
|
||||
.describe(
|
||||
'The markdown content for the document body (should NOT include a title H1 - the title parameter will be used)'
|
||||
),
|
||||
}),
|
||||
},
|
||||
async ({ title, content }) => {
|
||||
try {
|
||||
// Check if user can create docs in this workspace
|
||||
await this.ac
|
||||
if (env.dev || env.namespaces.canary) {
|
||||
// Write tools - create and update documents
|
||||
server.registerTool(
|
||||
'create_document',
|
||||
{
|
||||
title: 'Create Document',
|
||||
description:
|
||||
'Create a new document in the workspace with the given title and markdown content. Returns the ID of the created document. This tool not support insert or update database block and image yet.',
|
||||
inputSchema: z.object({
|
||||
title: z.string().min(1).describe('The title of the new document'),
|
||||
content: z
|
||||
.string()
|
||||
.describe('The markdown content for the document body'),
|
||||
}),
|
||||
},
|
||||
async ({ title, content }) => {
|
||||
try {
|
||||
// Check if user can create docs in this workspace
|
||||
await this.ac
|
||||
.user(userId)
|
||||
.workspace(workspaceId)
|
||||
.assert('Workspace.CreateDoc');
|
||||
|
||||
// Sanitize title by removing newlines and trimming
|
||||
const sanitizedTitle = title.replace(/[\r\n]+/g, ' ').trim();
|
||||
if (!sanitizedTitle) {
|
||||
throw new Error('Title cannot be empty');
|
||||
}
|
||||
|
||||
// Strip any leading H1 from content to prevent duplicates
|
||||
// Per CommonMark spec, ATX headings allow only 0-3 spaces before the #
|
||||
// Handles: "# Title", " # Title", "# Title #"
|
||||
const strippedContent = content.replace(
|
||||
/^[ \t]{0,3}#\s+[^\n]*#*\s*\n*/,
|
||||
''
|
||||
);
|
||||
|
||||
// Create the document
|
||||
const result = await this.writer.createDoc(
|
||||
workspaceId,
|
||||
sanitizedTitle,
|
||||
strippedContent,
|
||||
userId
|
||||
);
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: JSON.stringify({
|
||||
success: true,
|
||||
docId: result.docId,
|
||||
message: `Document "${title}" created successfully`,
|
||||
}),
|
||||
},
|
||||
],
|
||||
} as const;
|
||||
} catch (error) {
|
||||
return {
|
||||
isError: true,
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Failed to create document: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
'update_document',
|
||||
{
|
||||
title: 'Update Document',
|
||||
description:
|
||||
'Update an existing document with new markdown content (body only). Uses structural diffing to apply minimal changes, preserving document history and enabling real-time collaboration. This does NOT update the document title. This tool not support insert or update database block and image yet.',
|
||||
inputSchema: z.object({
|
||||
docId: z.string().describe('The ID of the document to update'),
|
||||
content: z
|
||||
.string()
|
||||
.describe(
|
||||
'The complete new markdown content for the document body (do NOT include a title H1)'
|
||||
),
|
||||
}),
|
||||
},
|
||||
async ({ docId, content }) => {
|
||||
const notFoundError: CallToolResult = {
|
||||
isError: true,
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Doc with id ${docId} not found.`,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// Use can() instead of assert() to avoid leaking doc existence info
|
||||
const accessible = await this.ac
|
||||
.user(userId)
|
||||
.workspace(workspaceId)
|
||||
.assert('Workspace.CreateDoc');
|
||||
.doc(docId)
|
||||
.can('Doc.Update');
|
||||
|
||||
// Combine title and content into markdown
|
||||
// Sanitize title by removing newlines and trimming
|
||||
const sanitizedTitle = title.replace(/[\r\n]+/g, ' ').trim();
|
||||
if (!sanitizedTitle) {
|
||||
throw new Error('Title cannot be empty');
|
||||
if (!accessible) {
|
||||
return notFoundError;
|
||||
}
|
||||
|
||||
// Strip any leading H1 from content to prevent duplicates
|
||||
// Per CommonMark spec, ATX headings allow only 0-3 spaces before the #
|
||||
// Handles: "# Title", " # Title", "# Title #"
|
||||
const strippedContent = content.replace(
|
||||
/^[ \t]{0,3}#\s+[^\n]*#*\s*\n*/,
|
||||
''
|
||||
);
|
||||
try {
|
||||
// Update the document
|
||||
await this.writer.updateDoc(workspaceId, docId, content, userId);
|
||||
|
||||
const markdown = `# ${sanitizedTitle}\n\n${strippedContent}`;
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: JSON.stringify({
|
||||
success: true,
|
||||
docId,
|
||||
message: `Document updated successfully`,
|
||||
}),
|
||||
},
|
||||
],
|
||||
} as const;
|
||||
} catch (error) {
|
||||
return {
|
||||
isError: true,
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Failed to update document: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// Create the document
|
||||
const result = await this.writer.createDoc(
|
||||
workspaceId,
|
||||
markdown,
|
||||
userId
|
||||
);
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: JSON.stringify({
|
||||
success: true,
|
||||
docId: result.docId,
|
||||
message: `Document "${title}" created successfully`,
|
||||
}),
|
||||
},
|
||||
],
|
||||
} as const;
|
||||
} catch (error) {
|
||||
return {
|
||||
server.registerTool(
|
||||
'update_document_meta',
|
||||
{
|
||||
title: 'Update Document Metadata',
|
||||
description: 'Update document metadata (currently title only).',
|
||||
inputSchema: z.object({
|
||||
docId: z.string().describe('The ID of the document to update'),
|
||||
title: z.string().min(1).describe('The new document title'),
|
||||
}),
|
||||
},
|
||||
async ({ docId, title }) => {
|
||||
const notFoundError: CallToolResult = {
|
||||
isError: true,
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Failed to create document: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
text: `Doc with id ${docId} not found.`,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
'update_document',
|
||||
{
|
||||
title: 'Update Document',
|
||||
description:
|
||||
'Update an existing document with new markdown content. Uses structural diffing to apply minimal changes, preserving document history and enabling real-time collaboration.',
|
||||
inputSchema: z.object({
|
||||
docId: z.string().describe('The ID of the document to update'),
|
||||
content: z
|
||||
.string()
|
||||
.describe(
|
||||
'The complete new markdown content for the document (including title as H1)'
|
||||
),
|
||||
}),
|
||||
},
|
||||
async ({ docId, content }) => {
|
||||
const notFoundError: CallToolResult = {
|
||||
isError: true,
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Doc with id ${docId} not found.`,
|
||||
},
|
||||
],
|
||||
};
|
||||
// Use can() instead of assert() to avoid leaking doc existence info
|
||||
const accessible = await this.ac
|
||||
.user(userId)
|
||||
.workspace(workspaceId)
|
||||
.doc(docId)
|
||||
.can('Doc.Update');
|
||||
|
||||
// Use can() instead of assert() to avoid leaking doc existence info
|
||||
const accessible = await this.ac
|
||||
.user(userId)
|
||||
.workspace(workspaceId)
|
||||
.doc(docId)
|
||||
.can('Doc.Update');
|
||||
if (!accessible) {
|
||||
return notFoundError;
|
||||
}
|
||||
|
||||
if (!accessible) {
|
||||
return notFoundError;
|
||||
}
|
||||
try {
|
||||
const sanitizedTitle = title.replace(/[\r\n]+/g, ' ').trim();
|
||||
if (!sanitizedTitle) {
|
||||
throw new Error('Title cannot be empty');
|
||||
}
|
||||
|
||||
try {
|
||||
// Update the document
|
||||
await this.writer.updateDoc(workspaceId, docId, content, userId);
|
||||
|
||||
return {
|
||||
content: [
|
||||
await this.writer.updateDocMeta(
|
||||
workspaceId,
|
||||
docId,
|
||||
{
|
||||
type: 'text',
|
||||
text: JSON.stringify({
|
||||
success: true,
|
||||
docId,
|
||||
message: `Document updated successfully`,
|
||||
}),
|
||||
title: sanitizedTitle,
|
||||
},
|
||||
],
|
||||
} as const;
|
||||
} catch (error) {
|
||||
return {
|
||||
isError: true,
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Failed to update document: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
},
|
||||
],
|
||||
};
|
||||
userId
|
||||
);
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: JSON.stringify({
|
||||
success: true,
|
||||
docId,
|
||||
message: `Document title updated successfully`,
|
||||
}),
|
||||
},
|
||||
],
|
||||
} as const;
|
||||
} catch (error) {
|
||||
return {
|
||||
isError: true,
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Failed to update document metadata: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
);
|
||||
}
|
||||
|
||||
return server;
|
||||
}
|
||||
|
||||
@@ -127,6 +127,7 @@ export class ChatPrompt {
|
||||
selectedMarkdown,
|
||||
selectedSnapshot,
|
||||
html,
|
||||
currentDocId,
|
||||
} = params;
|
||||
return {
|
||||
'affine::date': new Date().toLocaleDateString(),
|
||||
@@ -135,6 +136,8 @@ export class ChatPrompt {
|
||||
'affine::hasDocsRef': Array.isArray(docs) && docs.length > 0,
|
||||
'affine::hasFilesRef': Array.isArray(files) && files.length > 0,
|
||||
'affine::hasSelected': !!selectedMarkdown || !!selectedSnapshot || !!html,
|
||||
'affine::hasCurrentDoc':
|
||||
typeof currentDocId === 'string' && currentDocId.trim().length > 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1950,6 +1950,13 @@ User's preferred language is {{affine::language}}.
|
||||
User's timezone is {{affine::timezone}}.
|
||||
</real_world_info>
|
||||
|
||||
{{#affine::hasCurrentDoc}}
|
||||
<current_document_context>
|
||||
The user is chatting within the current document: {{currentDocId}}.
|
||||
If the user's request relates to this document, call the doc_read tool with docId {{currentDocId}} to read it before answering.
|
||||
</current_document_context>
|
||||
{{/affine::hasCurrentDoc}}
|
||||
|
||||
<content_analysis>
|
||||
- If documents are provided, analyze all documents based on the user's query
|
||||
- Identify key information relevant to the user's specific request
|
||||
@@ -2086,7 +2093,10 @@ Below is the user's query. Please respond in the user's preferred language witho
|
||||
config: {
|
||||
tools: [
|
||||
'docRead',
|
||||
'sectionEdit',
|
||||
'docCreate',
|
||||
'docUpdate',
|
||||
'docUpdateMeta',
|
||||
// 'sectionEdit',
|
||||
'docKeywordSearch',
|
||||
'docSemanticSearch',
|
||||
'webSearch',
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
CopilotProviderNotSupported,
|
||||
OnEvent,
|
||||
} from '../../../base';
|
||||
import { DocReader } from '../../../core/doc';
|
||||
import { DocReader, DocWriter } from '../../../core/doc';
|
||||
import { AccessController } from '../../../core/permission';
|
||||
import { Models } from '../../../models';
|
||||
import { IndexerService } from '../../indexer';
|
||||
@@ -19,16 +19,22 @@ import {
|
||||
buildBlobContentGetter,
|
||||
buildContentGetter,
|
||||
buildDocContentGetter,
|
||||
buildDocCreateHandler,
|
||||
buildDocKeywordSearchGetter,
|
||||
buildDocSearchGetter,
|
||||
buildDocUpdateHandler,
|
||||
buildDocUpdateMetaHandler,
|
||||
createBlobReadTool,
|
||||
createCodeArtifactTool,
|
||||
createConversationSummaryTool,
|
||||
createDocComposeTool,
|
||||
createDocCreateTool,
|
||||
createDocEditTool,
|
||||
createDocKeywordSearchTool,
|
||||
createDocReadTool,
|
||||
createDocSemanticSearchTool,
|
||||
createDocUpdateMetaTool,
|
||||
createDocUpdateTool,
|
||||
createExaCrawlTool,
|
||||
createExaSearchTool,
|
||||
createSectionEditTool,
|
||||
@@ -163,6 +169,7 @@ export abstract class CopilotProvider<C = any> {
|
||||
strict: false,
|
||||
});
|
||||
const docReader = this.moduleRef.get(DocReader, { strict: false });
|
||||
const docWriter = this.moduleRef.get(DocWriter, { strict: false });
|
||||
const models = this.moduleRef.get(Models, { strict: false });
|
||||
const prompt = this.moduleRef.get(PromptService, {
|
||||
strict: false,
|
||||
@@ -177,6 +184,12 @@ export abstract class CopilotProvider<C = any> {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
!(env.dev || env.namespaces.canary) &&
|
||||
['docCreate', 'docUpdate', 'docUpdateMeta'].includes(tool)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
switch (tool) {
|
||||
case 'blobRead': {
|
||||
const docContext = options.session
|
||||
@@ -244,6 +257,27 @@ export abstract class CopilotProvider<C = any> {
|
||||
tools.doc_read = createDocReadTool(getDoc.bind(null, options));
|
||||
break;
|
||||
}
|
||||
case 'docCreate': {
|
||||
const createDoc = buildDocCreateHandler(ac, docWriter);
|
||||
tools.doc_create = createDocCreateTool(
|
||||
createDoc.bind(null, options)
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'docUpdate': {
|
||||
const updateDoc = buildDocUpdateHandler(ac, docWriter);
|
||||
tools.doc_update = createDocUpdateTool(
|
||||
updateDoc.bind(null, options)
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'docUpdateMeta': {
|
||||
const updateDocMeta = buildDocUpdateMetaHandler(ac, docWriter);
|
||||
tools.doc_update_meta = createDocUpdateMetaTool(
|
||||
updateDocMeta.bind(null, options)
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'webSearch': {
|
||||
tools.web_search_exa = createExaSearchTool(this.AFFiNEConfig);
|
||||
tools.web_crawl_exa = createExaCrawlTool(this.AFFiNEConfig);
|
||||
|
||||
@@ -66,6 +66,9 @@ export const PromptToolsSchema = z
|
||||
'docEdit',
|
||||
// work with indexer
|
||||
'docRead',
|
||||
'docCreate',
|
||||
'docUpdate',
|
||||
'docUpdateMeta',
|
||||
'docKeywordSearch',
|
||||
// work with embeddings
|
||||
'docSemanticSearch',
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { DocWriter } from '../../../core/doc';
|
||||
import { AccessController } from '../../../core/permission';
|
||||
import type { CopilotChatOptions } from '../providers';
|
||||
import { toolError } from './error';
|
||||
|
||||
const logger = new Logger('DocWriteTool');
|
||||
|
||||
const stripLeadingH1 = (content: string) =>
|
||||
content.replace(/^[ \t]{0,3}#\s+[^\n]*#*\s*\n*/, '');
|
||||
|
||||
const sanitizeTitle = (title: string) => title.replace(/[\r\n]+/g, ' ').trim();
|
||||
|
||||
export const buildDocCreateHandler = (
|
||||
ac: AccessController,
|
||||
writer: DocWriter
|
||||
) => {
|
||||
return async (
|
||||
options: CopilotChatOptions,
|
||||
title: string,
|
||||
content: string
|
||||
) => {
|
||||
if (!options?.user || !options.workspace) {
|
||||
return toolError(
|
||||
'Doc Create Failed',
|
||||
'Missing user or workspace context'
|
||||
);
|
||||
}
|
||||
|
||||
await ac
|
||||
.user(options.user)
|
||||
.workspace(options.workspace)
|
||||
.assert('Workspace.CreateDoc');
|
||||
|
||||
const sanitizedTitle = sanitizeTitle(title);
|
||||
if (!sanitizedTitle) {
|
||||
return toolError('Doc Create Failed', 'Title cannot be empty');
|
||||
}
|
||||
|
||||
const strippedContent = stripLeadingH1(content);
|
||||
const result = await writer.createDoc(
|
||||
options.workspace,
|
||||
sanitizedTitle,
|
||||
strippedContent,
|
||||
options.user
|
||||
);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
docId: result.docId,
|
||||
message: `Document "${sanitizedTitle}" created successfully`,
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
export const buildDocUpdateHandler = (
|
||||
ac: AccessController,
|
||||
writer: DocWriter
|
||||
) => {
|
||||
return async (
|
||||
options: CopilotChatOptions,
|
||||
docId: string,
|
||||
content: string
|
||||
) => {
|
||||
const notFound = toolError(
|
||||
'Doc Update Failed',
|
||||
`Doc with id ${docId} not found.`
|
||||
);
|
||||
|
||||
if (!options?.user || !options.workspace) {
|
||||
return notFound;
|
||||
}
|
||||
|
||||
const canAccess = await ac
|
||||
.user(options.user)
|
||||
.workspace(options.workspace)
|
||||
.doc(docId)
|
||||
.can('Doc.Update');
|
||||
|
||||
if (!canAccess) {
|
||||
return notFound;
|
||||
}
|
||||
|
||||
await writer.updateDoc(options.workspace, docId, content, options.user);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
docId,
|
||||
message: 'Document updated successfully',
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
export const buildDocUpdateMetaHandler = (
|
||||
ac: AccessController,
|
||||
writer: DocWriter
|
||||
) => {
|
||||
return async (options: CopilotChatOptions, docId: string, title: string) => {
|
||||
const notFound = toolError(
|
||||
'Doc Meta Update Failed',
|
||||
`Doc with id ${docId} not found.`
|
||||
);
|
||||
|
||||
if (!options?.user || !options.workspace) {
|
||||
return notFound;
|
||||
}
|
||||
|
||||
const canAccess = await ac
|
||||
.user(options.user)
|
||||
.workspace(options.workspace)
|
||||
.doc(docId)
|
||||
.can('Doc.Update');
|
||||
|
||||
if (!canAccess) {
|
||||
return notFound;
|
||||
}
|
||||
|
||||
const sanitizedTitle = sanitizeTitle(title);
|
||||
if (!sanitizedTitle) {
|
||||
return toolError('Doc Meta Update Failed', 'Title cannot be empty');
|
||||
}
|
||||
|
||||
await writer.updateDocMeta(
|
||||
options.workspace,
|
||||
docId,
|
||||
{ title: sanitizedTitle },
|
||||
options.user
|
||||
);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
docId,
|
||||
message: 'Document title updated successfully',
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
export const createDocCreateTool = (
|
||||
createDoc: (title: string, content: string) => Promise<object>
|
||||
) => {
|
||||
return tool({
|
||||
description:
|
||||
'Create a new document in the workspace with the given title and markdown content. Returns the ID of the created document. This tool not support insert or update database block and image yet.',
|
||||
inputSchema: z.object({
|
||||
title: z.string().min(1).describe('The title of the new document'),
|
||||
content: z
|
||||
.string()
|
||||
.describe('The markdown content for the document body'),
|
||||
}),
|
||||
execute: async ({ title, content }) => {
|
||||
try {
|
||||
return await createDoc(title, content);
|
||||
} catch (err: any) {
|
||||
logger.error(`Failed to create document: ${title}`, err);
|
||||
return toolError('Doc Create Failed', err.message);
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
export const createDocUpdateTool = (
|
||||
updateDoc: (docId: string, content: string) => Promise<object>
|
||||
) => {
|
||||
return tool({
|
||||
description:
|
||||
'Update an existing document with new markdown content (body only). Uses structural diffing to apply minimal changes. This does NOT update the document title. This tool not support insert or update database block and image yet.',
|
||||
inputSchema: z.object({
|
||||
doc_id: z.string().describe('The ID of the document to update'),
|
||||
content: z
|
||||
.string()
|
||||
.describe(
|
||||
'The complete new markdown content for the document body (do NOT include a title H1)'
|
||||
),
|
||||
}),
|
||||
execute: async ({ doc_id, content }) => {
|
||||
try {
|
||||
return await updateDoc(doc_id, content);
|
||||
} catch (err: any) {
|
||||
logger.error(`Failed to update document: ${doc_id}`, err);
|
||||
return toolError('Doc Update Failed', err.message);
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
export const createDocUpdateMetaTool = (
|
||||
updateDocMeta: (docId: string, title: string) => Promise<object>
|
||||
) => {
|
||||
return tool({
|
||||
description: 'Update document metadata (currently title only).',
|
||||
inputSchema: z.object({
|
||||
doc_id: z.string().describe('The ID of the document to update'),
|
||||
title: z.string().min(1).describe('The new document title'),
|
||||
}),
|
||||
execute: async ({ doc_id, title }) => {
|
||||
try {
|
||||
return await updateDocMeta(doc_id, title);
|
||||
} catch (err: any) {
|
||||
logger.error(`Failed to update document meta: ${doc_id}`, err);
|
||||
return toolError('Doc Meta Update Failed', err.message);
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
@@ -8,6 +8,11 @@ import { createDocEditTool } from './doc-edit';
|
||||
import { createDocKeywordSearchTool } from './doc-keyword-search';
|
||||
import { createDocReadTool } from './doc-read';
|
||||
import { createDocSemanticSearchTool } from './doc-semantic-search';
|
||||
import {
|
||||
createDocCreateTool,
|
||||
createDocUpdateMetaTool,
|
||||
createDocUpdateTool,
|
||||
} from './doc-write';
|
||||
import { createExaCrawlTool } from './exa-crawl';
|
||||
import { createExaSearchTool } from './exa-search';
|
||||
import { createSectionEditTool } from './section-edit';
|
||||
@@ -20,6 +25,9 @@ export interface CustomAITools extends ToolSet {
|
||||
doc_semantic_search: ReturnType<typeof createDocSemanticSearchTool>;
|
||||
doc_keyword_search: ReturnType<typeof createDocKeywordSearchTool>;
|
||||
doc_read: ReturnType<typeof createDocReadTool>;
|
||||
doc_create: ReturnType<typeof createDocCreateTool>;
|
||||
doc_update: ReturnType<typeof createDocUpdateTool>;
|
||||
doc_update_meta: ReturnType<typeof createDocUpdateMetaTool>;
|
||||
doc_compose: ReturnType<typeof createDocComposeTool>;
|
||||
section_edit: ReturnType<typeof createSectionEditTool>;
|
||||
web_search_exa: ReturnType<typeof createExaSearchTool>;
|
||||
@@ -34,6 +42,7 @@ export * from './doc-edit';
|
||||
export * from './doc-keyword-search';
|
||||
export * from './doc-read';
|
||||
export * from './doc-semantic-search';
|
||||
export * from './doc-write';
|
||||
export * from './error';
|
||||
export * from './exa-crawl';
|
||||
export * from './exa-search';
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,601 @@
|
||||
use y_octo::{Any, Map, TextAttributes, TextDeltaOp, TextInsert};
|
||||
|
||||
use super::{
|
||||
ParseError,
|
||||
blocksuite::get_string,
|
||||
schema::{
|
||||
PROP_CAPTION, PROP_CHECKED, PROP_COLUMN_ID_SUFFIX, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER,
|
||||
PROP_ORDER_SUFFIX, PROP_ROW_ID_SUFFIX, PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL,
|
||||
PROP_VIDEO_ID, PROP_WIDTH, SYS_FLAVOUR, table_cell_text_key,
|
||||
},
|
||||
table::{MarkdownTableOptions, render_markdown_table},
|
||||
value::{value_to_f64, value_to_string},
|
||||
};
|
||||
|
||||
/// Block flavours used in AFFiNE documents.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BlockFlavour {
|
||||
Paragraph,
|
||||
List,
|
||||
Code,
|
||||
Divider,
|
||||
Image,
|
||||
Table,
|
||||
Bookmark,
|
||||
EmbedYoutube,
|
||||
EmbedIframe,
|
||||
Callout,
|
||||
}
|
||||
|
||||
impl BlockFlavour {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
BlockFlavour::Paragraph => "affine:paragraph",
|
||||
BlockFlavour::List => "affine:list",
|
||||
BlockFlavour::Code => "affine:code",
|
||||
BlockFlavour::Divider => "affine:divider",
|
||||
BlockFlavour::Image => "affine:image",
|
||||
BlockFlavour::Table => "affine:table",
|
||||
BlockFlavour::Bookmark => "affine:bookmark",
|
||||
BlockFlavour::EmbedYoutube => "affine:embed-youtube",
|
||||
BlockFlavour::EmbedIframe => "affine:embed-iframe",
|
||||
BlockFlavour::Callout => "affine:callout",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"affine:paragraph" => Some(BlockFlavour::Paragraph),
|
||||
"affine:list" => Some(BlockFlavour::List),
|
||||
"affine:code" => Some(BlockFlavour::Code),
|
||||
"affine:divider" => Some(BlockFlavour::Divider),
|
||||
"affine:image" => Some(BlockFlavour::Image),
|
||||
"affine:table" => Some(BlockFlavour::Table),
|
||||
"affine:bookmark" => Some(BlockFlavour::Bookmark),
|
||||
"affine:embed-youtube" => Some(BlockFlavour::EmbedYoutube),
|
||||
"affine:embed-iframe" => Some(BlockFlavour::EmbedIframe),
|
||||
"affine:callout" => Some(BlockFlavour::Callout),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Block types for paragraphs and lists.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum BlockType {
|
||||
// Paragraph types
|
||||
Text,
|
||||
H1,
|
||||
H2,
|
||||
H3,
|
||||
H4,
|
||||
H5,
|
||||
H6,
|
||||
Quote,
|
||||
// List types
|
||||
Bulleted,
|
||||
Numbered,
|
||||
Todo,
|
||||
// Preserve unknown types when loading from ydoc.
|
||||
Unknown(String),
|
||||
}
|
||||
|
||||
impl BlockType {
|
||||
pub fn as_str(&self) -> &str {
|
||||
match self {
|
||||
BlockType::Text => "text",
|
||||
BlockType::H1 => "h1",
|
||||
BlockType::H2 => "h2",
|
||||
BlockType::H3 => "h3",
|
||||
BlockType::H4 => "h4",
|
||||
BlockType::H5 => "h5",
|
||||
BlockType::H6 => "h6",
|
||||
BlockType::Quote => "quote",
|
||||
BlockType::Bulleted => "bulleted",
|
||||
BlockType::Numbered => "numbered",
|
||||
BlockType::Todo => "todo",
|
||||
BlockType::Unknown(value) => value.as_str(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"text" => Some(BlockType::Text),
|
||||
"h1" => Some(BlockType::H1),
|
||||
"h2" => Some(BlockType::H2),
|
||||
"h3" => Some(BlockType::H3),
|
||||
"h4" => Some(BlockType::H4),
|
||||
"h5" => Some(BlockType::H5),
|
||||
"h6" => Some(BlockType::H6),
|
||||
"quote" => Some(BlockType::Quote),
|
||||
"bulleted" => Some(BlockType::Bulleted),
|
||||
"numbered" => Some(BlockType::Numbered),
|
||||
"todo" => Some(BlockType::Todo),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_str_lossy(value: String) -> Self {
|
||||
Self::from_str(&value).unwrap_or(BlockType::Unknown(value))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct ImageSpec {
|
||||
pub(super) source_id: String,
|
||||
pub(super) caption: Option<String>,
|
||||
pub(super) width: Option<f64>,
|
||||
pub(super) height: Option<f64>,
|
||||
}
|
||||
|
||||
impl ImageSpec {
|
||||
pub(super) fn render_markdown(&self) -> String {
|
||||
let blob_url = format!("blob://{}", self.source_id);
|
||||
let caption = self.caption.as_deref().unwrap_or("");
|
||||
|
||||
if self.width.is_some() || self.height.is_some() || !caption.is_empty() {
|
||||
let width_text = self
|
||||
.width
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "auto".into());
|
||||
let height_text = self
|
||||
.height
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "auto".into());
|
||||
return format!(
|
||||
"<img\n src=\"{blob_url}\"\n alt=\"{caption}\"\n width=\"{width_text}\"\n height=\"{height_text}\"\n/>\n\n"
|
||||
);
|
||||
}
|
||||
|
||||
let alt = if caption.is_empty() {
|
||||
self.source_id.as_str()
|
||||
} else {
|
||||
caption
|
||||
};
|
||||
format!("\n\n\n")
|
||||
}
|
||||
|
||||
pub(super) fn from_block_map(block: &Map) -> Self {
|
||||
let source_id = get_string(block, PROP_SOURCE_ID).unwrap_or_default();
|
||||
let caption = get_string(block, PROP_CAPTION);
|
||||
let width = block.get(PROP_WIDTH).and_then(value_to_f64);
|
||||
let height = block.get(PROP_HEIGHT).and_then(value_to_f64);
|
||||
ImageSpec {
|
||||
source_id,
|
||||
caption,
|
||||
width,
|
||||
height,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn normalize_source(src: &str) -> Result<String, ParseError> {
|
||||
let trimmed = src.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(ParseError::ParserError("invalid_image_source".into()));
|
||||
}
|
||||
if let Some(rest) = trimmed.strip_prefix("blob://") {
|
||||
if rest.is_empty() {
|
||||
return Err(ParseError::ParserError("invalid_image_source".into()));
|
||||
}
|
||||
return Ok(rest.to_string());
|
||||
}
|
||||
if !trimmed.contains('/') && !trimmed.contains(':') {
|
||||
return Ok(trimmed.to_string());
|
||||
}
|
||||
if let Some(pos) = trimmed.rfind("/blobs/") {
|
||||
let id = &trimmed[pos + "/blobs/".len()..];
|
||||
let id = id.split(['?', '#']).next().unwrap_or("");
|
||||
if !id.is_empty() {
|
||||
return Ok(id.to_string());
|
||||
}
|
||||
}
|
||||
Err(ParseError::ParserError("unsupported_image_source".into()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct TableSpec {
|
||||
pub(super) rows: Vec<Vec<String>>,
|
||||
}
|
||||
|
||||
impl TableSpec {
|
||||
pub(super) fn from_block_map(block: &Map) -> Self {
|
||||
let mut row_entries: Vec<(String, String)> = Vec::new();
|
||||
let mut column_entries: Vec<(String, String)> = Vec::new();
|
||||
|
||||
for key in block.keys() {
|
||||
if key.starts_with(PROP_ROWS_PREFIX)
|
||||
&& key.ends_with(PROP_ROW_ID_SUFFIX)
|
||||
&& let Some(row_id) = block.get(key).and_then(|value| value_to_string(&value))
|
||||
{
|
||||
let base = key.trim_end_matches(PROP_ROW_ID_SUFFIX);
|
||||
let order_key = format!("{base}{PROP_ORDER_SUFFIX}");
|
||||
let order = block
|
||||
.get(&order_key)
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
row_entries.push((order, row_id));
|
||||
}
|
||||
if key.starts_with(PROP_COLUMNS_PREFIX)
|
||||
&& key.ends_with(PROP_COLUMN_ID_SUFFIX)
|
||||
&& let Some(column_id) = block.get(key).and_then(|value| value_to_string(&value))
|
||||
{
|
||||
let base = key.trim_end_matches(PROP_COLUMN_ID_SUFFIX);
|
||||
let order_key = format!("{base}{PROP_ORDER_SUFFIX}");
|
||||
let order = block
|
||||
.get(&order_key)
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
column_entries.push((order, column_id));
|
||||
}
|
||||
}
|
||||
|
||||
row_entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
column_entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for (_, row_id) in row_entries {
|
||||
let mut row = Vec::new();
|
||||
for (_, column_id) in &column_entries {
|
||||
let cell_key = table_cell_text_key(&row_id, column_id);
|
||||
let cell_text = block
|
||||
.get(&cell_key)
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
row.push(cell_text);
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
Self { rows }
|
||||
}
|
||||
|
||||
pub(super) fn render_markdown(&self) -> Option<String> {
|
||||
let options = MarkdownTableOptions::new(false, "<br />", true);
|
||||
render_markdown_table(&self.rows, options)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct BookmarkSpec {
|
||||
pub(super) url: String,
|
||||
pub(super) caption: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct EmbedYoutubeSpec {
|
||||
pub(super) video_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct EmbedIframeSpec {
|
||||
pub(super) url: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct BlockSpec {
|
||||
pub(super) flavour: BlockFlavour,
|
||||
pub(super) block_type: Option<BlockType>,
|
||||
pub(super) text: Vec<TextDeltaOp>,
|
||||
pub(super) checked: Option<bool>,
|
||||
pub(super) language: Option<String>,
|
||||
pub(super) order: Option<i64>,
|
||||
pub(super) image: Option<ImageSpec>,
|
||||
pub(super) table: Option<TableSpec>,
|
||||
pub(super) bookmark: Option<BookmarkSpec>,
|
||||
pub(super) embed_youtube: Option<EmbedYoutubeSpec>,
|
||||
pub(super) embed_iframe: Option<EmbedIframeSpec>,
|
||||
}
|
||||
|
||||
impl BlockSpec {
|
||||
pub(super) fn is_exact(&self, other: &BlockSpec) -> bool {
|
||||
self.flavour == other.flavour
|
||||
&& self.block_type == other.block_type
|
||||
&& self.checked == other.checked
|
||||
&& self.language == other.language
|
||||
&& self.image == other.image
|
||||
&& self.table == other.table
|
||||
&& self.bookmark == other.bookmark
|
||||
&& self.embed_youtube == other.embed_youtube
|
||||
&& self.embed_iframe == other.embed_iframe
|
||||
&& text_delta_eq(&self.text, &other.text)
|
||||
}
|
||||
|
||||
pub(super) fn is_similar(&self, other: &BlockSpec) -> bool {
|
||||
self.flavour == other.flavour && self.block_type == other.block_type
|
||||
}
|
||||
|
||||
pub(super) fn block_type_str(&self) -> Option<&str> {
|
||||
self.block_type.as_ref().map(BlockType::as_str)
|
||||
}
|
||||
|
||||
pub(super) fn from_block_map(block: &Map) -> Result<Self, ParseError> {
|
||||
let flavour_value = get_string(block, SYS_FLAVOUR).unwrap_or_default();
|
||||
let Some(flavour) = BlockFlavour::from_str(&flavour_value) else {
|
||||
return Err(ParseError::ParserError(format!(
|
||||
"unsupported block flavour: {flavour_value}"
|
||||
)));
|
||||
};
|
||||
Ok(Self::from_block_map_with_flavour(block, flavour))
|
||||
}
|
||||
|
||||
pub(super) fn from_block_map_with_flavour(block: &Map, flavour: BlockFlavour) -> Self {
|
||||
if flavour == BlockFlavour::Image {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: Some(ImageSpec::from_block_map(block)),
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::Table {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: Some(TableSpec::from_block_map(block)),
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::Bookmark {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: Some(BookmarkSpec {
|
||||
url: get_string(block, PROP_URL).unwrap_or_default(),
|
||||
caption: get_string(block, PROP_CAPTION),
|
||||
}),
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::EmbedYoutube {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: Some(EmbedYoutubeSpec {
|
||||
video_id: get_string(block, PROP_VIDEO_ID).unwrap_or_default(),
|
||||
}),
|
||||
embed_iframe: None,
|
||||
};
|
||||
}
|
||||
|
||||
if flavour == BlockFlavour::EmbedIframe {
|
||||
return BlockSpec {
|
||||
flavour,
|
||||
block_type: None,
|
||||
text: Vec::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: Some(EmbedIframeSpec {
|
||||
url: get_string(block, PROP_URL).unwrap_or_default(),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
let block_type = match get_string(block, PROP_TYPE) {
|
||||
Some(value) => Some(BlockType::from_str_lossy(value)),
|
||||
None => match flavour {
|
||||
BlockFlavour::Paragraph => Some(BlockType::Text),
|
||||
BlockFlavour::List => Some(BlockType::Bulleted),
|
||||
_ => None,
|
||||
},
|
||||
};
|
||||
|
||||
let text = block
|
||||
.get(PROP_TEXT)
|
||||
.and_then(|v| v.to_text())
|
||||
.map(|text| text.to_delta())
|
||||
.unwrap_or_default();
|
||||
|
||||
let checked = block.get(PROP_CHECKED).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::True => Some(true),
|
||||
Any::False => Some(false),
|
||||
_ => None,
|
||||
});
|
||||
let language = get_string(block, PROP_LANGUAGE);
|
||||
let order = block.get(PROP_ORDER).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::Integer(value) => Some(value as i64),
|
||||
Any::BigInt64(value) => Some(value),
|
||||
Any::Float32(value) => Some(value.0 as i64),
|
||||
Any::Float64(value) => Some(value.0 as i64),
|
||||
_ => None,
|
||||
});
|
||||
|
||||
BlockSpec {
|
||||
flavour,
|
||||
block_type,
|
||||
text,
|
||||
checked,
|
||||
language,
|
||||
order,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct BlockNode {
|
||||
pub(super) spec: BlockSpec,
|
||||
pub(super) children: Vec<BlockNode>,
|
||||
}
|
||||
|
||||
pub(super) trait TreeNode {
|
||||
fn children(&self) -> &[Self]
|
||||
where
|
||||
Self: Sized;
|
||||
}
|
||||
|
||||
impl TreeNode for BlockNode {
|
||||
fn children(&self) -> &[BlockNode] {
|
||||
&self.children
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn count_tree_nodes<T: TreeNode>(nodes: &[T]) -> usize {
|
||||
nodes.iter().map(|node| 1 + count_tree_nodes(node.children())).sum()
|
||||
}
|
||||
|
||||
pub(super) fn text_delta_eq(a: &[TextDeltaOp], b: &[TextDeltaOp]) -> bool {
|
||||
if a.len() != b.len() {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (left, right) in a.iter().zip(b.iter()) {
|
||||
match (left, right) {
|
||||
(
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text_a),
|
||||
format: format_a,
|
||||
},
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text_b),
|
||||
format: format_b,
|
||||
},
|
||||
) => {
|
||||
if text_a != text_b {
|
||||
return false;
|
||||
}
|
||||
if !attrs_eq(format_a.as_ref(), format_b.as_ref()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
_ => return false,
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
fn attrs_eq(a: Option<&TextAttributes>, b: Option<&TextAttributes>) -> bool {
|
||||
match (a, b) {
|
||||
(None, None) => true,
|
||||
(Some(left), Some(right)) => {
|
||||
if left.len() != right.len() {
|
||||
return false;
|
||||
}
|
||||
left.iter().all(|(key, value)| right.get(key) == Some(value))
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::{super::write::builder::text_ops_from_plain, *};
|
||||
use crate::doc_parser::build_full_doc;
|
||||
|
||||
fn spec_from_markdown(markdown: &str, doc_id: &str, flavour: &str) -> BlockSpec {
|
||||
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, SYS_FLAVOUR).as_deref() == Some(flavour)
|
||||
{
|
||||
return BlockSpec::from_block_map(&block_map).expect("spec");
|
||||
}
|
||||
}
|
||||
|
||||
panic!("block not found: {flavour}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_paragraph() {
|
||||
let spec = spec_from_markdown("Plain paragraph.", "block-spec-paragraph", "affine:paragraph");
|
||||
assert_eq!(spec.flavour, BlockFlavour::Paragraph);
|
||||
assert_eq!(spec.block_type, Some(BlockType::Text));
|
||||
assert_eq!(spec.text, text_ops_from_plain("Plain paragraph."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_list_checked() {
|
||||
let spec = spec_from_markdown("- [x] Done", "block-spec-list", "affine:list");
|
||||
assert_eq!(spec.flavour, BlockFlavour::List);
|
||||
assert_eq!(spec.block_type, Some(BlockType::Todo));
|
||||
assert_eq!(spec.checked, Some(true));
|
||||
assert_eq!(spec.text, text_ops_from_plain("Done"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_image() {
|
||||
let spec = spec_from_markdown("", "block-spec-image", "affine:image");
|
||||
assert_eq!(spec.flavour, BlockFlavour::Image);
|
||||
let image = spec.image.expect("image spec");
|
||||
assert_eq!(image.source_id, "image-id");
|
||||
assert_eq!(image.caption.as_deref(), Some("Alt"));
|
||||
assert_eq!(image.width, None);
|
||||
assert_eq!(image.height, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_table() {
|
||||
let spec = spec_from_markdown(
|
||||
"| A | B |\n| --- | --- |\n| 1 | 2 |",
|
||||
"block-spec-table",
|
||||
"affine:table",
|
||||
);
|
||||
assert_eq!(spec.flavour, BlockFlavour::Table);
|
||||
let table = spec.table.expect("table spec");
|
||||
assert_eq!(
|
||||
table.rows,
|
||||
vec![
|
||||
vec!["A".to_string(), "B".to_string()],
|
||||
vec!["1".to_string(), "2".to_string()]
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_block_map_embed_iframe() {
|
||||
let spec = spec_from_markdown(
|
||||
r#"<iframe src="https://example.com/embed"></iframe>"#,
|
||||
"block-spec-embed-iframe",
|
||||
"affine:embed-iframe",
|
||||
);
|
||||
assert_eq!(spec.flavour, BlockFlavour::EmbedIframe);
|
||||
assert_eq!(spec.embed_iframe.as_ref().unwrap().url, "https://example.com/embed");
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,10 @@ use std::collections::{HashMap, HashSet};
|
||||
|
||||
use y_octo::Map;
|
||||
|
||||
use super::value::value_to_string;
|
||||
use super::{
|
||||
schema::{SYS_CHILDREN, SYS_FLAVOUR, SYS_ID},
|
||||
value::value_to_string,
|
||||
};
|
||||
|
||||
pub(super) struct BlockIndex {
|
||||
pub(super) block_pool: HashMap<String, Map>,
|
||||
@@ -96,7 +99,7 @@ pub(super) fn find_block_id_by_flavour(block_pool: &HashMap<String, Map>, flavou
|
||||
|
||||
pub(super) fn collect_child_ids(block: &Map) -> Vec<String> {
|
||||
block
|
||||
.get("sys:children")
|
||||
.get(SYS_CHILDREN)
|
||||
.and_then(|value| value.to_array())
|
||||
.map(|array| {
|
||||
array
|
||||
@@ -108,11 +111,11 @@ pub(super) fn collect_child_ids(block: &Map) -> Vec<String> {
|
||||
}
|
||||
|
||||
pub(super) fn get_block_id(block: &Map) -> Option<String> {
|
||||
get_string(block, "sys:id")
|
||||
get_string(block, SYS_ID)
|
||||
}
|
||||
|
||||
pub(super) fn get_flavour(block: &Map) -> Option<String> {
|
||||
get_string(block, "sys:flavour")
|
||||
get_string(block, SYS_FLAVOUR)
|
||||
}
|
||||
|
||||
pub(super) fn get_string(block: &Map, key: &str) -> Option<String> {
|
||||
@@ -157,3 +160,17 @@ pub(super) fn nearest_by_flavour(
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn find_child_id_by_flavour(
|
||||
parent: &Map,
|
||||
block_pool: &HashMap<String, Map>,
|
||||
flavour: &str,
|
||||
) -> Option<String> {
|
||||
collect_child_ids(parent).into_iter().find(|id| {
|
||||
block_pool
|
||||
.get(id)
|
||||
.and_then(get_flavour)
|
||||
.as_deref()
|
||||
.is_some_and(|value| value == flavour)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
use y_octo::{Doc, DocOptions};
|
||||
|
||||
use super::ParseError;
|
||||
|
||||
pub(super) fn is_empty_doc(binary: &[u8]) -> bool {
|
||||
binary.is_empty() || binary == [0, 0]
|
||||
}
|
||||
|
||||
pub(super) fn load_doc(binary: &[u8], doc_id: Option<&str>) -> Result<Doc, ParseError> {
|
||||
if is_empty_doc(binary) {
|
||||
return Err(ParseError::InvalidBinary);
|
||||
}
|
||||
|
||||
let mut doc = build_doc(doc_id);
|
||||
doc
|
||||
.apply_update_from_binary_v1(binary)
|
||||
.map_err(|_| ParseError::InvalidBinary)?;
|
||||
Ok(doc)
|
||||
}
|
||||
|
||||
pub(super) fn load_doc_or_new(binary: &[u8]) -> Result<Doc, ParseError> {
|
||||
if is_empty_doc(binary) {
|
||||
return Ok(DocOptions::new().build());
|
||||
}
|
||||
|
||||
let mut doc = DocOptions::new().build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(binary)
|
||||
.map_err(|_| ParseError::InvalidBinary)?;
|
||||
Ok(doc)
|
||||
}
|
||||
|
||||
fn build_doc(doc_id: Option<&str>) -> Doc {
|
||||
let options = DocOptions::new();
|
||||
match doc_id {
|
||||
Some(doc_id) => options.with_guid(doc_id.to_string()).build(),
|
||||
None => options.build(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use thiserror::Error;
|
||||
use y_octo::JwstCodecError;
|
||||
|
||||
#[derive(Error, Debug, Serialize, Deserialize)]
|
||||
pub enum ParseError {
|
||||
#[error("doc_not_found")]
|
||||
DocNotFound,
|
||||
#[error("invalid_binary")]
|
||||
InvalidBinary,
|
||||
#[error("sqlite_error: {0}")]
|
||||
SqliteError(String),
|
||||
#[error("parser_error: {0}")]
|
||||
ParserError(String),
|
||||
#[error("unknown: {0}")]
|
||||
Unknown(String),
|
||||
}
|
||||
|
||||
impl From<JwstCodecError> for ParseError {
|
||||
fn from(value: JwstCodecError) -> Self {
|
||||
if matches!(
|
||||
value,
|
||||
JwstCodecError::DamagedDocumentJson
|
||||
| JwstCodecError::IncompleteDocument(_)
|
||||
| JwstCodecError::InvalidWriteBuffer(_)
|
||||
| JwstCodecError::UpdateInvalid(_)
|
||||
| JwstCodecError::StructClockInvalid { expect: _, actually: _ }
|
||||
| JwstCodecError::StructSequenceInvalid { client_id: _, clock: _ }
|
||||
| JwstCodecError::StructSequenceNotExists(_)
|
||||
| JwstCodecError::RootStructNotFound(_)
|
||||
| JwstCodecError::ParentNotFound
|
||||
| JwstCodecError::IndexOutOfBound(_)
|
||||
) {
|
||||
return ParseError::InvalidBinary;
|
||||
}
|
||||
Self::ParserError(value.to_string())
|
||||
}
|
||||
}
|
||||
+92
-34
@@ -6,8 +6,11 @@ use std::{
|
||||
|
||||
use y_octo::{AHashMap, Any, Map, Text, TextAttributes, TextDeltaOp, TextInsert, Value};
|
||||
|
||||
use super::value::{
|
||||
any_as_string, any_as_u64, any_truthy, build_reference_payload, params_any_map_to_json, value_to_any,
|
||||
use super::{
|
||||
super::value::{
|
||||
any_as_string, any_as_u64, any_truthy, build_reference_payload, params_any_map_to_json, value_to_any,
|
||||
},
|
||||
inline::InlineStyle,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -20,18 +23,18 @@ struct InlineReference {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct InlineReferencePayload {
|
||||
pub(super) doc_id: String,
|
||||
pub(super) payload: String,
|
||||
pub(crate) struct InlineReferencePayload {
|
||||
pub(crate) doc_id: String,
|
||||
pub(crate) payload: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct DeltaToMdOptions {
|
||||
pub(crate) struct DeltaToMdOptions {
|
||||
doc_url_prefix: Option<String>,
|
||||
}
|
||||
|
||||
impl DeltaToMdOptions {
|
||||
pub(super) fn new(doc_url_prefix: Option<String>) -> Self {
|
||||
pub(crate) fn new(doc_url_prefix: Option<String>) -> Self {
|
||||
Self { doc_url_prefix }
|
||||
}
|
||||
|
||||
@@ -54,21 +57,32 @@ impl DeltaToMdOptions {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn text_to_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option<String> {
|
||||
block
|
||||
.get(key)
|
||||
.and_then(|value| value.to_text())
|
||||
.map(|text| delta_to_markdown(&text, options))
|
||||
}
|
||||
|
||||
pub(super) fn text_to_inline_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option<String> {
|
||||
pub(crate) fn text_to_inline_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option<String> {
|
||||
block
|
||||
.get(key)
|
||||
.and_then(|value| value.to_text())
|
||||
.map(|text| delta_to_inline_markdown(&text, options))
|
||||
}
|
||||
|
||||
pub(super) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec<InlineReferencePayload> {
|
||||
pub(crate) fn delta_ops_to_markdown(ops: &[TextDeltaOp], options: &DeltaToMdOptions) -> String {
|
||||
delta_to_markdown_with_options(ops, options, true)
|
||||
}
|
||||
|
||||
pub(crate) fn delta_ops_to_plain_text(ops: &[TextDeltaOp]) -> String {
|
||||
let mut out = String::new();
|
||||
for op in ops {
|
||||
if let TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text),
|
||||
..
|
||||
} = op
|
||||
{
|
||||
out.push_str(text);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub(crate) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec<InlineReferencePayload> {
|
||||
let mut refs = Vec::new();
|
||||
let mut seen: HashSet<(String, String)> = HashSet::new();
|
||||
|
||||
@@ -80,7 +94,7 @@ pub(super) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec<InlineRefe
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let reference = match attrs.get("reference").and_then(parse_inline_reference) {
|
||||
let reference = match attrs.get(InlineStyle::Reference.key()).and_then(parse_inline_reference) {
|
||||
Some(reference) => reference,
|
||||
None => continue,
|
||||
};
|
||||
@@ -102,7 +116,7 @@ pub(super) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec<InlineRefe
|
||||
refs
|
||||
}
|
||||
|
||||
pub(super) fn extract_inline_references_from_value(value: &Value) -> Vec<InlineReferencePayload> {
|
||||
pub(crate) fn extract_inline_references_from_value(value: &Value) -> Vec<InlineReferencePayload> {
|
||||
if let Some(text) = value.to_text() {
|
||||
return extract_inline_references(&text.to_delta());
|
||||
}
|
||||
@@ -123,7 +137,11 @@ fn extract_inline_references_from_any(value: &Any) -> Vec<InlineReferencePayload
|
||||
let mut seen: HashSet<(String, String)> = HashSet::new();
|
||||
|
||||
for op in ops {
|
||||
let reference = match op.attributes.get("reference").and_then(parse_inline_reference) {
|
||||
let reference = match op
|
||||
.attributes
|
||||
.get(InlineStyle::Reference.key())
|
||||
.and_then(parse_inline_reference)
|
||||
{
|
||||
Some(reference) => reference,
|
||||
None => continue,
|
||||
};
|
||||
@@ -178,10 +196,6 @@ fn inline_reference_payload(reference: &InlineReference) -> Option<String> {
|
||||
Some(build_reference_payload(&reference.page_id, params))
|
||||
}
|
||||
|
||||
fn delta_to_markdown(text: &Text, options: &DeltaToMdOptions) -> String {
|
||||
delta_to_markdown_with_options(&text.to_delta(), options, true)
|
||||
}
|
||||
|
||||
fn delta_to_inline_markdown(text: &Text, options: &DeltaToMdOptions) -> String {
|
||||
delta_to_markdown_with_options(&text.to_delta(), options, false)
|
||||
}
|
||||
@@ -274,7 +288,7 @@ fn delta_any_to_inline_markdown(value: &Any, options: &DeltaToMdOptions) -> Opti
|
||||
delta_ops_from_any(value).map(|ops| delta_ops_to_markdown_with_options(&ops, options, false))
|
||||
}
|
||||
|
||||
pub(super) fn delta_value_to_inline_markdown(value: &Value, options: &DeltaToMdOptions) -> Option<String> {
|
||||
pub(crate) fn delta_value_to_inline_markdown(value: &Value, options: &DeltaToMdOptions) -> Option<String> {
|
||||
if let Some(text) = value.to_text() {
|
||||
return Some(delta_to_inline_markdown(&text, options));
|
||||
}
|
||||
@@ -538,19 +552,29 @@ fn apply_inline_attributes(
|
||||
}
|
||||
|
||||
fn inline_node_for_attr(attr: &str, attrs: &TextAttributes, options: &DeltaToMdOptions) -> Option<Rc<RefCell<Node>>> {
|
||||
match attr {
|
||||
"italic" => Some(Node::new_inline("_", "_")),
|
||||
"bold" => Some(Node::new_inline("**", "**")),
|
||||
"link" => attrs
|
||||
let style = InlineStyle::from_key(attr)?;
|
||||
if let Some(delimiter) = style.delimiter() {
|
||||
return Some(Node::new_inline(delimiter.open, delimiter.close));
|
||||
}
|
||||
|
||||
match style {
|
||||
InlineStyle::Underline => Some(Node::new_inline("<u>", "</u>")),
|
||||
InlineStyle::Color => {
|
||||
let color = attrs.get(attr).and_then(any_as_string)?.trim();
|
||||
if color.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(Node::new_inline(&format!("<span style=\"color: {color}\">"), "</span>"))
|
||||
}
|
||||
}
|
||||
InlineStyle::Link => attrs
|
||||
.get(attr)
|
||||
.and_then(any_as_string)
|
||||
.map(|url| Node::new_inline("[", &format!("]({url})"))),
|
||||
"reference" => attrs.get(attr).and_then(parse_inline_reference).map(|reference| {
|
||||
InlineStyle::Reference => attrs.get(attr).and_then(parse_inline_reference).map(|reference| {
|
||||
let (title, link) = options.build_reference_link(&reference);
|
||||
Node::new_inline("[", &format!("{title}]({link})"))
|
||||
}),
|
||||
"strike" => Some(Node::new_inline("~~", "~~")),
|
||||
"code" => Some(Node::new_inline("`", "`")),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -560,7 +584,7 @@ fn has_block_level_attribute(attrs: &TextAttributes) -> bool {
|
||||
}
|
||||
|
||||
fn is_inline_attribute(attr: &str) -> bool {
|
||||
matches!(attr, "italic" | "bold" | "link" | "reference" | "strike" | "code")
|
||||
InlineStyle::from_key(attr).is_some()
|
||||
}
|
||||
|
||||
fn encode_link(link: &str) -> String {
|
||||
@@ -741,13 +765,17 @@ fn new_line(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::Value;
|
||||
use y_octo::{Any, TextAttributes, TextDeltaOp, TextInsert};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_delta_to_inline_markdown_link() {
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert("link".into(), Any::String("https://example.com".into()));
|
||||
attrs.insert(
|
||||
InlineStyle::Link.key().into(),
|
||||
Any::String("https://example.com".into()),
|
||||
);
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("AFFiNE".into()),
|
||||
@@ -767,7 +795,7 @@ mod tests {
|
||||
ref_map.insert("type".into(), Any::String("LinkedPage".into()));
|
||||
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert("reference".into(), Any::Object(ref_map));
|
||||
attrs.insert(InlineStyle::Reference.key().into(), Any::Object(ref_map));
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Doc Title".into()),
|
||||
@@ -781,4 +809,34 @@ mod tests {
|
||||
let payload: Value = serde_json::from_str(&refs[0].payload).unwrap();
|
||||
assert_eq!(payload, serde_json::json!({ "docId": "doc123" }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delta_to_inline_markdown_underline() {
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert(InlineStyle::Underline.key().into(), Any::True);
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Under".into()),
|
||||
format: Some(attrs),
|
||||
}];
|
||||
|
||||
let options = DeltaToMdOptions::new(None);
|
||||
let rendered = delta_to_markdown_with_options(&delta, &options, false);
|
||||
assert_eq!(rendered, "<u>Under</u>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delta_to_inline_markdown_color() {
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert(InlineStyle::Color.key().into(), Any::String("red".into()));
|
||||
|
||||
let delta = vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Red".into()),
|
||||
format: Some(attrs),
|
||||
}];
|
||||
|
||||
let options = DeltaToMdOptions::new(None);
|
||||
let rendered = delta_to_markdown_with_options(&delta, &options, false);
|
||||
assert_eq!(rendered, "<span style=\"color: red\">Red</span>");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
const INLINE_ATTR_BOLD: &str = "bold";
|
||||
const INLINE_ATTR_ITALIC: &str = "italic";
|
||||
const INLINE_ATTR_UNDERLINE: &str = "underline";
|
||||
const INLINE_ATTR_STRIKE: &str = "strike";
|
||||
const INLINE_ATTR_CODE: &str = "code";
|
||||
const INLINE_ATTR_LINK: &str = "link";
|
||||
const INLINE_ATTR_REFERENCE: &str = "reference";
|
||||
const INLINE_ATTR_COLOR: &str = "color";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum InlineStyle {
|
||||
Bold,
|
||||
Italic,
|
||||
Underline,
|
||||
Strike,
|
||||
Code,
|
||||
Link,
|
||||
Reference,
|
||||
Color,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) struct InlineDelimiter {
|
||||
pub(super) open: &'static str,
|
||||
pub(super) close: &'static str,
|
||||
}
|
||||
|
||||
impl InlineStyle {
|
||||
pub(super) fn key(self) -> &'static str {
|
||||
match self {
|
||||
InlineStyle::Bold => INLINE_ATTR_BOLD,
|
||||
InlineStyle::Italic => INLINE_ATTR_ITALIC,
|
||||
InlineStyle::Underline => INLINE_ATTR_UNDERLINE,
|
||||
InlineStyle::Strike => INLINE_ATTR_STRIKE,
|
||||
InlineStyle::Code => INLINE_ATTR_CODE,
|
||||
InlineStyle::Link => INLINE_ATTR_LINK,
|
||||
InlineStyle::Reference => INLINE_ATTR_REFERENCE,
|
||||
InlineStyle::Color => INLINE_ATTR_COLOR,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn from_key(key: &str) -> Option<Self> {
|
||||
match key {
|
||||
INLINE_ATTR_BOLD => Some(InlineStyle::Bold),
|
||||
INLINE_ATTR_ITALIC => Some(InlineStyle::Italic),
|
||||
INLINE_ATTR_UNDERLINE => Some(InlineStyle::Underline),
|
||||
INLINE_ATTR_STRIKE => Some(InlineStyle::Strike),
|
||||
INLINE_ATTR_CODE => Some(InlineStyle::Code),
|
||||
INLINE_ATTR_LINK => Some(InlineStyle::Link),
|
||||
INLINE_ATTR_REFERENCE => Some(InlineStyle::Reference),
|
||||
INLINE_ATTR_COLOR => Some(InlineStyle::Color),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn delimiter(self) -> Option<InlineDelimiter> {
|
||||
match self {
|
||||
InlineStyle::Italic => Some(InlineDelimiter { open: "_", close: "_" }),
|
||||
InlineStyle::Bold => Some(InlineDelimiter {
|
||||
open: "**",
|
||||
close: "**",
|
||||
}),
|
||||
InlineStyle::Strike => Some(InlineDelimiter {
|
||||
open: "~~",
|
||||
close: "~~",
|
||||
}),
|
||||
InlineStyle::Code => Some(InlineDelimiter { open: "`", close: "`" }),
|
||||
InlineStyle::Link | InlineStyle::Reference | InlineStyle::Underline | InlineStyle::Color => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
mod delta;
|
||||
mod inline;
|
||||
mod parser;
|
||||
mod render;
|
||||
|
||||
pub(crate) use delta::{
|
||||
DeltaToMdOptions, InlineReferencePayload, delta_value_to_inline_markdown, extract_inline_references,
|
||||
extract_inline_references_from_value, text_to_inline_markdown,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use parser::MAX_MARKDOWN_CHARS;
|
||||
pub(crate) use parser::{MAX_BLOCKS, parse_markdown_blocks};
|
||||
pub(crate) use render::{MarkdownRenderer, MarkdownWriter};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,235 @@
|
||||
use super::{
|
||||
super::block_spec::{BlockFlavour, BlockSpec},
|
||||
delta::{DeltaToMdOptions, delta_ops_to_markdown, delta_ops_to_plain_text},
|
||||
};
|
||||
|
||||
pub(crate) struct MarkdownWriter<'a> {
|
||||
output: &'a mut String,
|
||||
}
|
||||
|
||||
impl<'a> MarkdownWriter<'a> {
|
||||
pub(crate) fn new(output: &'a mut String) -> Self {
|
||||
Self { output }
|
||||
}
|
||||
|
||||
pub(crate) fn push_paragraph(&mut self, prefix: &str, text: &str) {
|
||||
if prefix == "> " {
|
||||
let quoted = text
|
||||
.split('\n')
|
||||
.map(|line| format!("{prefix}{line}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
self.output.push_str("ed);
|
||||
if !text.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
self.output.push('\n');
|
||||
return;
|
||||
}
|
||||
|
||||
self.output.push_str(prefix);
|
||||
self.output.push_str(text);
|
||||
if !text.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
self.output.push('\n');
|
||||
}
|
||||
|
||||
pub(crate) fn push_list_item(&mut self, indent: &str, prefix: &str, text: &str) {
|
||||
self.output.push_str(indent);
|
||||
self.output.push_str(prefix);
|
||||
self.output.push_str(text);
|
||||
if !text.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn push_code_block(&mut self, lang: &str, text: &str) {
|
||||
self.output.push_str("```");
|
||||
self.output.push_str(lang);
|
||||
self.output.push('\n');
|
||||
self.output.push_str(text);
|
||||
self.output.push_str("\n```\n\n");
|
||||
}
|
||||
|
||||
pub(crate) fn push_divider(&mut self) {
|
||||
self.output.push_str("\n---\n\n");
|
||||
}
|
||||
|
||||
pub(crate) fn push_table(&mut self, table: &str) {
|
||||
if table.is_empty() {
|
||||
self.output.push('\n');
|
||||
return;
|
||||
}
|
||||
self.output.push_str(table);
|
||||
if !table.ends_with('\n') {
|
||||
self.output.push('\n');
|
||||
}
|
||||
self.output.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn paragraph_prefix(type_: &str) -> &'static str {
|
||||
match type_ {
|
||||
"h1" => "# ",
|
||||
"h2" => "## ",
|
||||
"h3" => "### ",
|
||||
"h4" => "#### ",
|
||||
"h5" => "##### ",
|
||||
"h6" => "###### ",
|
||||
"quote" => "> ",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn list_prefix(r#type: &str, checked: bool, order: Option<i64>) -> String {
|
||||
match r#type {
|
||||
"bulleted" => "* ".to_string(),
|
||||
"todo" => if checked { "- [x] " } else { "- [ ] " }.to_string(),
|
||||
_ => format!("{}. ", order.unwrap_or(1)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn list_indent(depth: usize) -> String {
|
||||
" ".repeat(depth)
|
||||
}
|
||||
|
||||
pub(crate) struct MarkdownRenderer<'a> {
|
||||
options: &'a DeltaToMdOptions,
|
||||
}
|
||||
|
||||
impl<'a> MarkdownRenderer<'a> {
|
||||
pub(crate) fn new(options: &'a DeltaToMdOptions) -> Self {
|
||||
Self { options }
|
||||
}
|
||||
|
||||
pub(crate) fn write_block(&self, output: &mut String, spec: &BlockSpec, list_depth: usize) {
|
||||
match spec.flavour {
|
||||
BlockFlavour::Paragraph => {
|
||||
let prefix = paragraph_prefix(spec.block_type_str().unwrap_or_default());
|
||||
let text_md = delta_ops_to_markdown(&spec.text, self.options);
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_paragraph(prefix, &text_md);
|
||||
}
|
||||
BlockFlavour::List => {
|
||||
let type_ = spec.block_type_str().unwrap_or("bulleted");
|
||||
let checked = spec.checked.unwrap_or(false);
|
||||
let prefix = list_prefix(type_, checked, spec.order);
|
||||
let indent = list_indent(list_depth);
|
||||
let text_md = delta_ops_to_markdown(&spec.text, self.options);
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_list_item(&indent, &prefix, &text_md);
|
||||
}
|
||||
BlockFlavour::Code => {
|
||||
let text = delta_ops_to_plain_text(&spec.text);
|
||||
let lang = spec.language.as_deref().unwrap_or_default();
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_code_block(lang, &text);
|
||||
}
|
||||
BlockFlavour::Divider => {
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_divider();
|
||||
}
|
||||
BlockFlavour::Image => {
|
||||
if let Some(image) = spec.image.as_ref() {
|
||||
output.push_str(&image.render_markdown());
|
||||
}
|
||||
}
|
||||
BlockFlavour::Bookmark => {
|
||||
if let Some(bookmark) = spec.bookmark.as_ref() {
|
||||
output.push_str(&format!("\n[](Bookmark,{})\n\n", bookmark.url));
|
||||
}
|
||||
}
|
||||
BlockFlavour::EmbedYoutube => {
|
||||
if let Some(embed) = spec.embed_youtube.as_ref() {
|
||||
output.push_str(
|
||||
&format!(
|
||||
"\n <iframe\n type=\"text/html\"\n width=\"100%\"\n height=\"410px\"\n src=\"https://www.youtube.com/embed/{}\"\n frameborder=\"0\"\n allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\"\n allowfullscreen\n credentialless>\n </iframe>\n\n",
|
||||
embed.video_id
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
BlockFlavour::EmbedIframe => {
|
||||
if let Some(embed) = spec.embed_iframe.as_ref() {
|
||||
output.push_str(&format!("\n<iframe src=\"{}\"></iframe>\n\n", embed.url));
|
||||
}
|
||||
}
|
||||
BlockFlavour::Callout => {}
|
||||
BlockFlavour::Table => {
|
||||
if let Some(table) = spec.table.as_ref()
|
||||
&& let Some(table_md) = table.render_markdown()
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(output);
|
||||
writer.push_table(&table_md);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_paragraph_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_paragraph("# ", "Title\n");
|
||||
}
|
||||
assert_eq!(markdown, "# Title\n\n");
|
||||
|
||||
markdown.clear();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_paragraph("", "Plain");
|
||||
}
|
||||
assert_eq!(markdown, "Plain\n\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_list_item(" ", "* ", "Item\n");
|
||||
}
|
||||
assert_eq!(markdown, " * Item\n");
|
||||
|
||||
markdown.clear();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_list_item("", "- [ ] ", "Task");
|
||||
}
|
||||
assert_eq!(markdown, "- [ ] Task\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_code_block_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_code_block("rs", "fn main() {}");
|
||||
}
|
||||
assert_eq!(markdown, "```rs\nfn main() {}\n```\n\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_table_newlines() {
|
||||
let mut markdown = String::new();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_table("|a|b|\n|---|---|\n|1|2|\n");
|
||||
}
|
||||
assert_eq!(markdown, "|a|b|\n|---|---|\n|1|2|\n\n");
|
||||
|
||||
markdown.clear();
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut markdown);
|
||||
writer.push_table("|a|b|");
|
||||
}
|
||||
assert_eq!(markdown, "|a|b|\n\n");
|
||||
}
|
||||
}
|
||||
@@ -1,492 +0,0 @@
|
||||
//! Markdown to YDoc conversion module
|
||||
//!
|
||||
//! Converts markdown content into AFFiNE-compatible y-octo document binary
|
||||
//! format.
|
||||
|
||||
use y_octo::{Any, DocOptions};
|
||||
|
||||
use super::{
|
||||
affine::ParseError,
|
||||
markdown_utils::{BlockType, ParsedBlock, extract_title, parse_markdown_blocks},
|
||||
};
|
||||
|
||||
/// Block types used in AFFiNE documents
|
||||
const PAGE_FLAVOUR: &str = "affine:page";
|
||||
const NOTE_FLAVOUR: &str = "affine:note";
|
||||
|
||||
/// Intermediate representation of a block for building y-octo documents
|
||||
struct BlockBuilder {
|
||||
id: String,
|
||||
flavour: String,
|
||||
text_content: String,
|
||||
block_type: Option<BlockType>,
|
||||
checked: Option<bool>,
|
||||
code_language: Option<String>,
|
||||
#[allow(dead_code)] // Reserved for future nested block support
|
||||
children: Vec<String>,
|
||||
}
|
||||
|
||||
impl BlockBuilder {
|
||||
fn new(flavour: &str) -> Self {
|
||||
Self {
|
||||
id: nanoid::nanoid!(),
|
||||
flavour: flavour.to_string(),
|
||||
text_content: String::new(),
|
||||
block_type: None,
|
||||
checked: None,
|
||||
code_language: None,
|
||||
children: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_text(mut self, text: &str) -> Self {
|
||||
self.text_content = text.to_string();
|
||||
self
|
||||
}
|
||||
|
||||
fn with_block_type(mut self, btype: BlockType) -> Self {
|
||||
self.block_type = Some(btype);
|
||||
self
|
||||
}
|
||||
|
||||
fn with_checked(mut self, checked: bool) -> Self {
|
||||
self.checked = Some(checked);
|
||||
self
|
||||
}
|
||||
|
||||
fn with_code_language(mut self, lang: &str) -> Self {
|
||||
if !lang.is_empty() {
|
||||
self.code_language = Some(lang.to_string());
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a ParsedBlock from the shared parser into a BlockBuilder
|
||||
impl From<ParsedBlock> for BlockBuilder {
|
||||
fn from(parsed: ParsedBlock) -> Self {
|
||||
let mut builder = BlockBuilder::new(parsed.flavour.as_str()).with_text(&parsed.content);
|
||||
|
||||
if let Some(btype) = parsed.block_type {
|
||||
builder = builder.with_block_type(btype);
|
||||
}
|
||||
|
||||
if let Some(checked) = parsed.checked {
|
||||
builder = builder.with_checked(checked);
|
||||
}
|
||||
|
||||
if let Some(lang) = parsed.language {
|
||||
builder = builder.with_code_language(&lang);
|
||||
}
|
||||
|
||||
builder
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses markdown and converts it to an AFFiNE-compatible y-octo document
|
||||
/// binary.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `markdown` - The markdown content to convert
|
||||
/// * `doc_id` - The document ID to use
|
||||
///
|
||||
/// # Returns
|
||||
/// A binary vector containing the y-octo encoded document
|
||||
pub fn markdown_to_ydoc(markdown: &str, doc_id: &str) -> Result<Vec<u8>, ParseError> {
|
||||
// Extract the title from the first H1 heading
|
||||
let title = extract_title(markdown);
|
||||
|
||||
// Parse markdown into blocks using the shared parser
|
||||
let parsed_blocks = parse_markdown_blocks(markdown, true);
|
||||
|
||||
// Convert ParsedBlocks to BlockBuilders and collect IDs
|
||||
let mut blocks: Vec<BlockBuilder> = Vec::new();
|
||||
let mut content_block_ids: Vec<String> = Vec::new();
|
||||
|
||||
for parsed in parsed_blocks {
|
||||
let builder: BlockBuilder = parsed.into();
|
||||
content_block_ids.push(builder.id.clone());
|
||||
blocks.push(builder);
|
||||
}
|
||||
|
||||
// Build the y-octo document
|
||||
build_ydoc(doc_id, &title, blocks, content_block_ids)
|
||||
}
|
||||
|
||||
/// Builds the y-octo document from parsed blocks.
|
||||
///
|
||||
/// Uses a two-phase approach to ensure Yjs compatibility:
|
||||
/// 1. Phase 1: Create and insert empty maps into blocks_map (establishes parent
|
||||
/// items)
|
||||
/// 2. Phase 2: Populate each map with properties (child items reference
|
||||
/// existing parents)
|
||||
///
|
||||
/// This ordering ensures that when items reference their parent map's ID in the
|
||||
/// encoded binary, the parent ID always has a lower clock value, which Yjs
|
||||
/// requires.
|
||||
fn build_ydoc(
|
||||
doc_id: &str,
|
||||
title: &str,
|
||||
content_blocks: Vec<BlockBuilder>,
|
||||
content_block_ids: Vec<String>,
|
||||
) -> Result<Vec<u8>, ParseError> {
|
||||
// Create the document with the specified ID
|
||||
let doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
|
||||
// Create the blocks map
|
||||
let mut blocks_map = doc
|
||||
.get_or_create_map("blocks")
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
|
||||
// Create block IDs
|
||||
let page_id = nanoid::nanoid!();
|
||||
let note_id = nanoid::nanoid!();
|
||||
|
||||
// ==== PHASE 1: Insert empty maps to establish parent items ====
|
||||
// This ensures parent items have lower clock values than their children
|
||||
|
||||
// Insert empty page block map
|
||||
blocks_map
|
||||
.insert(
|
||||
page_id.clone(),
|
||||
doc.create_map().map_err(|e| ParseError::ParserError(e.to_string()))?,
|
||||
)
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
|
||||
// Insert empty note block map
|
||||
blocks_map
|
||||
.insert(
|
||||
note_id.clone(),
|
||||
doc.create_map().map_err(|e| ParseError::ParserError(e.to_string()))?,
|
||||
)
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
|
||||
// Insert empty content block maps
|
||||
for block in &content_blocks {
|
||||
blocks_map
|
||||
.insert(
|
||||
block.id.clone(),
|
||||
doc.create_map().map_err(|e| ParseError::ParserError(e.to_string()))?,
|
||||
)
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
// ==== PHASE 2: Populate the maps with their properties ====
|
||||
// Now each map has an item with a lower clock, so children will reference
|
||||
// correctly
|
||||
|
||||
// Populate page block
|
||||
if let Some(page_map) = blocks_map.get(&page_id).and_then(|v| v.to_map()) {
|
||||
populate_block_map(
|
||||
&doc,
|
||||
page_map,
|
||||
&page_id,
|
||||
PAGE_FLAVOUR,
|
||||
Some(title),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
vec![note_id.clone()],
|
||||
)?;
|
||||
}
|
||||
|
||||
// Populate note block
|
||||
if let Some(note_map) = blocks_map.get(¬e_id).and_then(|v| v.to_map()) {
|
||||
populate_block_map(
|
||||
&doc,
|
||||
note_map,
|
||||
¬e_id,
|
||||
NOTE_FLAVOUR,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
content_block_ids.clone(),
|
||||
)?;
|
||||
}
|
||||
|
||||
// Populate content blocks
|
||||
for block in content_blocks {
|
||||
if let Some(block_map) = blocks_map.get(&block.id).and_then(|v| v.to_map()) {
|
||||
populate_block_map(
|
||||
&doc,
|
||||
block_map,
|
||||
&block.id,
|
||||
&block.flavour,
|
||||
None,
|
||||
if block.text_content.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(&block.text_content)
|
||||
},
|
||||
block.block_type,
|
||||
block.checked,
|
||||
block.code_language.as_deref(),
|
||||
Vec::new(),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Encode the document
|
||||
doc
|
||||
.encode_update_v1()
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))
|
||||
}
|
||||
|
||||
/// Populates an existing block map with the given properties.
|
||||
///
|
||||
/// This function takes an already-inserted map and populates it with
|
||||
/// properties. The two-phase approach (insert empty map first, then populate)
|
||||
/// ensures that when child items reference the map as their parent, the
|
||||
/// parent's clock is lower.
|
||||
///
|
||||
/// IMPORTANT: We use Any types (Any::Array, Any::String) instead of CRDT types
|
||||
/// (y_octo::Array, y_octo::Text) for nested values. Any types are encoded
|
||||
/// inline as part of the item content, avoiding the forward reference issue
|
||||
/// where child items would reference a parent with a higher clock value.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn populate_block_map(
|
||||
_doc: &y_octo::Doc,
|
||||
mut block: y_octo::Map,
|
||||
block_id: &str,
|
||||
flavour: &str,
|
||||
title: Option<&str>,
|
||||
text_content: Option<&str>,
|
||||
block_type: Option<BlockType>,
|
||||
checked: Option<bool>,
|
||||
code_language: Option<&str>,
|
||||
children: Vec<String>,
|
||||
) -> Result<(), ParseError> {
|
||||
// Required fields
|
||||
block
|
||||
.insert("sys:id".to_string(), Any::String(block_id.to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
block
|
||||
.insert("sys:flavour".to_string(), Any::String(flavour.to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
|
||||
// Children - use Any::Array which is encoded inline (no forward references)
|
||||
let children_any: Vec<Any> = children.into_iter().map(Any::String).collect();
|
||||
block
|
||||
.insert("sys:children".to_string(), Any::Array(children_any))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
|
||||
// Title
|
||||
if let Some(title) = title {
|
||||
block
|
||||
.insert("prop:title".to_string(), Any::String(title.to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Text content - use Any::String instead of Y.Text
|
||||
// This is simpler and avoids CRDT overhead for initial document creation
|
||||
if let Some(content) = text_content {
|
||||
block
|
||||
.insert("prop:text".to_string(), Any::String(content.to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Block type
|
||||
if let Some(btype) = block_type {
|
||||
block
|
||||
.insert("prop:type".to_string(), Any::String(btype.as_str().to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Checked state
|
||||
if let Some(is_checked) = checked {
|
||||
block
|
||||
.insert(
|
||||
"prop:checked".to_string(),
|
||||
if is_checked { Any::True } else { Any::False },
|
||||
)
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Code language
|
||||
if let Some(lang) = code_language {
|
||||
block
|
||||
.insert("prop:language".to_string(), Any::String(lang.to_string()))
|
||||
.map_err(|e| ParseError::ParserError(e.to_string()))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_simple_markdown() {
|
||||
let markdown = "# Hello World\n\nThis is a test paragraph.";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_list() {
|
||||
let markdown = "# Test List\n\n- Item 1\n- Item 2\n- Item 3";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_code() {
|
||||
let markdown = "# Code Example\n\n```rust\nfn main() {\n println!(\"Hello\");\n}\n```";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_headings() {
|
||||
let markdown = "# H1\n\n## H2\n\n### H3\n\nParagraph text.";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_usage() {
|
||||
assert_eq!(extract_title("# My Title\n\nContent"), "My Title");
|
||||
assert_eq!(extract_title("No heading"), "Untitled");
|
||||
assert_eq!(extract_title("## Secondary\n\nContent"), "Untitled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_markdown() {
|
||||
let result = markdown_to_ydoc("", "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty()); // Should still create valid doc structure
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_whitespace_only_markdown() {
|
||||
let result = markdown_to_ydoc(" \n\n\t\n ", "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_without_h1() {
|
||||
// Should use "Untitled" as default title
|
||||
let markdown = "## Secondary Heading\n\nSome content without H1.";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nested_lists() {
|
||||
let markdown = "# Nested Lists\n\n- Item 1\n - Nested 1.1\n - Nested 1.2\n- Item 2\n - Nested 2.1";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blockquote() {
|
||||
let markdown = "# Title\n\n> A blockquote";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_divider() {
|
||||
let markdown = "# Title\n\nBefore divider\n\n---\n\nAfter divider";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_numbered_list() {
|
||||
let markdown = "# Title\n\n1. First item\n2. Second item";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_four_paragraphs() {
|
||||
// Test with 4 paragraphs
|
||||
let markdown = "# Title\n\nP1.\n\nP2.\n\nP3.\n\nP4.";
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mixed_content() {
|
||||
let markdown = r#"# Mixed Content
|
||||
|
||||
Some intro text.
|
||||
|
||||
- List item 1
|
||||
- List item 2
|
||||
|
||||
```python
|
||||
def hello():
|
||||
print("world")
|
||||
```
|
||||
|
||||
## Another Section
|
||||
|
||||
More text here.
|
||||
|
||||
1. Numbered item
|
||||
2. Another numbered
|
||||
|
||||
> A blockquote
|
||||
|
||||
---
|
||||
|
||||
Final paragraph.
|
||||
"#;
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_code_block_preserves_indentation() {
|
||||
// Code blocks should preserve leading whitespace (indentation) which is
|
||||
// semantically significant in languages like Python, YAML, etc.
|
||||
let markdown = r#"# Code Test
|
||||
|
||||
```python
|
||||
def indented():
|
||||
return "preserved"
|
||||
```
|
||||
"#;
|
||||
let result = markdown_to_ydoc(markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
// The test passes if the conversion succeeds without errors.
|
||||
// Full verification would require roundtrip testing.
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_document_creation() {
|
||||
// Test that markdown_to_ydoc creates a valid binary
|
||||
let original_md = "# Test Document\n\nHello world.";
|
||||
let doc_id = "creation-test";
|
||||
|
||||
let bin = markdown_to_ydoc(original_md, doc_id).expect("Should convert to ydoc");
|
||||
|
||||
// Binary should not be empty
|
||||
assert!(!bin.is_empty(), "Binary should not be empty");
|
||||
assert!(bin.len() > 10, "Binary should have meaningful content");
|
||||
}
|
||||
|
||||
// NOTE: Full roundtrip tests (markdown -> ydoc -> markdown) are not included
|
||||
// because y-octo has a limitation where nested maps created with create_map()
|
||||
// lose their content after encode/decode. This is a known y-octo limitation.
|
||||
//
|
||||
// However, the documents we create ARE valid and can be:
|
||||
// 1. Pushed to the AFFiNE server via DocStorageAdapter.pushDocUpdates
|
||||
// 2. Read by the AFFiNE client which uses JavaScript Yjs (not y-octo)
|
||||
//
|
||||
// The MCP write tools work because:
|
||||
// - markdown_to_ydoc creates valid y-octo binary
|
||||
// - The server stores the binary directly
|
||||
// - The client (browser) uses Yjs to decode and render
|
||||
}
|
||||
@@ -1,463 +0,0 @@
|
||||
//! Shared markdown utilities for the doc_parser module
|
||||
|
||||
use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, Options, Parser, Tag, TagEnd};
|
||||
|
||||
/// Block flavours used in AFFiNE documents
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BlockFlavour {
|
||||
Paragraph,
|
||||
List,
|
||||
Code,
|
||||
Divider,
|
||||
}
|
||||
|
||||
impl BlockFlavour {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
BlockFlavour::Paragraph => "affine:paragraph",
|
||||
BlockFlavour::List => "affine:list",
|
||||
BlockFlavour::Code => "affine:code",
|
||||
BlockFlavour::Divider => "affine:divider",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Block types for paragraphs and lists
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BlockType {
|
||||
// Paragraph types
|
||||
#[allow(dead_code)] // Used via as_str() for default paragraph type
|
||||
Text,
|
||||
H1,
|
||||
H2,
|
||||
H3,
|
||||
H4,
|
||||
H5,
|
||||
H6,
|
||||
Quote,
|
||||
// List types
|
||||
Bulleted,
|
||||
Numbered,
|
||||
Todo,
|
||||
}
|
||||
|
||||
impl BlockType {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
BlockType::Text => "text",
|
||||
BlockType::H1 => "h1",
|
||||
BlockType::H2 => "h2",
|
||||
BlockType::H3 => "h3",
|
||||
BlockType::H4 => "h4",
|
||||
BlockType::H5 => "h5",
|
||||
BlockType::H6 => "h6",
|
||||
BlockType::Quote => "quote",
|
||||
BlockType::Bulleted => "bulleted",
|
||||
BlockType::Numbered => "numbered",
|
||||
BlockType::Todo => "todo",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_heading_level(level: HeadingLevel) -> Self {
|
||||
match level {
|
||||
HeadingLevel::H1 => BlockType::H1,
|
||||
HeadingLevel::H2 => BlockType::H2,
|
||||
HeadingLevel::H3 => BlockType::H3,
|
||||
HeadingLevel::H4 => BlockType::H4,
|
||||
HeadingLevel::H5 => BlockType::H5,
|
||||
HeadingLevel::H6 => BlockType::H6,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A parsed block from markdown content
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ParsedBlock {
|
||||
pub flavour: BlockFlavour,
|
||||
pub block_type: Option<BlockType>,
|
||||
pub content: String,
|
||||
pub checked: Option<bool>,
|
||||
pub language: Option<String>,
|
||||
}
|
||||
|
||||
/// Parses markdown content into a list of parsed blocks.
|
||||
///
|
||||
/// This is the shared parsing logic used by both `markdown_to_ydoc` and
|
||||
/// `update_ydoc`.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `markdown` - The markdown content to parse
|
||||
/// * `skip_first_h1` - If true, the first H1 heading is skipped (used as
|
||||
/// document title)
|
||||
///
|
||||
/// # Returns
|
||||
/// A vector of parsed blocks
|
||||
pub fn parse_markdown_blocks(markdown: &str, skip_first_h1: bool) -> Vec<ParsedBlock> {
|
||||
// Note: ENABLE_TABLES is included for future support, but table events
|
||||
// currently fall through to the catch-all match arm. Table content appears as
|
||||
// plain text.
|
||||
let options = Options::ENABLE_STRIKETHROUGH
|
||||
| Options::ENABLE_TABLES
|
||||
| Options::ENABLE_TASKLISTS
|
||||
| Options::ENABLE_HEADING_ATTRIBUTES;
|
||||
let parser = Parser::new_ext(markdown, options);
|
||||
|
||||
let mut blocks = Vec::new();
|
||||
let mut current_text = String::new();
|
||||
let mut current_type: Option<BlockType> = None;
|
||||
let mut current_flavour = BlockFlavour::Paragraph;
|
||||
let mut in_list = false;
|
||||
let mut list_type_stack: Vec<BlockType> = Vec::new();
|
||||
// Per-item type override for task list markers (resets at each Item start)
|
||||
let mut current_item_type: Option<BlockType> = None;
|
||||
let mut in_code_block = false;
|
||||
let mut code_language = String::new();
|
||||
let mut first_h1_seen = !skip_first_h1; // If not skipping, mark as already seen
|
||||
let mut current_checked: Option<bool> = None;
|
||||
let mut pending_link_url: Option<String> = None;
|
||||
|
||||
for event in parser {
|
||||
match event {
|
||||
Event::Start(Tag::Heading { level, .. }) => {
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
|
||||
if level == HeadingLevel::H1 && !first_h1_seen {
|
||||
// Skip the first H1 - it's used as the document title
|
||||
current_type = Some(BlockType::H1);
|
||||
} else {
|
||||
current_type = Some(BlockType::from_heading_level(level));
|
||||
}
|
||||
current_flavour = BlockFlavour::Paragraph;
|
||||
}
|
||||
Event::End(TagEnd::Heading(level)) => {
|
||||
if level == HeadingLevel::H1 && !first_h1_seen {
|
||||
first_h1_seen = true;
|
||||
current_text.clear();
|
||||
current_type = None;
|
||||
} else {
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
Event::Start(Tag::Paragraph) => {}
|
||||
Event::End(TagEnd::Paragraph) => {
|
||||
if !in_list {
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
Event::Start(Tag::BlockQuote(_)) => {
|
||||
current_type = Some(BlockType::Quote);
|
||||
current_flavour = BlockFlavour::Paragraph;
|
||||
}
|
||||
Event::End(TagEnd::BlockQuote(_)) => {
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
}
|
||||
Event::Start(Tag::List(start_num)) => {
|
||||
in_list = true;
|
||||
let list_type = if start_num.is_some() {
|
||||
BlockType::Numbered
|
||||
} else {
|
||||
BlockType::Bulleted
|
||||
};
|
||||
list_type_stack.push(list_type);
|
||||
}
|
||||
Event::End(TagEnd::List(_)) => {
|
||||
list_type_stack.pop();
|
||||
if list_type_stack.is_empty() {
|
||||
in_list = false;
|
||||
}
|
||||
}
|
||||
Event::Start(Tag::Item) => {
|
||||
current_flavour = BlockFlavour::List;
|
||||
// Reset per-item type override
|
||||
current_item_type = None;
|
||||
if let Some(lt) = list_type_stack.last() {
|
||||
current_type = Some(*lt);
|
||||
}
|
||||
}
|
||||
Event::End(TagEnd::Item) => {
|
||||
// Use per-item override if set (for task items), otherwise use current_type
|
||||
if let Some(item_type) = current_item_type.take() {
|
||||
current_type = Some(item_type);
|
||||
}
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
current_flavour = BlockFlavour::Paragraph;
|
||||
}
|
||||
Event::TaskListMarker(checked) => {
|
||||
// Set per-item type override for this specific item only
|
||||
current_item_type = Some(BlockType::Todo);
|
||||
current_checked = Some(checked);
|
||||
}
|
||||
Event::Start(Tag::CodeBlock(kind)) => {
|
||||
in_code_block = true;
|
||||
current_flavour = BlockFlavour::Code;
|
||||
code_language = match kind {
|
||||
CodeBlockKind::Fenced(lang) => lang.to_string(),
|
||||
CodeBlockKind::Indented => String::new(),
|
||||
};
|
||||
}
|
||||
Event::End(TagEnd::CodeBlock) => {
|
||||
flush_code_block(&mut blocks, &mut current_text, &code_language);
|
||||
in_code_block = false;
|
||||
code_language.clear();
|
||||
current_flavour = BlockFlavour::Paragraph;
|
||||
}
|
||||
Event::Text(text) => {
|
||||
current_text.push_str(&text);
|
||||
}
|
||||
Event::Code(code) => {
|
||||
// Inline code - wrap in backticks
|
||||
current_text.push('`');
|
||||
current_text.push_str(&code);
|
||||
current_text.push('`');
|
||||
}
|
||||
Event::SoftBreak | Event::HardBreak => {
|
||||
if in_code_block {
|
||||
current_text.push('\n');
|
||||
} else {
|
||||
current_text.push(' ');
|
||||
}
|
||||
}
|
||||
Event::Rule => {
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type.take(),
|
||||
current_checked.take(),
|
||||
None,
|
||||
);
|
||||
blocks.push(ParsedBlock {
|
||||
flavour: BlockFlavour::Divider,
|
||||
block_type: None,
|
||||
content: String::new(),
|
||||
checked: None,
|
||||
language: None,
|
||||
});
|
||||
}
|
||||
Event::Start(Tag::Strong) => current_text.push_str("**"),
|
||||
Event::End(TagEnd::Strong) => current_text.push_str("**"),
|
||||
Event::Start(Tag::Emphasis) => current_text.push('_'),
|
||||
Event::End(TagEnd::Emphasis) => current_text.push('_'),
|
||||
Event::Start(Tag::Strikethrough) => current_text.push_str("~~"),
|
||||
Event::End(TagEnd::Strikethrough) => current_text.push_str("~~"),
|
||||
Event::Start(Tag::Link { dest_url, .. }) => {
|
||||
current_text.push('[');
|
||||
pending_link_url = Some(dest_url.to_string());
|
||||
}
|
||||
Event::End(TagEnd::Link) => {
|
||||
if let Some(url) = pending_link_url.take() {
|
||||
current_text.push_str(&format!("]({})", url));
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush any remaining content
|
||||
flush_block(
|
||||
&mut blocks,
|
||||
&mut current_text,
|
||||
current_flavour,
|
||||
current_type,
|
||||
current_checked,
|
||||
None,
|
||||
);
|
||||
|
||||
blocks
|
||||
}
|
||||
|
||||
fn flush_block(
|
||||
blocks: &mut Vec<ParsedBlock>,
|
||||
text: &mut String,
|
||||
flavour: BlockFlavour,
|
||||
block_type: Option<BlockType>,
|
||||
checked: Option<bool>,
|
||||
language: Option<String>,
|
||||
) {
|
||||
let trimmed = text.trim();
|
||||
if !trimmed.is_empty() || flavour == BlockFlavour::Divider {
|
||||
blocks.push(ParsedBlock {
|
||||
flavour,
|
||||
block_type,
|
||||
content: trimmed.to_string(),
|
||||
checked,
|
||||
language,
|
||||
});
|
||||
}
|
||||
text.clear();
|
||||
}
|
||||
|
||||
fn flush_code_block(blocks: &mut Vec<ParsedBlock>, text: &mut String, language: &str) {
|
||||
// Preserve leading whitespace (indentation) in code blocks as it may be
|
||||
// semantically significant (e.g., Python, YAML). Only strip leading/trailing
|
||||
// newlines which are typically artifacts from code fence parsing.
|
||||
let content = text.trim_matches('\n');
|
||||
if !content.is_empty() {
|
||||
blocks.push(ParsedBlock {
|
||||
flavour: BlockFlavour::Code,
|
||||
block_type: None,
|
||||
content: content.to_string(),
|
||||
checked: None,
|
||||
language: if language.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(language.to_string())
|
||||
},
|
||||
});
|
||||
}
|
||||
text.clear();
|
||||
}
|
||||
|
||||
/// Extracts the title from the first H1 heading in markdown content.
|
||||
///
|
||||
/// Returns "Untitled" if no H1 heading is found.
|
||||
pub(crate) fn extract_title(markdown: &str) -> String {
|
||||
let parser = Parser::new(markdown);
|
||||
let mut in_heading = false;
|
||||
let mut title = String::new();
|
||||
|
||||
for event in parser {
|
||||
match event {
|
||||
Event::Start(Tag::Heading {
|
||||
level: HeadingLevel::H1,
|
||||
..
|
||||
}) => {
|
||||
in_heading = true;
|
||||
}
|
||||
Event::Text(text) if in_heading => {
|
||||
title.push_str(&text);
|
||||
}
|
||||
Event::Code(code) if in_heading => {
|
||||
title.push_str(&code);
|
||||
}
|
||||
Event::End(TagEnd::Heading(_)) if in_heading => {
|
||||
break;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if title.is_empty() {
|
||||
"Untitled".to_string()
|
||||
} else {
|
||||
title.trim().to_string()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_simple() {
|
||||
assert_eq!(extract_title("# Hello World\n\nContent"), "Hello World");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_with_code() {
|
||||
assert_eq!(extract_title("# Hello `code` World"), "Hello code World");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_empty() {
|
||||
assert_eq!(extract_title("No heading here"), "Untitled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_title_h2_not_used() {
|
||||
assert_eq!(extract_title("## H2 heading\n\nContent"), "Untitled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_simple() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\nParagraph text.", true);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert_eq!(blocks[0].flavour, BlockFlavour::Paragraph);
|
||||
assert_eq!(blocks[0].content, "Paragraph text.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_with_headings() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\n## Section\n\nText.", true);
|
||||
assert_eq!(blocks.len(), 2);
|
||||
assert_eq!(blocks[0].block_type, Some(BlockType::H2));
|
||||
assert_eq!(blocks[0].content, "Section");
|
||||
assert_eq!(blocks[1].content, "Text.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_lists() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\n- Item 1\n- Item 2", true);
|
||||
assert_eq!(blocks.len(), 2);
|
||||
assert_eq!(blocks[0].flavour, BlockFlavour::List);
|
||||
assert_eq!(blocks[0].block_type, Some(BlockType::Bulleted));
|
||||
assert_eq!(blocks[0].content, "Item 1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_task_list() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\n- [ ] Unchecked\n- [x] Checked", true);
|
||||
assert_eq!(blocks.len(), 2);
|
||||
assert_eq!(blocks[0].block_type, Some(BlockType::Todo));
|
||||
assert_eq!(blocks[0].checked, Some(false));
|
||||
assert_eq!(blocks[1].block_type, Some(BlockType::Todo));
|
||||
assert_eq!(blocks[1].checked, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_code() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\n```rust\nfn main() {}\n```", true);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert_eq!(blocks[0].flavour, BlockFlavour::Code);
|
||||
assert_eq!(blocks[0].language, Some("rust".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_divider() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\nBefore\n\n---\n\nAfter", true);
|
||||
assert_eq!(blocks.len(), 3);
|
||||
assert_eq!(blocks[1].flavour, BlockFlavour::Divider);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_markdown_blocks_code_preserves_indentation() {
|
||||
let blocks = parse_markdown_blocks("# Title\n\n```python\n def indented():\n pass\n```", true);
|
||||
assert_eq!(blocks.len(), 1);
|
||||
assert!(blocks[0].content.starts_with(" def"));
|
||||
}
|
||||
}
|
||||
@@ -1,19 +1,21 @@
|
||||
mod affine;
|
||||
mod block_spec;
|
||||
mod blocksuite;
|
||||
mod delta_markdown;
|
||||
#[cfg(feature = "ydoc-loader")]
|
||||
mod markdown_to_ydoc;
|
||||
#[cfg(feature = "ydoc-loader")]
|
||||
mod markdown_utils;
|
||||
#[cfg(feature = "ydoc-loader")]
|
||||
mod update_ydoc;
|
||||
mod doc_loader;
|
||||
mod error;
|
||||
mod markdown;
|
||||
mod read;
|
||||
#[cfg(test)]
|
||||
mod roundtrip_tests;
|
||||
mod schema;
|
||||
mod table;
|
||||
mod value;
|
||||
mod write;
|
||||
|
||||
pub use affine::{
|
||||
BlockInfo, CrawlResult, MarkdownResult, PageDocContent, ParseError, WorkspaceDocContent, add_doc_to_root_doc,
|
||||
get_doc_ids_from_binary, parse_doc_from_binary, parse_doc_to_markdown, parse_page_doc, parse_workspace_doc,
|
||||
pub use error::ParseError;
|
||||
pub use read::{
|
||||
BlockInfo, CrawlResult, MarkdownResult, PageDocContent, WorkspaceDocContent, get_doc_ids_from_binary,
|
||||
parse_doc_from_binary, parse_doc_to_markdown, parse_page_doc, parse_workspace_doc,
|
||||
};
|
||||
pub use write::{
|
||||
add_doc_to_root_doc, build_full_doc, update_doc, update_doc_properties, update_doc_title, update_root_doc_meta_title,
|
||||
};
|
||||
#[cfg(feature = "ydoc-loader")]
|
||||
pub use markdown_to_ydoc::markdown_to_ydoc;
|
||||
#[cfg(feature = "ydoc-loader")]
|
||||
pub use update_ydoc::update_ydoc;
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use y_octo::{Any, Map, Value};
|
||||
|
||||
use super::{text_content, text_content_for_summary};
|
||||
use crate::doc_parser::{
|
||||
blocksuite::{DocContext, collect_child_ids, get_string},
|
||||
markdown::{
|
||||
DeltaToMdOptions, InlineReferencePayload, delta_value_to_inline_markdown, extract_inline_references_from_value,
|
||||
text_to_inline_markdown,
|
||||
},
|
||||
table::{MarkdownTableOptions, render_markdown_table},
|
||||
value::{any_as_string, value_to_string},
|
||||
};
|
||||
|
||||
pub(super) struct DatabaseTable {
|
||||
pub(super) columns: Vec<DatabaseColumn>,
|
||||
pub(super) rows: Vec<Vec<String>>,
|
||||
}
|
||||
|
||||
pub(super) struct DatabaseOption {
|
||||
id: Option<String>,
|
||||
value: Option<String>,
|
||||
color: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) struct DatabaseColumn {
|
||||
pub(super) id: String,
|
||||
pub(super) name: Option<String>,
|
||||
pub(super) col_type: String,
|
||||
pub(super) options: Vec<DatabaseOption>,
|
||||
}
|
||||
|
||||
pub(super) fn build_database_table(
|
||||
block: &Map,
|
||||
context: &DocContext,
|
||||
md_options: &DeltaToMdOptions,
|
||||
) -> Option<DatabaseTable> {
|
||||
let columns = parse_database_columns(block)?;
|
||||
let cells_map = block.get("prop:cells").and_then(|v| v.to_map())?;
|
||||
let child_ids = collect_child_ids(block);
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for child_id in child_ids {
|
||||
let row_cells = cells_map.get(&child_id).and_then(|v| v.to_map());
|
||||
let mut row = Vec::new();
|
||||
|
||||
for column in columns.iter() {
|
||||
let mut cell_text = String::new();
|
||||
if column.col_type == "title" {
|
||||
if let Some(child_block) = context.block_pool.get(&child_id) {
|
||||
if let Some(text_md) = text_to_inline_markdown(child_block, "prop:text", md_options) {
|
||||
cell_text = text_md;
|
||||
} else if let Some((text, _)) = text_content(child_block, "prop:text") {
|
||||
cell_text = text;
|
||||
} else if let Some((text, _)) = text_content_for_summary(child_block, "prop:text") {
|
||||
cell_text = text;
|
||||
}
|
||||
}
|
||||
} else if let Some(row_cells) = &row_cells
|
||||
&& let Some(cell_val) = row_cells.get(&column.id).and_then(|v| v.to_map())
|
||||
&& let Some(value) = cell_val.get("value")
|
||||
{
|
||||
if let Some(text_md) = delta_value_to_inline_markdown(&value, md_options) {
|
||||
cell_text = text_md;
|
||||
} else {
|
||||
cell_text = format_cell_value(&value, column);
|
||||
}
|
||||
}
|
||||
|
||||
row.push(cell_text);
|
||||
}
|
||||
if row.iter().any(|cell| !cell.is_empty()) {
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
Some(DatabaseTable { columns, rows })
|
||||
}
|
||||
|
||||
pub(super) fn database_table_markdown(table: DatabaseTable) -> Option<String> {
|
||||
let mut rows = Vec::with_capacity(table.rows.len() + 1);
|
||||
let header = table
|
||||
.columns
|
||||
.iter()
|
||||
.map(|column| column.name.as_deref().unwrap_or_default().to_string())
|
||||
.collect::<Vec<_>>();
|
||||
rows.push(header);
|
||||
rows.extend(table.rows);
|
||||
|
||||
let options = MarkdownTableOptions::new(false, "<br />", true);
|
||||
render_markdown_table(&rows, options)
|
||||
}
|
||||
|
||||
pub(super) fn database_summary_text(block: &Map, context: &DocContext) -> Option<String> {
|
||||
let md_options = DeltaToMdOptions::new(None);
|
||||
let table = build_database_table(block, context, &md_options)?;
|
||||
let mut summary = String::new();
|
||||
|
||||
if let Some(title) = get_string(block, "prop:title")
|
||||
&& !title.is_empty()
|
||||
{
|
||||
summary.push_str(&title);
|
||||
summary.push('|');
|
||||
}
|
||||
|
||||
for column in table.columns.iter() {
|
||||
if let Some(name) = column.name.as_ref()
|
||||
&& !name.is_empty()
|
||||
{
|
||||
summary.push_str(name);
|
||||
summary.push('|');
|
||||
}
|
||||
for option in column.options.iter() {
|
||||
if let Some(value) = option.value.as_ref()
|
||||
&& !value.is_empty()
|
||||
{
|
||||
summary.push_str(value);
|
||||
summary.push('|');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for row in table.rows.iter() {
|
||||
for cell_text in row.iter() {
|
||||
if !cell_text.is_empty() {
|
||||
summary.push_str(cell_text);
|
||||
summary.push('|');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if summary.is_empty() { None } else { Some(summary) }
|
||||
}
|
||||
|
||||
pub(super) fn gather_database_texts(block: &Map) -> (Vec<String>, Option<String>) {
|
||||
let mut texts = Vec::new();
|
||||
let database_title = get_string(block, "prop:title");
|
||||
if let Some(title) = &database_title {
|
||||
texts.push(title.clone());
|
||||
}
|
||||
|
||||
if let Some(columns) = parse_database_columns(block) {
|
||||
for column in columns.iter() {
|
||||
if let Some(name) = column.name.as_ref() {
|
||||
texts.push(name.clone());
|
||||
}
|
||||
for option in column.options.iter() {
|
||||
if let Some(value) = option.value.as_ref() {
|
||||
texts.push(value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(texts, database_title)
|
||||
}
|
||||
|
||||
pub(super) fn collect_database_cell_references(block: &Map) -> Vec<InlineReferencePayload> {
|
||||
let cells_map = match block.get("prop:cells").and_then(|value| value.to_map()) {
|
||||
Some(map) => map,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
|
||||
let mut refs = Vec::new();
|
||||
let mut seen: HashSet<(String, String)> = HashSet::new();
|
||||
|
||||
for row in cells_map.values() {
|
||||
let Some(row_map) = row.to_map() else {
|
||||
continue;
|
||||
};
|
||||
for cell in row_map.values() {
|
||||
let Some(cell_map) = cell.to_map() else {
|
||||
continue;
|
||||
};
|
||||
let Some(value) = cell_map.get("value") else {
|
||||
continue;
|
||||
};
|
||||
for reference in extract_inline_references_from_value(&value) {
|
||||
let key = (reference.doc_id.clone(), reference.payload.clone());
|
||||
if seen.insert(key) {
|
||||
refs.push(reference);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
refs
|
||||
}
|
||||
|
||||
fn parse_database_columns(block: &Map) -> Option<Vec<DatabaseColumn>> {
|
||||
let columns = block.get("prop:columns").and_then(|value| value.to_array())?;
|
||||
let mut parsed = Vec::new();
|
||||
for column_value in columns.iter() {
|
||||
if let Some(column) = column_value.to_map() {
|
||||
let id = column
|
||||
.get("id")
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
let name = column.get("name").and_then(|value| value_to_string(&value));
|
||||
let col_type = column
|
||||
.get("type")
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.unwrap_or_default();
|
||||
let options = parse_database_options(&column);
|
||||
parsed.push(DatabaseColumn {
|
||||
id,
|
||||
name,
|
||||
col_type,
|
||||
options,
|
||||
});
|
||||
}
|
||||
}
|
||||
Some(parsed)
|
||||
}
|
||||
|
||||
fn parse_database_options(column: &Map) -> Vec<DatabaseOption> {
|
||||
let Some(data) = column.get("data").and_then(|value| value.to_map()) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Some(options) = data.get("options").and_then(|value| value.to_array()) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut parsed = Vec::new();
|
||||
for option_value in options.iter() {
|
||||
if let Some(option) = option_value.to_map() {
|
||||
parsed.push(DatabaseOption {
|
||||
id: option.get("id").and_then(|value| value_to_string(&value)),
|
||||
value: option.get("value").and_then(|value| value_to_string(&value)),
|
||||
color: option.get("color").and_then(|value| value_to_string(&value)),
|
||||
});
|
||||
}
|
||||
}
|
||||
parsed
|
||||
}
|
||||
|
||||
fn format_option_tag(option: &DatabaseOption) -> String {
|
||||
let id = option.id.as_deref().unwrap_or_default();
|
||||
let value = option.value.as_deref().unwrap_or_default();
|
||||
let color = option.color.as_deref().unwrap_or_default();
|
||||
|
||||
format!("<span data-affine-option data-value=\"{id}\" data-option-color=\"{color}\">{value}</span>")
|
||||
}
|
||||
|
||||
fn format_cell_value(value: &Value, column: &DatabaseColumn) -> String {
|
||||
match column.col_type.as_str() {
|
||||
"select" => {
|
||||
let id = match value {
|
||||
Value::Any(any) => any_as_string(any).map(str::to_string),
|
||||
Value::Text(text) => Some(text.to_string()),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(id) = id {
|
||||
for option in column.options.iter() {
|
||||
if option.id.as_deref() == Some(id.as_str()) {
|
||||
return format_option_tag(option);
|
||||
}
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
"multi-select" => {
|
||||
let ids: Vec<String> = match value {
|
||||
Value::Any(Any::Array(ids)) => ids.iter().filter_map(any_as_string).map(str::to_string).collect(),
|
||||
Value::Array(array) => array.iter().filter_map(|id_val| value_to_string(&id_val)).collect(),
|
||||
_ => Vec::new(),
|
||||
};
|
||||
|
||||
if ids.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let mut selected = Vec::new();
|
||||
for id in ids.iter() {
|
||||
for option in column.options.iter() {
|
||||
if option.id.as_deref() == Some(id.as_str()) {
|
||||
selected.push(format_option_tag(option));
|
||||
}
|
||||
}
|
||||
}
|
||||
selected.join("")
|
||||
}
|
||||
_ => value_to_string(value).unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,795 @@
|
||||
mod database;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map as JsonMap, Value as JsonValue};
|
||||
use y_octo::{Any, Map};
|
||||
|
||||
use self::database::{
|
||||
build_database_table, collect_database_cell_references, database_summary_text, database_table_markdown,
|
||||
gather_database_texts,
|
||||
};
|
||||
use super::{
|
||||
ParseError,
|
||||
block_spec::{BlockFlavour, BlockSpec, ImageSpec},
|
||||
blocksuite::{DocContext, get_block_id, get_flavour, get_list_depth, get_string, nearest_by_flavour},
|
||||
doc_loader::load_doc,
|
||||
markdown::{DeltaToMdOptions, MarkdownRenderer, MarkdownWriter, extract_inline_references},
|
||||
schema::{NOTE_FLAVOUR, PAGE_FLAVOUR},
|
||||
value::{any_as_string, any_truthy, build_reference_payload, params_value_to_json, value_to_string},
|
||||
};
|
||||
|
||||
const SUMMARY_LIMIT: usize = 1000;
|
||||
const DEFAULT_PAGE_TITLE: &str = "Untitled";
|
||||
|
||||
const BOOKMARK_FLAVOURS: [&str; 6] = [
|
||||
"affine:bookmark",
|
||||
"affine:embed-youtube",
|
||||
"affine:embed-iframe",
|
||||
"affine:embed-figma",
|
||||
"affine:embed-github",
|
||||
"affine:embed-loom",
|
||||
];
|
||||
|
||||
struct SummaryBuilder {
|
||||
summary: String,
|
||||
remaining: Option<isize>,
|
||||
}
|
||||
|
||||
impl SummaryBuilder {
|
||||
fn new(limit: isize) -> Self {
|
||||
let remaining = if limit < 0 { None } else { Some(limit) };
|
||||
Self {
|
||||
summary: String::new(),
|
||||
remaining,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_unlimited(&self) -> bool {
|
||||
self.remaining.is_none()
|
||||
}
|
||||
|
||||
fn push_text(&mut self, text: &str, len: usize) {
|
||||
match self.remaining {
|
||||
None => self.summary.push_str(text),
|
||||
Some(remaining) if remaining > 0 => {
|
||||
self.summary.push_str(text);
|
||||
self.remaining = Some(remaining - len as isize);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn push_raw(&mut self, text: &str) {
|
||||
self.summary.push_str(text);
|
||||
}
|
||||
|
||||
fn into_string(self) -> String {
|
||||
self.summary
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BlockInfo {
|
||||
pub block_id: String,
|
||||
pub flavour: String,
|
||||
pub content: Option<Vec<String>>,
|
||||
pub blob: Option<Vec<String>>,
|
||||
pub ref_doc_id: Option<Vec<String>>,
|
||||
pub ref_info: Option<Vec<String>>,
|
||||
pub parent_flavour: Option<String>,
|
||||
pub parent_block_id: Option<String>,
|
||||
pub additional: Option<String>,
|
||||
}
|
||||
|
||||
impl BlockInfo {
|
||||
fn base(
|
||||
block_id: &str,
|
||||
flavour: &str,
|
||||
parent_flavour: Option<&String>,
|
||||
parent_block_id: Option<&String>,
|
||||
additional: Option<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
block_id: block_id.to_string(),
|
||||
flavour: flavour.to_string(),
|
||||
content: None,
|
||||
blob: None,
|
||||
ref_doc_id: None,
|
||||
ref_info: None,
|
||||
parent_flavour: parent_flavour.cloned(),
|
||||
parent_block_id: parent_block_id.cloned(),
|
||||
additional,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CrawlResult {
|
||||
pub blocks: Vec<BlockInfo>,
|
||||
pub title: String,
|
||||
pub summary: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PageDocContent {
|
||||
pub title: String,
|
||||
pub summary: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WorkspaceDocContent {
|
||||
pub name: String,
|
||||
#[serde(rename = "avatarKey")]
|
||||
pub avatar_key: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MarkdownResult {
|
||||
pub title: String,
|
||||
pub markdown: String,
|
||||
}
|
||||
|
||||
pub fn parse_workspace_doc(doc_bin: Vec<u8>) -> Result<Option<WorkspaceDocContent>, ParseError> {
|
||||
let doc = load_doc(&doc_bin, None)?;
|
||||
|
||||
let meta = match doc.get_map("meta") {
|
||||
Ok(meta) => meta,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
|
||||
let name = get_string(&meta, "name").unwrap_or_default();
|
||||
let avatar_key = get_string(&meta, "avatar").unwrap_or_default();
|
||||
|
||||
Ok(Some(WorkspaceDocContent { name, avatar_key }))
|
||||
}
|
||||
|
||||
pub fn parse_page_doc(
|
||||
doc_bin: Vec<u8>,
|
||||
max_summary_length: Option<isize>,
|
||||
) -> Result<Option<PageDocContent>, ParseError> {
|
||||
let doc = load_doc(&doc_bin, None)?;
|
||||
|
||||
let blocks_map = match doc.get_map("blocks") {
|
||||
Ok(map) => map,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
|
||||
if blocks_map.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let Some(context) = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let mut walker = context.walker();
|
||||
let mut content = PageDocContent {
|
||||
title: context
|
||||
.block_pool
|
||||
.get(&context.root_block_id)
|
||||
.and_then(|block| get_string(block, "prop:title"))
|
||||
.unwrap_or_default(),
|
||||
summary: String::new(),
|
||||
};
|
||||
let mut summary = SummaryBuilder::new(max_summary_length.unwrap_or(150));
|
||||
|
||||
while let Some((_parent_block_id, block_id)) = walker.next() {
|
||||
let Some(block) = context.block_pool.get(&block_id) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let Some(flavour) = get_flavour(block) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match flavour.as_str() {
|
||||
"affine:page" | "affine:note" => {
|
||||
walker.enqueue_children(&block_id, block);
|
||||
}
|
||||
"affine:attachment" | "affine:transcription" | "affine:callout" => {
|
||||
if summary.is_unlimited() {
|
||||
walker.enqueue_children(&block_id, block);
|
||||
}
|
||||
}
|
||||
"affine:database" => {
|
||||
if summary.is_unlimited()
|
||||
&& let Some(text) = database_summary_text(block, &context)
|
||||
{
|
||||
summary.push_raw(&text);
|
||||
}
|
||||
}
|
||||
"affine:table" => {
|
||||
if summary.is_unlimited() {
|
||||
let contents = table_cell_texts(block);
|
||||
if !contents.is_empty() {
|
||||
summary.push_raw(&contents.join("|"));
|
||||
}
|
||||
}
|
||||
}
|
||||
"affine:paragraph" | "affine:list" | "affine:code" => {
|
||||
walker.enqueue_children(&block_id, block);
|
||||
if let Some((text, len)) = text_content_for_summary(block, "prop:text") {
|
||||
summary.push_text(&text, len);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
content.summary = summary.into_string();
|
||||
Ok(Some(content))
|
||||
}
|
||||
|
||||
pub fn parse_doc_to_markdown(
|
||||
doc_bin: Vec<u8>,
|
||||
doc_id: String,
|
||||
ai_editable: bool,
|
||||
doc_url_prefix: Option<String>,
|
||||
) -> Result<MarkdownResult, ParseError> {
|
||||
let doc = load_doc(&doc_bin, Some(doc_id.as_str()))?;
|
||||
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Ok(MarkdownResult {
|
||||
title: "".into(),
|
||||
markdown: "".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("root block not found".into()))?;
|
||||
let root_block_id = context.root_block_id.clone();
|
||||
let mut walker = context.walker();
|
||||
let mut doc_title = String::from(DEFAULT_PAGE_TITLE);
|
||||
let mut markdown = String::new();
|
||||
let md_options = DeltaToMdOptions::new(doc_url_prefix);
|
||||
let renderer = MarkdownRenderer::new(&md_options);
|
||||
|
||||
while let Some((_parent_block_id, block_id)) = walker.next() {
|
||||
let block = match context.block_pool.get(&block_id) {
|
||||
Some(block) => block,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flavour = match get_flavour(block) {
|
||||
Some(flavour) => flavour,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let parent_id = context.parent_lookup.get(&block_id);
|
||||
let parent_flavour = parent_id
|
||||
.and_then(|id| context.block_pool.get(id))
|
||||
.and_then(get_flavour);
|
||||
|
||||
if parent_flavour.as_deref() == Some("affine:database") {
|
||||
continue;
|
||||
}
|
||||
|
||||
// enqueue children first to keep traversal order similar to JS implementation
|
||||
walker.enqueue_children(&block_id, block);
|
||||
|
||||
if flavour == PAGE_FLAVOUR {
|
||||
let title = get_string(block, "prop:title").unwrap_or_default();
|
||||
doc_title = title.clone();
|
||||
continue;
|
||||
}
|
||||
|
||||
let block_level = if ai_editable {
|
||||
block_level(&block_id, &root_block_id, &context.parent_lookup)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let ai_block = ai_editable && block_level == 2;
|
||||
|
||||
let mut block_markdown = String::new();
|
||||
|
||||
match flavour.as_str() {
|
||||
"affine:database" => {
|
||||
let title = get_string(block, "prop:title").unwrap_or_default();
|
||||
block_markdown.push_str(&format!("\n### {title}\n"));
|
||||
|
||||
if let Some(table) = build_database_table(block, &context, &md_options)
|
||||
&& let Some(table_md) = database_table_markdown(table)
|
||||
{
|
||||
let mut writer = MarkdownWriter::new(&mut block_markdown);
|
||||
writer.push_table(&table_md);
|
||||
}
|
||||
}
|
||||
"affine:note" | "affine:surface" | "affine:frame" => {}
|
||||
_ => {
|
||||
if let Some(block_flavour) = BlockFlavour::from_str(flavour.as_str()) {
|
||||
let spec = BlockSpec::from_block_map_with_flavour(block, block_flavour);
|
||||
let list_depth = if block_flavour == BlockFlavour::List {
|
||||
get_list_depth(&block_id, &context.parent_lookup, &context.block_pool)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
renderer.write_block(&mut block_markdown, &spec, list_depth);
|
||||
} else {
|
||||
return Err(ParseError::ParserError(format!("unsupported_block_flavour:{flavour}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ai_block {
|
||||
markdown.push_str(&format!("<!-- block_id={block_id} flavour={flavour} -->\n"));
|
||||
}
|
||||
markdown.push_str(&block_markdown);
|
||||
}
|
||||
|
||||
Ok(MarkdownResult {
|
||||
title: doc_title,
|
||||
markdown,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlResult, ParseError> {
|
||||
let doc = load_doc(&doc_bin, Some(doc_id.as_str()))?;
|
||||
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Err(ParseError::ParserError("blocks map is empty".into()));
|
||||
}
|
||||
|
||||
let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("root block not found".into()))?;
|
||||
let mut walker = context.walker();
|
||||
let mut blocks: Vec<BlockInfo> = Vec::with_capacity(context.block_pool.len());
|
||||
let mut doc_title = String::new();
|
||||
let mut summary = SummaryBuilder::new(SUMMARY_LIMIT as isize);
|
||||
|
||||
while let Some((parent_block_id, block_id)) = walker.next() {
|
||||
let block = match context.block_pool.get(&block_id) {
|
||||
Some(block) => block,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flavour = match get_flavour(block) {
|
||||
Some(flavour) => flavour,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let parent_block = parent_block_id.as_ref().and_then(|id| context.block_pool.get(id));
|
||||
let parent_flavour = parent_block.and_then(get_flavour);
|
||||
|
||||
let note_block = nearest_by_flavour(&block_id, NOTE_FLAVOUR, &context.parent_lookup, &context.block_pool);
|
||||
let note_block_id = note_block.as_ref().and_then(get_block_id);
|
||||
let display_mode = determine_display_mode(note_block.as_ref());
|
||||
|
||||
// enqueue children first to keep traversal order similar to JS implementation
|
||||
walker.enqueue_children(&block_id, block);
|
||||
|
||||
let build_block = |database_name: Option<&String>| {
|
||||
BlockInfo::base(
|
||||
&block_id,
|
||||
&flavour,
|
||||
parent_flavour.as_ref(),
|
||||
parent_block_id.as_ref(),
|
||||
compose_additional(&display_mode, note_block_id.as_ref(), database_name),
|
||||
)
|
||||
};
|
||||
|
||||
if flavour == PAGE_FLAVOUR {
|
||||
let title = get_string(block, "prop:title").unwrap_or_default();
|
||||
doc_title = title.clone();
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(vec![title]);
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if matches!(flavour.as_str(), "affine:paragraph" | "affine:list" | "affine:code") {
|
||||
if let Some(text) = block.get("prop:text").and_then(|value| value.to_text()) {
|
||||
let database_name = if flavour == "affine:paragraph" && parent_flavour.as_deref() == Some("affine:database") {
|
||||
parent_block.and_then(|map| get_string(map, "prop:title"))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let content = text.to_string();
|
||||
let text_len = text.len() as usize;
|
||||
let refs = extract_inline_references(&text.to_delta());
|
||||
|
||||
let mut info = build_block(database_name.as_ref());
|
||||
info.content = Some(vec![content.clone()]);
|
||||
if !refs.is_empty() {
|
||||
info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect());
|
||||
info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect());
|
||||
}
|
||||
blocks.push(info);
|
||||
summary.push_text(&content, text_len);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if matches!(flavour.as_str(), "affine:embed-linked-doc" | "affine:embed-synced-doc") {
|
||||
if let Some(page_id) = get_string(block, "prop:pageId") {
|
||||
let mut info = build_block(None);
|
||||
let payload = embed_ref_payload(block, &page_id);
|
||||
apply_doc_ref(&mut info, page_id, payload);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:attachment" {
|
||||
if let Some(blob_id) = get_string(block, "prop:sourceId") {
|
||||
let mut info = build_block(None);
|
||||
let name = get_string(block, "prop:name").unwrap_or_default();
|
||||
apply_blob_info(&mut info, blob_id, name);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:image" {
|
||||
let image = ImageSpec::from_block_map(block);
|
||||
if !image.source_id.is_empty() {
|
||||
let mut info = build_block(None);
|
||||
let caption = image.caption.unwrap_or_default();
|
||||
apply_blob_info(&mut info, image.source_id, caption);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:surface" {
|
||||
let texts = gather_surface_texts(block);
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(texts);
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:database" {
|
||||
let (texts, database_name) = gather_database_texts(block);
|
||||
let mut info = BlockInfo::base(
|
||||
&block_id,
|
||||
&flavour,
|
||||
parent_flavour.as_ref(),
|
||||
parent_block_id.as_ref(),
|
||||
compose_additional(&display_mode, note_block_id.as_ref(), database_name.as_ref()),
|
||||
);
|
||||
info.content = Some(texts);
|
||||
let refs = collect_database_cell_references(block);
|
||||
if !refs.is_empty() {
|
||||
info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect());
|
||||
info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect());
|
||||
}
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:latex" {
|
||||
if let Some(content) = get_string(block, "prop:latex") {
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(vec![content]);
|
||||
blocks.push(info);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if flavour == "affine:table" {
|
||||
let contents = table_cell_texts(block);
|
||||
let mut info = build_block(None);
|
||||
info.content = Some(contents);
|
||||
blocks.push(info);
|
||||
continue;
|
||||
}
|
||||
|
||||
if BOOKMARK_FLAVOURS.contains(&flavour.as_str()) {
|
||||
blocks.push(build_block(None));
|
||||
}
|
||||
}
|
||||
|
||||
if doc_title.is_empty() {
|
||||
doc_title = DEFAULT_PAGE_TITLE.into();
|
||||
}
|
||||
|
||||
Ok(CrawlResult {
|
||||
blocks,
|
||||
title: doc_title,
|
||||
summary: summary.into_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_doc_ids_from_binary(doc_bin: Vec<u8>, include_trash: bool) -> Result<Vec<String>, ParseError> {
|
||||
let doc = load_doc(&doc_bin, None)?;
|
||||
|
||||
let mut doc_ids = Vec::new();
|
||||
let meta = doc.get_map("meta")?;
|
||||
let pages_value = meta.get("pages");
|
||||
if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) {
|
||||
for page_val in pages.iter() {
|
||||
if let Some(page) = page_val.to_map() {
|
||||
let id = get_string(&page, "id");
|
||||
if let Some(id) = id {
|
||||
let trash = page
|
||||
.get("trash")
|
||||
.and_then(|v| match v.to_any() {
|
||||
Some(Any::True) => Some(true),
|
||||
Some(Any::False) => Some(false),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or(false);
|
||||
|
||||
if include_trash || !trash {
|
||||
doc_ids.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Ok(doc_ids);
|
||||
}
|
||||
|
||||
if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) {
|
||||
for entry in entries {
|
||||
let Any::Object(map) = entry else {
|
||||
continue;
|
||||
};
|
||||
let id = map.get("id").and_then(any_as_string).map(str::to_string);
|
||||
if let Some(id) = id {
|
||||
let trash = map.get("trash").map(any_truthy).unwrap_or(false);
|
||||
if include_trash || !trash {
|
||||
doc_ids.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(doc_ids)
|
||||
}
|
||||
|
||||
fn block_level(block_id: &str, root_id: &str, parent_lookup: &HashMap<String, String>) -> usize {
|
||||
let mut level = 0;
|
||||
let mut cursor = block_id;
|
||||
while let Some(parent) = parent_lookup.get(cursor) {
|
||||
level += 1;
|
||||
if parent == root_id {
|
||||
break;
|
||||
}
|
||||
cursor = parent;
|
||||
}
|
||||
level
|
||||
}
|
||||
|
||||
pub(super) fn text_content(block: &Map, key: &str) -> Option<(String, usize)> {
|
||||
block.get(key).and_then(|value| {
|
||||
value.to_text().map(|text| {
|
||||
let content = text.to_string();
|
||||
let len = text.len() as usize;
|
||||
(content, len)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn determine_display_mode(note_block: Option<&Map>) -> String {
|
||||
match note_block.and_then(|block| get_string(block, "prop:displayMode")) {
|
||||
Some(mode) if mode == "both" => "page".into(),
|
||||
Some(mode) => mode,
|
||||
None => "edgeless".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn compose_additional(
|
||||
display_mode: &str,
|
||||
note_block_id: Option<&String>,
|
||||
database_name: Option<&String>,
|
||||
) -> Option<String> {
|
||||
let mut payload = JsonMap::new();
|
||||
payload.insert("displayMode".into(), JsonValue::String(display_mode.to_string()));
|
||||
if let Some(note_id) = note_block_id {
|
||||
payload.insert("noteBlockId".into(), JsonValue::String(note_id.clone()));
|
||||
}
|
||||
if let Some(name) = database_name {
|
||||
payload.insert("databaseName".into(), JsonValue::String(name.clone()));
|
||||
}
|
||||
Some(JsonValue::Object(payload).to_string())
|
||||
}
|
||||
|
||||
fn apply_blob_info(info: &mut BlockInfo, blob_id: String, content: String) {
|
||||
info.blob = Some(vec![blob_id]);
|
||||
info.content = Some(vec![content]);
|
||||
}
|
||||
|
||||
fn apply_doc_ref(info: &mut BlockInfo, page_id: String, payload: Option<String>) {
|
||||
info.ref_doc_id = Some(vec![page_id]);
|
||||
if let Some(payload) = payload {
|
||||
info.ref_info = Some(vec![payload]);
|
||||
}
|
||||
}
|
||||
|
||||
fn embed_ref_payload(block: &Map, page_id: &str) -> Option<String> {
|
||||
let params = block.get("prop:params").as_ref().and_then(params_value_to_json);
|
||||
Some(build_reference_payload(page_id, params))
|
||||
}
|
||||
|
||||
fn gather_surface_texts(block: &Map) -> Vec<String> {
|
||||
let mut texts = Vec::new();
|
||||
let elements = match block.get("prop:elements").and_then(|value| value.to_map()) {
|
||||
Some(map) => map,
|
||||
None => return texts,
|
||||
};
|
||||
|
||||
if elements
|
||||
.get("type")
|
||||
.and_then(|value| value_to_string(&value))
|
||||
.as_deref()
|
||||
!= Some("$blocksuite:internal:native$")
|
||||
{
|
||||
return texts;
|
||||
}
|
||||
|
||||
if let Some(value_map) = elements.get("value").and_then(|value| value.to_map()) {
|
||||
for value in value_map.values() {
|
||||
if let Some(element) = value.to_map()
|
||||
&& let Some(text) = element.get("text").and_then(|value| value.to_text())
|
||||
{
|
||||
texts.push(text.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
texts.sort();
|
||||
texts
|
||||
}
|
||||
|
||||
fn table_cell_texts(block: &Map) -> Vec<String> {
|
||||
let mut contents = Vec::new();
|
||||
for key in block.keys() {
|
||||
if key.starts_with("prop:cells.")
|
||||
&& key.ends_with(".text")
|
||||
&& let Some(value) = block.get(key).and_then(|value| value_to_string(&value))
|
||||
&& !value.is_empty()
|
||||
{
|
||||
contents.push(value);
|
||||
}
|
||||
}
|
||||
contents
|
||||
}
|
||||
|
||||
pub(super) fn text_content_for_summary(block: &Map, key: &str) -> Option<(String, usize)> {
|
||||
if let Some((text, len)) = text_content(block, key) {
|
||||
return Some((text, len));
|
||||
}
|
||||
|
||||
block.get(key).and_then(|value| {
|
||||
value_to_string(&value).map(|text| {
|
||||
let len = text.chars().count();
|
||||
(text, len)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
use y_octo::{AHashMap, Any, DocOptions, TextAttributes, TextDeltaOp, TextInsert, Value};
|
||||
|
||||
use super::*;
|
||||
use crate::doc_parser::build_full_doc;
|
||||
|
||||
#[test]
|
||||
fn test_parse_doc_from_binary() {
|
||||
let json = include_bytes!("../../../fixtures/demo.ydoc.json");
|
||||
let doc_bin = include_bytes!("../../../fixtures/demo.ydoc").to_vec();
|
||||
let doc_id = "dYpV7PPhk8amRkY5IAcVO".to_string();
|
||||
|
||||
let result = parse_doc_from_binary(doc_bin, doc_id).unwrap();
|
||||
let config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict)
|
||||
.numeric_mode(assert_json_diff::NumericMode::AssumeFloat);
|
||||
assert_json_diff::assert_json_matches!(
|
||||
serde_json::from_slice::<serde_json::Value>(json).unwrap(),
|
||||
serde_json::json!(result),
|
||||
config
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_database_cell_references() {
|
||||
let doc_id = "doc-with-db".to_string();
|
||||
let doc = DocOptions::new().with_guid(doc_id.clone()).build();
|
||||
let mut blocks = doc.get_or_create_map("blocks").unwrap();
|
||||
|
||||
let mut page = doc.create_map().unwrap();
|
||||
page.insert("sys:id".into(), "page").unwrap();
|
||||
page.insert("sys:flavour".into(), "affine:page").unwrap();
|
||||
let mut page_children = doc.create_array().unwrap();
|
||||
page_children.push("note").unwrap();
|
||||
page.insert("sys:children".into(), Value::Array(page_children)).unwrap();
|
||||
let mut page_title = doc.create_text().unwrap();
|
||||
page_title.insert(0, "Page").unwrap();
|
||||
page.insert("prop:title".into(), Value::Text(page_title)).unwrap();
|
||||
blocks.insert("page".into(), Value::Map(page)).unwrap();
|
||||
|
||||
let mut note = doc.create_map().unwrap();
|
||||
note.insert("sys:id".into(), "note").unwrap();
|
||||
note.insert("sys:flavour".into(), "affine:note").unwrap();
|
||||
let mut note_children = doc.create_array().unwrap();
|
||||
note_children.push("db").unwrap();
|
||||
note.insert("sys:children".into(), Value::Array(note_children)).unwrap();
|
||||
note.insert("prop:displayMode".into(), "page").unwrap();
|
||||
blocks.insert("note".into(), Value::Map(note)).unwrap();
|
||||
|
||||
let mut db = doc.create_map().unwrap();
|
||||
db.insert("sys:id".into(), "db").unwrap();
|
||||
db.insert("sys:flavour".into(), "affine:database").unwrap();
|
||||
db.insert("sys:children".into(), Value::Array(doc.create_array().unwrap()))
|
||||
.unwrap();
|
||||
let mut db_title = doc.create_text().unwrap();
|
||||
db_title.insert(0, "Database").unwrap();
|
||||
db.insert("prop:title".into(), Value::Text(db_title)).unwrap();
|
||||
|
||||
let mut columns = doc.create_array().unwrap();
|
||||
let mut column = doc.create_map().unwrap();
|
||||
column.insert("id".into(), "col1").unwrap();
|
||||
column.insert("name".into(), "Text").unwrap();
|
||||
column.insert("type".into(), "rich-text").unwrap();
|
||||
column
|
||||
.insert("data".into(), Value::Map(doc.create_map().unwrap()))
|
||||
.unwrap();
|
||||
columns.push(Value::Map(column)).unwrap();
|
||||
db.insert("prop:columns".into(), Value::Array(columns)).unwrap();
|
||||
|
||||
let mut cell_text = doc.create_text().unwrap();
|
||||
let mut reference = AHashMap::default();
|
||||
reference.insert("pageId".into(), Any::String("target-doc".into()));
|
||||
let mut params = AHashMap::default();
|
||||
params.insert("mode".into(), Any::String("page".into()));
|
||||
reference.insert("params".into(), Any::Object(params));
|
||||
let mut attrs = TextAttributes::new();
|
||||
attrs.insert("reference".into(), Any::Object(reference));
|
||||
cell_text
|
||||
.apply_delta(&[
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("See ".into()),
|
||||
format: None,
|
||||
},
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text("Target".into()),
|
||||
format: Some(attrs),
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let mut cell = doc.create_map().unwrap();
|
||||
cell.insert("columnId".into(), "col1").unwrap();
|
||||
cell.insert("value".into(), Value::Text(cell_text)).unwrap();
|
||||
let mut row = doc.create_map().unwrap();
|
||||
row.insert("col1".into(), Value::Map(cell)).unwrap();
|
||||
let mut cells = doc.create_map().unwrap();
|
||||
cells.insert("row1".into(), Value::Map(row)).unwrap();
|
||||
db.insert("prop:cells".into(), Value::Map(cells)).unwrap();
|
||||
|
||||
blocks.insert("db".into(), Value::Map(db)).unwrap();
|
||||
|
||||
let doc_bin = doc.encode_update_v1().unwrap();
|
||||
let result = parse_doc_from_binary(doc_bin, doc_id).unwrap();
|
||||
let db_block = result.blocks.iter().find(|block| block.block_id == "db").unwrap();
|
||||
assert_eq!(db_block.ref_doc_id, Some(vec!["target-doc".to_string()]));
|
||||
assert_eq!(
|
||||
db_block.ref_info,
|
||||
Some(vec![build_reference_payload(
|
||||
"target-doc",
|
||||
Some(json!({"mode": "page"}))
|
||||
)])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_doc_to_markdown_ai_editable_image_table() {
|
||||
let doc_id = "ai-editable-doc";
|
||||
let markdown = "\n\n| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let doc_bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let result = parse_doc_to_markdown(doc_bin, doc_id.to_string(), true, None).expect("parse doc");
|
||||
let md = result.markdown;
|
||||
|
||||
assert!(md.contains("flavour=affine:image"));
|
||||
assert!(md.contains("blob://image-id"));
|
||||
assert!(md.contains("|A|B|"));
|
||||
assert!(md.contains("|---|---|"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
use super::{build_full_doc, parse_doc_to_markdown};
|
||||
|
||||
fn assert_markdown_roundtrip(markdown: &str, expected: &str) {
|
||||
let doc_id = "roundtrip-doc";
|
||||
let title = "Roundtrip Title";
|
||||
let bin = build_full_doc(title, markdown, doc_id).expect("create doc");
|
||||
let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc");
|
||||
assert_eq!(result.title, title);
|
||||
assert_eq!(result.markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_inline_styles() {
|
||||
let markdown = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com).";
|
||||
let expected = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com).\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_list_items() {
|
||||
let markdown = "- Item 1\n- Item 2\n- [ ] Task\n- [x] Done";
|
||||
let expected = "* Item 1\n* Item 2\n- [ ] Task\n- [x] Done\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_code_block() {
|
||||
let markdown = "```rust\nfn main() {}\n```";
|
||||
let expected = "```rust\nfn main() {}\n\n```\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_code_block_indentation() {
|
||||
let markdown = "```python\n def indented():\n return \"ok\"\n```";
|
||||
let doc_id = "roundtrip-indent";
|
||||
let title = "Roundtrip Title";
|
||||
let bin = build_full_doc(title, markdown, doc_id).expect("create doc");
|
||||
let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc");
|
||||
assert!(result.markdown.contains("\n def indented():"));
|
||||
assert!(result.markdown.contains("\n return \"ok\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_table() {
|
||||
let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let expected = "|A|B|\n|---|---|\n|1|2|\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_image_with_caption() {
|
||||
let markdown = "";
|
||||
let expected = "<img\n src=\"blob://image-id\"\n alt=\"Alt\"\n width=\"auto\"\n height=\"auto\"\n/>\n\n";
|
||||
assert_markdown_roundtrip(markdown, expected);
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
pub(super) const PAGE_FLAVOUR: &str = "affine:page";
|
||||
pub(super) const NOTE_FLAVOUR: &str = "affine:note";
|
||||
pub(super) const SURFACE_FLAVOUR: &str = "affine:surface";
|
||||
|
||||
pub(super) const SYS_ID: &str = "sys:id";
|
||||
pub(super) const SYS_FLAVOUR: &str = "sys:flavour";
|
||||
pub(super) const SYS_VERSION: &str = "sys:version";
|
||||
pub(super) const SYS_CHILDREN: &str = "sys:children";
|
||||
|
||||
pub(super) const PROP_TITLE: &str = "prop:title";
|
||||
pub(super) const PROP_TEXT: &str = "prop:text";
|
||||
pub(super) const PROP_TYPE: &str = "prop:type";
|
||||
pub(super) const PROP_CHECKED: &str = "prop:checked";
|
||||
pub(super) const PROP_LANGUAGE: &str = "prop:language";
|
||||
pub(super) const PROP_ORDER: &str = "prop:order";
|
||||
|
||||
pub(super) const PROP_ELEMENTS: &str = "prop:elements";
|
||||
pub(super) const PROP_BACKGROUND: &str = "prop:background";
|
||||
pub(super) const PROP_XYWH: &str = "prop:xywh";
|
||||
pub(super) const PROP_INDEX: &str = "prop:index";
|
||||
pub(super) const PROP_HIDDEN: &str = "prop:hidden";
|
||||
pub(super) const PROP_DISPLAY_MODE: &str = "prop:displayMode";
|
||||
|
||||
pub(super) const PROP_SOURCE_ID: &str = "prop:sourceId";
|
||||
pub(super) const PROP_CAPTION: &str = "prop:caption";
|
||||
pub(super) const PROP_WIDTH: &str = "prop:width";
|
||||
pub(super) const PROP_HEIGHT: &str = "prop:height";
|
||||
pub(super) const PROP_URL: &str = "prop:url";
|
||||
pub(super) const PROP_VIDEO_ID: &str = "prop:videoId";
|
||||
|
||||
pub(super) const PROP_ROWS_PREFIX: &str = "prop:rows.";
|
||||
pub(super) const PROP_COLUMNS_PREFIX: &str = "prop:columns.";
|
||||
pub(super) const PROP_CELLS_PREFIX: &str = "prop:cells.";
|
||||
pub(super) const PROP_ROW_ID_SUFFIX: &str = ".rowId";
|
||||
pub(super) const PROP_COLUMN_ID_SUFFIX: &str = ".columnId";
|
||||
pub(super) const PROP_ORDER_SUFFIX: &str = ".order";
|
||||
pub(super) const PROP_TEXT_SUFFIX: &str = ".text";
|
||||
|
||||
pub(super) fn table_row_id_key(row_id: &str) -> String {
|
||||
format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ROW_ID_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_row_order_key(row_id: &str) -> String {
|
||||
format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ORDER_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_column_id_key(column_id: &str) -> String {
|
||||
format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_COLUMN_ID_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_column_order_key(column_id: &str) -> String {
|
||||
format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_ORDER_SUFFIX}")
|
||||
}
|
||||
|
||||
pub(super) fn table_cell_text_key(row_id: &str, column_id: &str) -> String {
|
||||
format!("{PROP_CELLS_PREFIX}{row_id}:{column_id}{PROP_TEXT_SUFFIX}")
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct MarkdownTableOptions {
|
||||
pub(super) escape_pipes: bool,
|
||||
pub(super) newline_replacement: &'static str,
|
||||
pub(super) trim: bool,
|
||||
}
|
||||
|
||||
impl MarkdownTableOptions {
|
||||
pub(super) const fn new(escape_pipes: bool, newline_replacement: &'static str, trim: bool) -> Self {
|
||||
Self {
|
||||
escape_pipes,
|
||||
newline_replacement,
|
||||
trim,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn render_markdown_table(rows: &[Vec<String>], options: MarkdownTableOptions) -> Option<String> {
|
||||
let (header, body) = rows.split_first()?;
|
||||
let header_line = format_table_row(header, options);
|
||||
let separator_line = format_table_row(&vec!["---".to_string(); header.len()], options);
|
||||
let mut lines = vec![header_line, separator_line];
|
||||
for row in body {
|
||||
lines.push(format_table_row(row, options));
|
||||
}
|
||||
Some(lines.join("\n"))
|
||||
}
|
||||
|
||||
fn format_table_row(row: &[String], options: MarkdownTableOptions) -> String {
|
||||
let cells = row
|
||||
.iter()
|
||||
.map(|cell| format_table_cell(cell, options))
|
||||
.collect::<Vec<_>>();
|
||||
format!("|{}|", cells.join("|"))
|
||||
}
|
||||
|
||||
fn format_table_cell(cell: &str, options: MarkdownTableOptions) -> String {
|
||||
let mut value = if options.trim {
|
||||
cell.trim().to_string()
|
||||
} else {
|
||||
cell.to_string()
|
||||
};
|
||||
|
||||
if options.escape_pipes {
|
||||
value = value.replace('|', "\\|");
|
||||
}
|
||||
if !options.newline_replacement.is_empty() {
|
||||
value = collapse_newlines(&value, options.newline_replacement);
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
fn collapse_newlines(value: &str, replacement: &str) -> String {
|
||||
if replacement.is_empty() {
|
||||
return value.to_string();
|
||||
}
|
||||
let mut out = String::with_capacity(value.len());
|
||||
let mut in_newline = false;
|
||||
for ch in value.chars() {
|
||||
if ch == '\n' {
|
||||
if !in_newline {
|
||||
out.push_str(replacement);
|
||||
in_newline = true;
|
||||
}
|
||||
} else {
|
||||
in_newline = false;
|
||||
out.push(ch);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -81,6 +81,16 @@ pub(super) fn value_to_any(value: &Value) -> Option<Any> {
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn value_to_f64(value: Value) -> Option<f64> {
|
||||
value.to_any().and_then(|any| match any {
|
||||
Any::Integer(v) => Some(v as f64),
|
||||
Any::BigInt64(v) => Some(v as f64),
|
||||
Any::Float32(v) => Some(v.0 as f64),
|
||||
Any::Float64(v) => Some(v.0),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn any_to_string(any: &Any) -> Option<String> {
|
||||
match any {
|
||||
Any::String(value) => Some(value.to_string()),
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
use y_octo::{TextDeltaOp, TextInsert};
|
||||
|
||||
use super::{
|
||||
super::schema::{
|
||||
PROP_CAPTION, PROP_CELLS_PREFIX, PROP_CHECKED, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER,
|
||||
PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL, PROP_VIDEO_ID, PROP_WIDTH, SYS_CHILDREN,
|
||||
SYS_FLAVOUR, SYS_ID, SYS_VERSION, table_cell_text_key, table_column_id_key, table_column_order_key,
|
||||
table_row_id_key, table_row_order_key,
|
||||
},
|
||||
*,
|
||||
};
|
||||
|
||||
pub(super) const BOXED_NATIVE_TYPE: &str = "$blocksuite:internal:native$";
|
||||
pub(super) const NOTE_BG_LIGHT: &str = "#ffffff";
|
||||
pub(super) const NOTE_BG_DARK: &str = "#252525";
|
||||
const TABLE_ORDER_WIDTH: usize = 6;
|
||||
|
||||
pub(super) fn block_version(flavour: &str) -> i32 {
|
||||
match flavour {
|
||||
"affine:page" => 2,
|
||||
"affine:surface" => 5,
|
||||
"affine:note" => 1,
|
||||
"affine:paragraph" => 1,
|
||||
"affine:list" => 1,
|
||||
"affine:code" => 1,
|
||||
"affine:divider" => 1,
|
||||
"affine:image" => 1,
|
||||
"affine:table" => 1,
|
||||
"affine:bookmark" => 1,
|
||||
"affine:embed-youtube" => 1,
|
||||
"affine:embed-iframe" => 1,
|
||||
"affine:callout" => 1,
|
||||
_ => 1,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct TextBlockProps<'a> {
|
||||
pub block_type: Option<&'a str>,
|
||||
pub checked: Option<bool>,
|
||||
pub language: Option<&'a str>,
|
||||
pub order: Option<i64>,
|
||||
pub text: &'a [TextDeltaOp],
|
||||
}
|
||||
|
||||
pub(super) struct ImageBlockProps<'a> {
|
||||
pub source_id: &'a str,
|
||||
pub caption: Option<&'a str>,
|
||||
pub width: Option<f64>,
|
||||
pub height: Option<f64>,
|
||||
}
|
||||
|
||||
pub(super) struct BookmarkBlockProps<'a> {
|
||||
pub url: &'a str,
|
||||
pub caption: Option<&'a str>,
|
||||
}
|
||||
|
||||
pub(super) struct EmbedYoutubeBlockProps<'a> {
|
||||
pub video_id: &'a str,
|
||||
}
|
||||
|
||||
pub(super) struct EmbedIframeBlockProps<'a> {
|
||||
pub url: &'a str,
|
||||
}
|
||||
|
||||
pub(super) fn insert_text(doc: &Doc, block: &mut Map, key: &str, ops: &[TextDeltaOp]) -> Result<(), ParseError> {
|
||||
let mut text = doc.create_text()?;
|
||||
// Attach first so updates encode parent types before their contents.
|
||||
block.insert(key.to_string(), Value::Text(text.clone()))?;
|
||||
if !ops.is_empty() {
|
||||
text.apply_delta(ops)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn text_ops_from_plain(text: &str) -> Vec<TextDeltaOp> {
|
||||
if text.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text.to_string()),
|
||||
format: None,
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn insert_children(doc: &Doc, block: &mut Map, children: &[String]) -> Result<(), ParseError> {
|
||||
let mut array = doc.create_array()?;
|
||||
// Attach first so updates encode parent types before their contents.
|
||||
block.insert(SYS_CHILDREN.to_string(), Value::Array(array.clone()))?;
|
||||
for child_id in children {
|
||||
array.push(child_id.to_string())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn insert_block_map(doc: &Doc, blocks_map: &mut Map, block_id: &str) -> Result<Map, ParseError> {
|
||||
let empty_map = doc.create_map()?;
|
||||
blocks_map.insert(block_id.to_string(), Value::Map(empty_map))?;
|
||||
|
||||
blocks_map
|
||||
.get(block_id)
|
||||
.and_then(|value| value.to_map())
|
||||
.ok_or_else(|| ParseError::ParserError("Failed to retrieve inserted block map".into()))
|
||||
}
|
||||
|
||||
pub(super) fn insert_sys_fields(block: &mut Map, block_id: &str, flavour: &str) -> Result<(), ParseError> {
|
||||
block.insert(SYS_ID.to_string(), Any::String(block_id.to_string()))?;
|
||||
block.insert(SYS_FLAVOUR.to_string(), Any::String(flavour.to_string()))?;
|
||||
block.insert(SYS_VERSION.to_string(), Any::Integer(block_version(flavour)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_text_block_props(
|
||||
doc: &Doc,
|
||||
block: &mut Map,
|
||||
props: &TextBlockProps<'_>,
|
||||
preserve_text: bool,
|
||||
clear_missing: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
match props.block_type {
|
||||
Some(block_type) => {
|
||||
block.insert(PROP_TYPE.to_string(), Any::String(block_type.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_TYPE).is_some() {
|
||||
block.remove(PROP_TYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !preserve_text && !props.text.is_empty() {
|
||||
insert_text(doc, block, PROP_TEXT, props.text)?;
|
||||
} else if !preserve_text && clear_missing && block.get(PROP_TEXT).is_some() {
|
||||
block.remove(PROP_TEXT);
|
||||
}
|
||||
|
||||
match props.checked {
|
||||
Some(checked) => {
|
||||
block.insert(PROP_CHECKED.to_string(), if checked { Any::True } else { Any::False })?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_CHECKED).is_some() {
|
||||
block.remove(PROP_CHECKED);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.language {
|
||||
Some(language) => {
|
||||
block.insert(PROP_LANGUAGE.to_string(), Any::String(language.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_LANGUAGE).is_some() {
|
||||
block.remove(PROP_LANGUAGE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.order {
|
||||
Some(order) => {
|
||||
block.insert(PROP_ORDER.to_string(), Any::Float64((order as f64).into()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_ORDER).is_some() {
|
||||
block.remove(PROP_ORDER);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_image_block_props(
|
||||
block: &mut Map,
|
||||
props: &ImageBlockProps<'_>,
|
||||
clear_missing: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_SOURCE_ID.to_string(), Any::String(props.source_id.to_string()))?;
|
||||
|
||||
match props.caption {
|
||||
Some(caption) => {
|
||||
block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_CAPTION).is_some() {
|
||||
block.remove(PROP_CAPTION);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.width {
|
||||
Some(width) => {
|
||||
block.insert(PROP_WIDTH.to_string(), Any::Float64(width.into()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_WIDTH).is_some() {
|
||||
block.remove(PROP_WIDTH);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match props.height {
|
||||
Some(height) => {
|
||||
block.insert(PROP_HEIGHT.to_string(), Any::Float64(height.into()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_HEIGHT).is_some() {
|
||||
block.remove(PROP_HEIGHT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_bookmark_block_props(
|
||||
block: &mut Map,
|
||||
props: &BookmarkBlockProps<'_>,
|
||||
clear_missing: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?;
|
||||
|
||||
match props.caption {
|
||||
Some(caption) => {
|
||||
block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?;
|
||||
}
|
||||
None => {
|
||||
if clear_missing && block.get(PROP_CAPTION).is_some() {
|
||||
block.remove(PROP_CAPTION);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_embed_youtube_block_props(
|
||||
block: &mut Map,
|
||||
props: &EmbedYoutubeBlockProps<'_>,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_VIDEO_ID.to_string(), Any::String(props.video_id.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_embed_iframe_block_props(
|
||||
block: &mut Map,
|
||||
props: &EmbedIframeBlockProps<'_>,
|
||||
) -> Result<(), ParseError> {
|
||||
block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply_table_block_props(block: &mut Map, rows: &[Vec<String>]) -> Result<(), ParseError> {
|
||||
clear_table_props(block);
|
||||
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let column_count = rows.iter().map(|row| row.len()).max().unwrap_or(0);
|
||||
let column_ids: Vec<String> = (0..column_count).map(|_| nanoid::nanoid!()).collect();
|
||||
|
||||
for (col_idx, column_id) in column_ids.iter().enumerate() {
|
||||
let order = format_table_order(col_idx);
|
||||
block.insert(table_column_id_key(column_id), Any::String(column_id.to_string()))?;
|
||||
block.insert(table_column_order_key(column_id), Any::String(order))?;
|
||||
}
|
||||
|
||||
for (row_idx, row) in rows.iter().enumerate() {
|
||||
let row_id = nanoid::nanoid!();
|
||||
let order = format_table_order(row_idx);
|
||||
block.insert(table_row_id_key(&row_id), Any::String(row_id.to_string()))?;
|
||||
block.insert(table_row_order_key(&row_id), Any::String(order))?;
|
||||
|
||||
for (col_idx, column_id) in column_ids.iter().enumerate() {
|
||||
let cell_text = row.get(col_idx).cloned().unwrap_or_default();
|
||||
block.insert(table_cell_text_key(&row_id, column_id), Any::String(cell_text))?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) struct ApplyBlockOptions {
|
||||
pub preserve_text: bool,
|
||||
pub clear_missing: bool,
|
||||
}
|
||||
|
||||
pub(super) fn apply_block_spec(
|
||||
doc: &Doc,
|
||||
block: &mut Map,
|
||||
spec: &BlockSpec,
|
||||
options: ApplyBlockOptions,
|
||||
) -> Result<(), ParseError> {
|
||||
match spec.flavour {
|
||||
BlockFlavour::Image => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let image = spec
|
||||
.image
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("image spec missing".into()))?;
|
||||
let props = ImageBlockProps {
|
||||
source_id: &image.source_id,
|
||||
caption: image.caption.as_deref(),
|
||||
width: image.width,
|
||||
height: image.height,
|
||||
};
|
||||
apply_image_block_props(block, &props, options.clear_missing)?;
|
||||
}
|
||||
BlockFlavour::Bookmark => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let bookmark = spec
|
||||
.bookmark
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("bookmark spec missing".into()))?;
|
||||
let props = BookmarkBlockProps {
|
||||
url: &bookmark.url,
|
||||
caption: bookmark.caption.as_deref(),
|
||||
};
|
||||
apply_bookmark_block_props(block, &props, options.clear_missing)?;
|
||||
}
|
||||
BlockFlavour::EmbedYoutube => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let embed = spec
|
||||
.embed_youtube
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?;
|
||||
let props = EmbedYoutubeBlockProps {
|
||||
video_id: &embed.video_id,
|
||||
};
|
||||
apply_embed_youtube_block_props(block, &props)?;
|
||||
}
|
||||
BlockFlavour::EmbedIframe => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let embed = spec
|
||||
.embed_iframe
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?;
|
||||
let props = EmbedIframeBlockProps { url: &embed.url };
|
||||
apply_embed_iframe_block_props(block, &props)?;
|
||||
}
|
||||
BlockFlavour::Callout => {
|
||||
return Ok(());
|
||||
}
|
||||
BlockFlavour::Table => {
|
||||
if options.preserve_text {
|
||||
return Ok(());
|
||||
}
|
||||
let table = spec
|
||||
.table
|
||||
.as_ref()
|
||||
.ok_or_else(|| ParseError::ParserError("table spec missing".into()))?;
|
||||
apply_table_block_props(block, &table.rows)?;
|
||||
}
|
||||
_ => {
|
||||
let props = TextBlockProps {
|
||||
block_type: spec.block_type_str(),
|
||||
checked: spec.checked,
|
||||
language: spec.language.as_deref(),
|
||||
order: spec.order,
|
||||
text: &spec.text,
|
||||
};
|
||||
apply_text_block_props(doc, block, &props, options.preserve_text, options.clear_missing)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn insert_block_tree(doc: &Doc, blocks_map: &mut Map, node: &BlockNode) -> Result<String, ParseError> {
|
||||
let block_id = nanoid::nanoid!();
|
||||
let mut block_map = insert_block_map(doc, blocks_map, &block_id)?;
|
||||
|
||||
insert_sys_fields(&mut block_map, &block_id, node.spec.flavour.as_str())?;
|
||||
apply_block_spec(
|
||||
doc,
|
||||
&mut block_map,
|
||||
&node.spec,
|
||||
ApplyBlockOptions {
|
||||
preserve_text: false,
|
||||
clear_missing: false,
|
||||
},
|
||||
)?;
|
||||
|
||||
let child_ids = node
|
||||
.children
|
||||
.iter()
|
||||
.map(|child| insert_block_tree(doc, blocks_map, child))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
insert_children(doc, &mut block_map, &child_ids)?;
|
||||
|
||||
Ok(block_id)
|
||||
}
|
||||
|
||||
fn clear_table_props(block: &mut Map) {
|
||||
let keys = block
|
||||
.keys()
|
||||
.filter(|key| {
|
||||
key.starts_with(PROP_ROWS_PREFIX) || key.starts_with(PROP_COLUMNS_PREFIX) || key.starts_with(PROP_CELLS_PREFIX)
|
||||
})
|
||||
.map(|s| s.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
for key in keys {
|
||||
block.remove(&key);
|
||||
}
|
||||
}
|
||||
|
||||
fn format_table_order(index: usize) -> String {
|
||||
format!("{index:0width$}", width = TABLE_ORDER_WIDTH)
|
||||
}
|
||||
|
||||
pub(super) fn boxed_empty_map(doc: &Doc) -> Result<Map, ParseError> {
|
||||
doc.create_map().map_err(ParseError::from)
|
||||
}
|
||||
|
||||
pub(super) fn note_background_map(doc: &Doc) -> Result<Map, ParseError> {
|
||||
doc.create_map().map_err(ParseError::from)
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
//! Markdown to YDoc conversion module
|
||||
//!
|
||||
//! Converts markdown content into AFFiNE-compatible y-octo document binary
|
||||
//! format.
|
||||
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::{
|
||||
super::{
|
||||
markdown::parse_markdown_blocks,
|
||||
schema::{PROP_BACKGROUND, PROP_DISPLAY_MODE, PROP_ELEMENTS, PROP_HIDDEN, PROP_INDEX, PROP_XYWH, SURFACE_FLAVOUR},
|
||||
},
|
||||
builder::{
|
||||
BOXED_NATIVE_TYPE, NOTE_BG_DARK, NOTE_BG_LIGHT, boxed_empty_map, insert_block_map, insert_block_tree,
|
||||
insert_children, insert_sys_fields, insert_text, note_background_map, text_ops_from_plain,
|
||||
},
|
||||
*,
|
||||
};
|
||||
|
||||
/// Converts markdown into an AFFiNE-compatible y-octo document binary.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `title` - The document title
|
||||
/// * `markdown` - The markdown content to convert
|
||||
/// * `doc_id` - The document ID to use
|
||||
///
|
||||
/// # Returns
|
||||
/// A binary vector containing the y-octo encoded document update
|
||||
pub fn build_full_doc(title: &str, markdown: &str, doc_id: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let nodes = parse_markdown_blocks(markdown)?;
|
||||
build_doc_update(doc_id, title, &nodes)
|
||||
}
|
||||
|
||||
fn build_doc_update(doc_id: &str, title: &str, blocks: &[BlockNode]) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
let mut blocks_map = doc.get_or_create_map("blocks")?;
|
||||
|
||||
let page_id = nanoid::nanoid!();
|
||||
let surface_id = nanoid::nanoid!();
|
||||
let note_id = nanoid::nanoid!();
|
||||
|
||||
// Insert root blocks first to establish stable IDs.
|
||||
let mut page_map = insert_block_map(&doc, &mut blocks_map, &page_id)?;
|
||||
let mut surface_map = insert_block_map(&doc, &mut blocks_map, &surface_id)?;
|
||||
let mut note_map = insert_block_map(&doc, &mut blocks_map, ¬e_id)?;
|
||||
|
||||
// Create content blocks under note.
|
||||
let content_ids = insert_block_trees(&doc, &mut blocks_map, blocks)?;
|
||||
|
||||
// Page block
|
||||
insert_sys_fields(&mut page_map, &page_id, PAGE_FLAVOUR)?;
|
||||
insert_children(&doc, &mut page_map, &[surface_id.clone(), note_id.clone()])?;
|
||||
insert_text(&doc, &mut page_map, PROP_TITLE, &text_ops_from_plain(title))?;
|
||||
|
||||
// Surface block
|
||||
insert_sys_fields(&mut surface_map, &surface_id, SURFACE_FLAVOUR)?;
|
||||
insert_children(&doc, &mut surface_map, &[])?;
|
||||
let mut boxed = boxed_empty_map(&doc)?;
|
||||
surface_map.insert(PROP_ELEMENTS.to_string(), Value::Map(boxed.clone()))?;
|
||||
boxed.insert("type".to_string(), Any::String(BOXED_NATIVE_TYPE.to_string()))?;
|
||||
let value = doc.create_map()?;
|
||||
boxed.insert("value".to_string(), Value::Map(value))?;
|
||||
|
||||
// Note block
|
||||
insert_sys_fields(&mut note_map, ¬e_id, NOTE_FLAVOUR)?;
|
||||
insert_children(&doc, &mut note_map, &content_ids)?;
|
||||
let mut background = note_background_map(&doc)?;
|
||||
note_map.insert(PROP_BACKGROUND.to_string(), Value::Map(background.clone()))?;
|
||||
background.insert("light".to_string(), Any::String(NOTE_BG_LIGHT.to_string()))?;
|
||||
background.insert("dark".to_string(), Any::String(NOTE_BG_DARK.to_string()))?;
|
||||
note_map.insert(PROP_XYWH.to_string(), Any::String("[0,0,800,95]".to_string()))?;
|
||||
note_map.insert(PROP_INDEX.to_string(), Any::String("a0".to_string()))?;
|
||||
note_map.insert(PROP_HIDDEN.to_string(), Any::False)?;
|
||||
note_map.insert(PROP_DISPLAY_MODE.to_string(), Any::String("both".to_string()))?;
|
||||
|
||||
Ok(doc.encode_update_v1()?)
|
||||
}
|
||||
|
||||
fn insert_block_trees(doc: &Doc, blocks_map: &mut Map, blocks: &[BlockNode]) -> Result<Vec<String>, ParseError> {
|
||||
let mut ids = Vec::with_capacity(blocks.len());
|
||||
for block in blocks {
|
||||
let id = insert_block_tree(doc, blocks_map, block)?;
|
||||
ids.push(id);
|
||||
}
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::{Any, DocOptions};
|
||||
|
||||
use super::{
|
||||
super::super::{
|
||||
blocksuite::get_string,
|
||||
markdown::{MAX_BLOCKS, MAX_MARKDOWN_CHARS},
|
||||
schema::PAGE_FLAVOUR,
|
||||
},
|
||||
*,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn test_simple_markdown() {
|
||||
let markdown = "# Hello World\n\nThis is a test paragraph.";
|
||||
let result = build_full_doc("Hello World", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_title_from_param() {
|
||||
let markdown = "# Markdown Title\n\nContent.";
|
||||
let doc_id = "title-param-test";
|
||||
let bin = build_full_doc("External Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut title = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
|
||||
{
|
||||
title = get_string(&block_map, "prop:title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(title.as_deref(), Some("External Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_list() {
|
||||
let markdown = "# Test List\n\n- Item 1\n- Item 2\n- Item 3";
|
||||
let result = build_full_doc("Test List", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_code() {
|
||||
let markdown = "# Code Example\n\n```rust\nfn main() {\n println!(\"Hello\");\n}\n```";
|
||||
let result = build_full_doc("Code Example", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_headings() {
|
||||
let markdown = "# H1\n\n## H2\n\n### H3\n\nParagraph text.";
|
||||
let result = build_full_doc("H1", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_markdown() {
|
||||
let result = build_full_doc("Untitled", "", "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_whitespace_only_markdown() {
|
||||
let result = build_full_doc("Untitled", " \n\n\t\n ", "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
let bin = result.unwrap();
|
||||
assert!(!bin.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_without_h1() {
|
||||
let markdown = "## Secondary Heading\n\nSome content without H1.";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nested_lists() {
|
||||
let markdown = "# Nested Lists\n\n- Item 1\n - Nested 1.1\n - Nested 1.2\n- Item 2\n - Nested 2.1";
|
||||
let result = build_full_doc("Nested Lists", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blockquote() {
|
||||
let markdown = "# Title\n\n> A blockquote";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_divider() {
|
||||
let markdown = "# Title\n\nBefore divider\n\n---\n\nAfter divider";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_numbered_list() {
|
||||
let markdown = "# Title\n\n1. First item\n2. Second item";
|
||||
let result = build_full_doc("Title", markdown, "test-doc-id");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_too_large() {
|
||||
let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1);
|
||||
let result = build_full_doc("Title", &markdown, "test-doc-id");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_block_limit() {
|
||||
let mut markdown = String::from("# Title\n\n");
|
||||
for i in 0..=MAX_BLOCKS {
|
||||
markdown.push_str(&format!("Paragraph {i}\n\n"));
|
||||
}
|
||||
let result = build_full_doc("Title", &markdown, "test-doc-id");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_image() {
|
||||
let markdown = "";
|
||||
let doc_id = "image-doc";
|
||||
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut found = false;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image")
|
||||
{
|
||||
let source_id = get_string(&block_map, "prop:sourceId");
|
||||
assert_eq!(source_id.as_deref(), Some("image-id"));
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_with_table() {
|
||||
let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let doc_id = "table-doc";
|
||||
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&bin).expect("apply update");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut found_cell = false;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table")
|
||||
{
|
||||
for key in block_map.keys() {
|
||||
if key.starts_with("prop:cells.") && key.ends_with(".text") {
|
||||
let value = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::String(value) => Some(value),
|
||||
_ => None,
|
||||
});
|
||||
if let Some(value) = value
|
||||
&& (value == "A" || value == "1")
|
||||
{
|
||||
found_cell = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found_cell);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
use super::{
|
||||
builder::{insert_text, text_ops_from_plain},
|
||||
root_doc::ensure_pages_array,
|
||||
*,
|
||||
};
|
||||
|
||||
pub fn update_doc_title(existing_binary: &[u8], doc_id: &str, title: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = load_doc(existing_binary, Some(doc_id))?;
|
||||
|
||||
let state_before = doc.get_state_vector();
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Err(ParseError::ParserError("blocks map is empty".into()));
|
||||
}
|
||||
|
||||
let mut page_block = find_page_block(&blocks_map)?;
|
||||
let current = get_string(&page_block, PROP_TITLE).unwrap_or_default();
|
||||
if current != title {
|
||||
insert_text(&doc, &mut page_block, PROP_TITLE, &text_ops_from_plain(title))?;
|
||||
}
|
||||
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
pub fn update_root_doc_meta_title(root_doc_bin: &[u8], doc_id: &str, title: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = load_doc_or_new(root_doc_bin)?;
|
||||
|
||||
let state_before = doc.get_state_vector();
|
||||
let mut meta = doc.get_or_create_map("meta")?;
|
||||
let mut pages = ensure_pages_array(&doc, &mut meta)?;
|
||||
|
||||
let mut found = false;
|
||||
for idx in 0..pages.len() {
|
||||
let Some(mut page) = pages.get(idx).and_then(|v| v.to_map()) else {
|
||||
continue;
|
||||
};
|
||||
if get_string(&page, "id").as_deref() == Some(doc_id) {
|
||||
page.insert("title".to_string(), Any::String(title.to_string()))?;
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
let page_map = doc.create_map()?;
|
||||
|
||||
let idx = pages.len();
|
||||
pages.insert(idx, Value::Map(page_map))?;
|
||||
|
||||
if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) {
|
||||
inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?;
|
||||
inserted_page.insert("title".to_string(), Any::String(title.to_string()))?;
|
||||
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as i64)
|
||||
.unwrap_or(0);
|
||||
inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?;
|
||||
|
||||
let tags = doc.create_array()?;
|
||||
inserted_page.insert("tags".to_string(), Value::Array(tags))?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
fn find_page_block(blocks_map: &Map) -> Result<Map, ParseError> {
|
||||
let index = build_block_index(blocks_map);
|
||||
let page_id = find_block_id_by_flavour(&index.block_pool, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
|
||||
blocks_map
|
||||
.get(&page_id)
|
||||
.and_then(|value| value.to_map())
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::*;
|
||||
use crate::doc_parser::{add_doc_to_root_doc, build_full_doc};
|
||||
|
||||
#[test]
|
||||
fn test_update_doc_title() {
|
||||
let doc_id = "doc-meta-title-test";
|
||||
let initial = build_full_doc("Old Title", "Content.", doc_id).expect("create doc");
|
||||
let delta = update_doc_title(&initial, doc_id, "New Title").expect("update title");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&initial).expect("apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut title = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
|
||||
{
|
||||
title = get_string(&block_map, "prop:title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(title.as_deref(), Some("New Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_root_doc_meta_title() {
|
||||
let doc_id = "root-meta-title-test";
|
||||
let root_bin = add_doc_to_root_doc(Vec::new(), doc_id, Some("Old Title")).expect("create root meta");
|
||||
let delta = update_root_doc_meta_title(&root_bin, doc_id, "New Title").expect("update meta");
|
||||
|
||||
let mut doc = DocOptions::new().build();
|
||||
doc.apply_update_from_binary_v1(&root_bin).expect("apply root");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let meta = doc.get_map("meta").expect("meta map");
|
||||
let pages = meta.get("pages").and_then(|v| v.to_array()).expect("pages array");
|
||||
let mut title = None;
|
||||
for page in pages.iter() {
|
||||
if let Some(page_map) = page.to_map()
|
||||
&& get_string(&page_map, "id").as_deref() == Some(doc_id)
|
||||
{
|
||||
title = get_string(&page_map, "title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(title.as_deref(), Some("New Title"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
use y_octo::{Any, DocOptions, Map};
|
||||
|
||||
use super::{
|
||||
super::{doc_loader::is_empty_doc, value::value_to_string},
|
||||
ParseError,
|
||||
};
|
||||
|
||||
pub fn update_doc_properties(
|
||||
existing_binary: &[u8],
|
||||
properties_doc_id: &str,
|
||||
target_doc_id: &str,
|
||||
created_by: Option<&str>,
|
||||
updated_by: Option<&str>,
|
||||
) -> Result<Vec<u8>, ParseError> {
|
||||
let doc = if is_empty_doc(existing_binary) {
|
||||
DocOptions::new().with_guid(properties_doc_id.to_string()).build()
|
||||
} else {
|
||||
super::load_doc(existing_binary, Some(properties_doc_id))?
|
||||
};
|
||||
|
||||
let state_before = doc.get_state_vector();
|
||||
let mut record = doc.get_or_create_map(target_doc_id)?;
|
||||
let mut changed = false;
|
||||
|
||||
if record.get("id").is_none() {
|
||||
record.insert("id".to_string(), Any::String(target_doc_id.to_string()))?;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if let Some(created_by) = created_by
|
||||
&& get_record_string(&record, "createdBy").as_deref() != Some(created_by)
|
||||
{
|
||||
record.insert("createdBy".to_string(), Any::String(created_by.to_string()))?;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if let Some(updated_by) = updated_by
|
||||
&& get_record_string(&record, "updatedBy").as_deref() != Some(updated_by)
|
||||
{
|
||||
record.insert("updatedBy".to_string(), Any::String(updated_by.to_string()))?;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if !changed {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
fn get_record_string(record: &Map, key: &str) -> Option<String> {
|
||||
record.get(key).and_then(|value| value_to_string(&value))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::DocOptions;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_update_doc_properties_creates_record() {
|
||||
let properties_doc_id = "doc-properties";
|
||||
let target_doc_id = "doc-1";
|
||||
let update = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-1"))
|
||||
.expect("update properties");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(properties_doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&update).expect("apply");
|
||||
|
||||
let record = doc.get_map(target_doc_id).expect("record map");
|
||||
assert_eq!(get_record_string(&record, "id").as_deref(), Some(target_doc_id));
|
||||
assert_eq!(get_record_string(&record, "createdBy").as_deref(), Some("user-1"));
|
||||
assert_eq!(get_record_string(&record, "updatedBy").as_deref(), Some("user-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_doc_properties_no_change() {
|
||||
let properties_doc_id = "doc-properties-no-change";
|
||||
let target_doc_id = "doc-2";
|
||||
let initial = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-2"))
|
||||
.expect("initial update");
|
||||
|
||||
let delta = update_doc_properties(
|
||||
&initial,
|
||||
properties_doc_id,
|
||||
target_doc_id,
|
||||
Some("user-1"),
|
||||
Some("user-2"),
|
||||
)
|
||||
.expect("no change update");
|
||||
|
||||
assert!(delta.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
pub mod builder;
|
||||
mod create;
|
||||
mod doc_meta;
|
||||
mod doc_properties;
|
||||
mod root_doc;
|
||||
mod update;
|
||||
|
||||
pub use create::build_full_doc;
|
||||
pub use doc_meta::{update_doc_title, update_root_doc_meta_title};
|
||||
pub use doc_properties::update_doc_properties;
|
||||
pub use root_doc::add_doc_to_root_doc;
|
||||
pub use update::update_doc;
|
||||
use y_octo::{Any, Doc, Map, Value};
|
||||
|
||||
use super::{
|
||||
ParseError,
|
||||
block_spec::{BlockFlavour, BlockNode, BlockSpec},
|
||||
blocksuite::{build_block_index, find_block_id_by_flavour, get_string},
|
||||
doc_loader::{load_doc, load_doc_or_new},
|
||||
schema::{NOTE_FLAVOUR, PAGE_FLAVOUR, PROP_TITLE},
|
||||
};
|
||||
@@ -0,0 +1,106 @@
|
||||
use y_octo::Array;
|
||||
|
||||
use super::*;
|
||||
|
||||
const DEFAULT_DOC_TITLE: &str = "Untitled";
|
||||
|
||||
fn any_to_value(doc: &Doc, any: Any) -> Result<Value, ParseError> {
|
||||
match any {
|
||||
Any::Array(values) => {
|
||||
let mut array = doc.create_array()?;
|
||||
for value in values {
|
||||
let item = any_to_value(doc, value)?;
|
||||
array.push(item)?;
|
||||
}
|
||||
Ok(Value::Array(array))
|
||||
}
|
||||
Any::Object(values) => {
|
||||
let mut map = doc.create_map()?;
|
||||
for (key, value) in values {
|
||||
let item = any_to_value(doc, value)?;
|
||||
map.insert(key, item)?;
|
||||
}
|
||||
Ok(Value::Map(map))
|
||||
}
|
||||
_ => Ok(Value::Any(any)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn ensure_pages_array(doc: &Doc, meta: &mut Map) -> Result<Array, ParseError> {
|
||||
let pages_value = meta.get("pages");
|
||||
if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) {
|
||||
return Ok(pages);
|
||||
}
|
||||
|
||||
if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) {
|
||||
let mut pages = doc.create_array()?;
|
||||
for entry in entries {
|
||||
let value = any_to_value(doc, entry)?;
|
||||
pages.push(value)?;
|
||||
}
|
||||
meta.insert("pages".to_string(), Value::Array(pages.clone()))?;
|
||||
return Ok(pages);
|
||||
}
|
||||
|
||||
let pages = doc.create_array()?;
|
||||
meta.insert("pages".to_string(), Value::Array(pages.clone()))?;
|
||||
Ok(pages)
|
||||
}
|
||||
|
||||
/// Adds a document ID to the root doc's meta.pages array.
|
||||
/// Returns a binary update that can be applied to the root doc.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `root_doc_bin` - The current root doc binary
|
||||
/// * `doc_id` - The document ID to add
|
||||
/// * `title` - Optional title for the document
|
||||
///
|
||||
/// # Returns
|
||||
/// A Vec<u8> containing the y-octo update binary to add the doc
|
||||
pub fn add_doc_to_root_doc(root_doc_bin: Vec<u8>, doc_id: &str, title: Option<&str>) -> Result<Vec<u8>, ParseError> {
|
||||
// Handle empty or minimal root doc - create a new one
|
||||
let doc = load_doc_or_new(&root_doc_bin)?;
|
||||
|
||||
// Capture state before modifications to encode only the delta
|
||||
let state_before = doc.get_state_vector();
|
||||
|
||||
// Get or create the meta map
|
||||
let mut meta = doc.get_or_create_map("meta")?;
|
||||
|
||||
let mut pages = ensure_pages_array(&doc, &mut meta)?;
|
||||
|
||||
// Check if doc already exists
|
||||
let doc_exists = pages.iter().any(|page_val| {
|
||||
page_val
|
||||
.to_map()
|
||||
.and_then(|page| get_string(&page, "id"))
|
||||
.map(|id| id == doc_id)
|
||||
.unwrap_or(false)
|
||||
});
|
||||
|
||||
if !doc_exists {
|
||||
let page_map = doc.create_map()?;
|
||||
|
||||
let idx = pages.len();
|
||||
pages.insert(idx, Value::Map(page_map))?;
|
||||
|
||||
if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) {
|
||||
inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?;
|
||||
|
||||
let page_title = title.unwrap_or(DEFAULT_DOC_TITLE);
|
||||
inserted_page.insert("title".to_string(), Any::String(page_title.to_string()))?;
|
||||
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as i64)
|
||||
.unwrap_or(0);
|
||||
inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?;
|
||||
|
||||
let tags = doc.create_array()?;
|
||||
inserted_page.insert("tags".to_string(), Value::Array(tags))?;
|
||||
}
|
||||
}
|
||||
|
||||
// Encode only the changes (delta) since state_before
|
||||
Ok(doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
@@ -0,0 +1,671 @@
|
||||
//! Update YDoc module
|
||||
//!
|
||||
//! Provides functionality to update existing AFFiNE documents by applying
|
||||
//! surgical y-octo operations based on content differences.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use super::{
|
||||
super::{
|
||||
block_spec::{TreeNode, count_tree_nodes, text_delta_eq},
|
||||
blocksuite::{collect_child_ids, find_child_id_by_flavour},
|
||||
markdown::{MAX_BLOCKS, parse_markdown_blocks},
|
||||
},
|
||||
builder::{ApplyBlockOptions, apply_block_spec, insert_block_tree, insert_children},
|
||||
*,
|
||||
};
|
||||
|
||||
const MAX_LCS_CELLS: usize = 2_000_000;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct StoredNode {
|
||||
id: String,
|
||||
spec: BlockSpec,
|
||||
children: Vec<StoredNode>,
|
||||
}
|
||||
|
||||
impl TreeNode for StoredNode {
|
||||
fn children(&self) -> &[StoredNode] {
|
||||
&self.children
|
||||
}
|
||||
}
|
||||
|
||||
struct DocState {
|
||||
doc: Doc,
|
||||
note_id: String,
|
||||
blocks: Vec<StoredNode>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum PatchOp {
|
||||
Keep(usize, usize),
|
||||
Delete(usize),
|
||||
Insert(usize),
|
||||
Update(usize, usize),
|
||||
}
|
||||
|
||||
/// Updates an existing document with new markdown content.
|
||||
///
|
||||
/// This function performs structural diffing between the existing document
|
||||
/// and the new markdown content, then applies block-level replacements
|
||||
/// for changed blocks. This enables proper CRDT merging with concurrent
|
||||
/// edits from other clients.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `existing_binary` - The current document binary
|
||||
/// * `new_markdown` - The new markdown content (document title is not updated)
|
||||
/// * `doc_id` - The document ID
|
||||
///
|
||||
/// # Returns
|
||||
/// A binary vector representing only the delta (changes) to apply
|
||||
pub fn update_doc(existing_binary: &[u8], new_markdown: &str, doc_id: &str) -> Result<Vec<u8>, ParseError> {
|
||||
let mut new_nodes = parse_markdown_blocks(new_markdown)?;
|
||||
let state = load_doc_state(existing_binary, doc_id)?;
|
||||
|
||||
check_limits(&state.blocks, &new_nodes)?;
|
||||
|
||||
let state_before = state.doc.get_state_vector();
|
||||
|
||||
let mut blocks_map = state.doc.get_map("blocks")?;
|
||||
|
||||
let new_children = sync_nodes(&state.doc, &mut blocks_map, &state.blocks, &mut new_nodes)?;
|
||||
sync_children(&state.doc, &mut blocks_map, &state.note_id, &new_children)?;
|
||||
|
||||
Ok(state.doc.encode_state_as_update_v1(&state_before)?)
|
||||
}
|
||||
|
||||
fn load_doc_state(binary: &[u8], doc_id: &str) -> Result<DocState, ParseError> {
|
||||
let doc = load_doc(binary, Some(doc_id))?;
|
||||
|
||||
let blocks_map = doc.get_map("blocks")?;
|
||||
if blocks_map.is_empty() {
|
||||
return Err(ParseError::ParserError("blocks map is empty".into()));
|
||||
}
|
||||
|
||||
let block_index = build_block_index(&blocks_map);
|
||||
let page_id = find_block_id_by_flavour(&block_index.block_pool, PAGE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
|
||||
let page_block = block_index
|
||||
.block_pool
|
||||
.get(&page_id)
|
||||
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
|
||||
let note_id = find_child_id_by_flavour(page_block, &block_index.block_pool, NOTE_FLAVOUR)
|
||||
.ok_or_else(|| ParseError::ParserError("note block not found".into()))?;
|
||||
let note_block = block_index
|
||||
.block_pool
|
||||
.get(¬e_id)
|
||||
.ok_or_else(|| ParseError::ParserError("note block not found".into()))?;
|
||||
let content_ids = collect_child_ids(note_block);
|
||||
|
||||
let mut blocks = Vec::new();
|
||||
for block_id in content_ids {
|
||||
let block = block_index
|
||||
.block_pool
|
||||
.get(&block_id)
|
||||
.ok_or_else(|| ParseError::ParserError("content block not found".into()))?;
|
||||
blocks.push(build_stored_tree(&block_id, block, &block_index.block_pool)?);
|
||||
}
|
||||
|
||||
Ok(DocState { doc, note_id, blocks })
|
||||
}
|
||||
|
||||
fn build_stored_tree(block_id: &str, block: &Map, pool: &HashMap<String, Map>) -> Result<StoredNode, ParseError> {
|
||||
let spec = BlockSpec::from_block_map(block)?;
|
||||
|
||||
let child_ids = collect_child_ids(block);
|
||||
if !child_ids.is_empty() && !matches!(spec.flavour, BlockFlavour::List | BlockFlavour::Callout) {
|
||||
return Err(ParseError::ParserError(format!(
|
||||
"unsupported children on block: {block_id}"
|
||||
)));
|
||||
}
|
||||
let mut children = Vec::new();
|
||||
for child_id in child_ids {
|
||||
let child_block = pool
|
||||
.get(&child_id)
|
||||
.ok_or_else(|| ParseError::ParserError("child block not found".into()))?;
|
||||
children.push(build_stored_tree(&child_id, child_block, pool)?);
|
||||
}
|
||||
|
||||
Ok(StoredNode {
|
||||
id: block_id.to_string(),
|
||||
spec,
|
||||
children,
|
||||
})
|
||||
}
|
||||
|
||||
fn sync_nodes(
|
||||
doc: &Doc,
|
||||
blocks_map: &mut Map,
|
||||
current: &[StoredNode],
|
||||
target: &mut [BlockNode],
|
||||
) -> Result<Vec<String>, ParseError> {
|
||||
let ops = diff_blocks(current, target);
|
||||
let mut new_children = Vec::new();
|
||||
let mut to_remove = Vec::new();
|
||||
|
||||
for op in ops {
|
||||
match op {
|
||||
PatchOp::Keep(old_idx, new_idx) => {
|
||||
let old_node = ¤t[old_idx];
|
||||
let new_node = &target[new_idx];
|
||||
update_block_props(doc, blocks_map, old_node, &new_node.spec, true)?;
|
||||
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
|
||||
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
|
||||
new_children.push(old_node.id.clone());
|
||||
}
|
||||
PatchOp::Update(old_idx, new_idx) => {
|
||||
let old_node = ¤t[old_idx];
|
||||
let new_node = &target[new_idx];
|
||||
update_block_props(doc, blocks_map, old_node, &new_node.spec, false)?;
|
||||
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
|
||||
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
|
||||
new_children.push(old_node.id.clone());
|
||||
}
|
||||
PatchOp::Insert(new_idx) => {
|
||||
let new_id = insert_block_tree(doc, blocks_map, &target[new_idx])?;
|
||||
new_children.push(new_id);
|
||||
}
|
||||
PatchOp::Delete(old_idx) => {
|
||||
let node = ¤t[old_idx];
|
||||
if node.spec.flavour == BlockFlavour::Callout {
|
||||
new_children.push(node.id.clone());
|
||||
} else {
|
||||
collect_tree_ids(node, &mut to_remove);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for id in to_remove {
|
||||
blocks_map.remove(&id);
|
||||
}
|
||||
|
||||
Ok(new_children)
|
||||
}
|
||||
|
||||
fn diff_blocks(current: &[StoredNode], target: &[BlockNode]) -> Vec<PatchOp> {
|
||||
let old_len = current.len();
|
||||
let new_len = target.len();
|
||||
|
||||
if old_len == 0 {
|
||||
return (0..new_len).map(PatchOp::Insert).collect();
|
||||
}
|
||||
if new_len == 0 {
|
||||
return (0..old_len).map(PatchOp::Delete).collect();
|
||||
}
|
||||
|
||||
let mut lcs = vec![vec![0usize; new_len + 1]; old_len + 1];
|
||||
|
||||
for i in 1..=old_len {
|
||||
for j in 1..=new_len {
|
||||
let old_spec = ¤t[i - 1].spec;
|
||||
let new_spec = &target[j - 1].spec;
|
||||
|
||||
if old_spec.is_exact(new_spec) {
|
||||
lcs[i][j] = lcs[i - 1][j - 1] + 1;
|
||||
} else {
|
||||
lcs[i][j] = std::cmp::max(lcs[i - 1][j], lcs[i][j - 1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut ops = Vec::new();
|
||||
let mut i = old_len;
|
||||
let mut j = new_len;
|
||||
|
||||
while i > 0 || j > 0 {
|
||||
if i > 0 && j > 0 {
|
||||
let old_spec = ¤t[i - 1].spec;
|
||||
let new_spec = &target[j - 1].spec;
|
||||
|
||||
if old_spec.is_exact(new_spec) {
|
||||
ops.push(PatchOp::Keep(i - 1, j - 1));
|
||||
i -= 1;
|
||||
j -= 1;
|
||||
} else if old_spec.is_similar(new_spec)
|
||||
&& lcs[i - 1][j - 1] >= lcs[i - 1][j]
|
||||
&& lcs[i - 1][j - 1] >= lcs[i][j - 1]
|
||||
{
|
||||
ops.push(PatchOp::Update(i - 1, j - 1));
|
||||
i -= 1;
|
||||
j -= 1;
|
||||
} else if lcs[i][j - 1] >= lcs[i - 1][j] {
|
||||
ops.push(PatchOp::Insert(j - 1));
|
||||
j -= 1;
|
||||
} else {
|
||||
ops.push(PatchOp::Delete(i - 1));
|
||||
i -= 1;
|
||||
}
|
||||
} else if j > 0 {
|
||||
ops.push(PatchOp::Insert(j - 1));
|
||||
j -= 1;
|
||||
} else {
|
||||
ops.push(PatchOp::Delete(i - 1));
|
||||
i -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
ops.reverse();
|
||||
ops
|
||||
}
|
||||
|
||||
fn update_block_props(
|
||||
doc: &Doc,
|
||||
blocks_map: &mut Map,
|
||||
node: &StoredNode,
|
||||
target: &BlockSpec,
|
||||
preserve_text: bool,
|
||||
) -> Result<(), ParseError> {
|
||||
let Some(mut block) = blocks_map.get(&node.id).and_then(|v| v.to_map()) else {
|
||||
return Err(ParseError::ParserError(format!("Block {} not found", node.id)));
|
||||
};
|
||||
|
||||
let preserve = match target.flavour {
|
||||
BlockFlavour::Image
|
||||
| BlockFlavour::Table
|
||||
| BlockFlavour::Bookmark
|
||||
| BlockFlavour::EmbedYoutube
|
||||
| BlockFlavour::EmbedIframe => preserve_text,
|
||||
_ => preserve_text || text_delta_eq(&node.spec.text, &target.text),
|
||||
};
|
||||
|
||||
apply_block_spec(
|
||||
doc,
|
||||
&mut block,
|
||||
target,
|
||||
ApplyBlockOptions {
|
||||
preserve_text: preserve,
|
||||
clear_missing: true,
|
||||
},
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sync_children(doc: &Doc, blocks_map: &mut Map, block_id: &str, children: &[String]) -> Result<(), ParseError> {
|
||||
let Some(mut block) = blocks_map.get(block_id).and_then(|v| v.to_map()) else {
|
||||
return Err(ParseError::ParserError("Block not found".into()));
|
||||
};
|
||||
|
||||
let current_children = collect_child_ids(&block);
|
||||
if current_children != children {
|
||||
insert_children(doc, &mut block, children)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_tree_ids(node: &StoredNode, output: &mut Vec<String>) {
|
||||
output.push(node.id.clone());
|
||||
for child in &node.children {
|
||||
collect_tree_ids(child, output);
|
||||
}
|
||||
}
|
||||
|
||||
fn check_limits(current: &[StoredNode], target: &[BlockNode]) -> Result<(), ParseError> {
|
||||
let current_count = count_tree_nodes(current);
|
||||
let target_count = count_tree_nodes(target);
|
||||
|
||||
if current_count > MAX_BLOCKS || target_count > MAX_BLOCKS {
|
||||
return Err(ParseError::ParserError("block_count_too_large".into()));
|
||||
}
|
||||
|
||||
if current_count.saturating_mul(target_count) > MAX_LCS_CELLS {
|
||||
return Err(ParseError::ParserError("diff_matrix_too_large".into()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use y_octo::{Any, DocOptions, TextDeltaOp, TextInsert};
|
||||
|
||||
use super::{super::builder::text_ops_from_plain, *};
|
||||
use crate::doc_parser::{
|
||||
block_spec::BlockType, blocksuite::get_string, build_full_doc, markdown::MAX_MARKDOWN_CHARS, parse_doc_to_markdown,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn test_compute_text_diff_simple() {
|
||||
let ops = text_ops_from_plain("hello world");
|
||||
assert_eq!(ops.len(), 1);
|
||||
match &ops[0] {
|
||||
TextDeltaOp::Insert {
|
||||
insert: TextInsert::Text(text),
|
||||
format: None,
|
||||
} => {
|
||||
assert_eq!(text, "hello world");
|
||||
}
|
||||
_ => panic!("unexpected delta op"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_content_block_similarity() {
|
||||
let b1 = BlockSpec {
|
||||
flavour: BlockFlavour::Paragraph,
|
||||
block_type: Some(BlockType::H1),
|
||||
text: text_ops_from_plain("Hello"),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
let b2 = BlockSpec {
|
||||
flavour: BlockFlavour::Paragraph,
|
||||
block_type: Some(BlockType::H1),
|
||||
text: text_ops_from_plain("World"),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
let b3 = BlockSpec {
|
||||
flavour: BlockFlavour::Paragraph,
|
||||
block_type: Some(BlockType::H2),
|
||||
text: text_ops_from_plain("Hello"),
|
||||
checked: None,
|
||||
language: None,
|
||||
order: None,
|
||||
image: None,
|
||||
table: None,
|
||||
bookmark: None,
|
||||
embed_youtube: None,
|
||||
embed_iframe: None,
|
||||
};
|
||||
|
||||
assert!(b1.is_similar(&b2));
|
||||
assert!(!b1.is_similar(&b3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_roundtrip() {
|
||||
let initial_md = "# Test Document\n\nFirst paragraph.\n\nSecond paragraph.";
|
||||
let doc_id = "update-test";
|
||||
|
||||
let initial_bin = build_full_doc("Test Document", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let updated_md = "# Test Document\n\nFirst paragraph.\n\nModified second paragraph.\n\nNew third paragraph.";
|
||||
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
assert!(!delta.is_empty(), "Delta should contain changes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_does_not_update_page_title() {
|
||||
let initial_md = "# Original Title\n\nContent here.";
|
||||
let doc_id = "title-test";
|
||||
|
||||
let initial_bin = build_full_doc("Original Title", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let updated_md = "# New Title\n\nContent here.";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("Should apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map exists");
|
||||
let mut title = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
|
||||
{
|
||||
title = get_string(&block_map, "prop:title");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(title.as_deref(), Some("Original Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_no_changes() {
|
||||
let markdown = "# Same Title\n\nSame content.";
|
||||
let doc_id = "no-change-test";
|
||||
|
||||
let initial_bin = build_full_doc("Same Title", markdown, doc_id).expect("Should create initial doc");
|
||||
let delta = update_doc(&initial_bin, markdown, doc_id).expect("Should compute delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
doc
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("Should apply delta even with no changes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_ignores_ai_editable_comments() {
|
||||
let markdown = "Plain paragraph.";
|
||||
let doc_id = "ai-comment-test";
|
||||
|
||||
let initial_bin = build_full_doc("Title", markdown, doc_id).expect("Should create initial doc");
|
||||
|
||||
let ai_markdown = parse_doc_to_markdown(initial_bin.clone(), doc_id.to_string(), true, None)
|
||||
.expect("parse doc")
|
||||
.markdown;
|
||||
assert!(ai_markdown.contains("block_id="));
|
||||
|
||||
let delta = update_doc(&initial_bin, &ai_markdown, doc_id).expect("Should compute delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("Should apply delta");
|
||||
|
||||
let before = parse_doc_to_markdown(initial_bin, doc_id.to_string(), false, None)
|
||||
.expect("parse before")
|
||||
.markdown;
|
||||
let after = parse_doc_to_markdown(doc.encode_update_v1().unwrap(), doc_id.to_string(), false, None)
|
||||
.expect("parse after")
|
||||
.markdown;
|
||||
|
||||
assert_eq!(after, before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_add_block() {
|
||||
let initial_md = "# Add Block Test\n\nOriginal paragraph.";
|
||||
let doc_id = "add-block-test";
|
||||
|
||||
let initial_bin = build_full_doc("Add Block Test", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
initial_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
|
||||
let updated_md = "# Add Block Test\n\nOriginal paragraph.\n\nNew paragraph added.";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
assert!(!delta.is_empty(), "Delta should contain changes");
|
||||
|
||||
let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("Should apply delta with new block");
|
||||
|
||||
let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
assert!(
|
||||
updated_count > initial_count,
|
||||
"Expected more blocks after insert, got {updated_count} vs {initial_count}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_delete_block() {
|
||||
let initial_md = "# Delete Block Test\n\nFirst paragraph.\n\nSecond paragraph to delete.";
|
||||
let doc_id = "delete-block-test";
|
||||
|
||||
let initial_bin = build_full_doc("Delete Block Test", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
initial_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
|
||||
let updated_md = "# Delete Block Test\n\nFirst paragraph.";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
|
||||
assert!(!delta.is_empty(), "Delta should contain changes");
|
||||
|
||||
let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&initial_bin)
|
||||
.expect("Should apply initial");
|
||||
updated_doc
|
||||
.apply_update_from_binary_v1(&delta)
|
||||
.expect("Should apply delta with block deletion");
|
||||
|
||||
let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
assert!(
|
||||
updated_count < initial_count,
|
||||
"Expected fewer blocks after deletion, got {updated_count} vs {initial_count}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_update_image_caption() {
|
||||
let initial_md = "";
|
||||
let doc_id = "image-update-test";
|
||||
let initial_bin = build_full_doc("Image", initial_md, doc_id).expect("create doc");
|
||||
|
||||
let updated_md = "";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut caption = None;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image")
|
||||
{
|
||||
caption = get_string(&block_map, "prop:caption");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(caption.as_deref(), Some("New Caption"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_update_table_cell() {
|
||||
let initial_md = "| A | B |\n| --- | --- |\n| 1 | 2 |";
|
||||
let doc_id = "table-update-test";
|
||||
let initial_bin = build_full_doc("Table", initial_md, doc_id).expect("create doc");
|
||||
|
||||
let updated_md = "| A | B |\n| --- | --- |\n| 1 | 9 |";
|
||||
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta");
|
||||
|
||||
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial");
|
||||
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
|
||||
|
||||
let blocks_map = doc.get_map("blocks").expect("blocks map");
|
||||
let mut found = false;
|
||||
for (_, value) in blocks_map.iter() {
|
||||
if let Some(block_map) = value.to_map()
|
||||
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table")
|
||||
{
|
||||
for key in block_map.keys() {
|
||||
if key.starts_with("prop:cells.")
|
||||
&& key.ends_with(".text")
|
||||
&& let Some(value) = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a {
|
||||
Any::String(value) => Some(value),
|
||||
_ => None,
|
||||
})
|
||||
&& value == "9"
|
||||
{
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_concurrent_merge_simulation() {
|
||||
let base_md = "# Concurrent Test\n\nBase paragraph.";
|
||||
let doc_id = "concurrent-test";
|
||||
|
||||
let base_bin = build_full_doc("Concurrent Test", base_md, doc_id).expect("Should create base doc");
|
||||
|
||||
let mut base_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
base_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base");
|
||||
let base_count = base_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
|
||||
let client_a_md = "# Concurrent Test\n\nModified by client A.";
|
||||
let delta_a = update_doc(&base_bin, client_a_md, doc_id).expect("Delta A");
|
||||
|
||||
let client_b_md = "# Concurrent Test\n\nBase paragraph.\n\nAdded by client B.";
|
||||
let delta_b = update_doc(&base_bin, client_b_md, doc_id).expect("Delta B");
|
||||
|
||||
let mut final_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
|
||||
final_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base");
|
||||
final_doc.apply_update_from_binary_v1(&delta_a).expect("Apply delta A");
|
||||
final_doc.apply_update_from_binary_v1(&delta_b).expect("Apply delta B");
|
||||
|
||||
let final_count = final_doc.get_map("blocks").expect("blocks map exists").len();
|
||||
assert!(
|
||||
final_count > base_count,
|
||||
"Expected merged blocks after concurrent updates, got {final_count} vs {base_count}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_empty_binary_errors() {
|
||||
let markdown = "# New Document\n\nCreated from empty binary.";
|
||||
let doc_id = "empty-fallback-test";
|
||||
|
||||
let result = update_doc(&[], markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
|
||||
let result = update_doc(&[0, 0], markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_markdown_too_large() {
|
||||
let initial_md = "# Title\n\nContent.";
|
||||
let doc_id = "size-limit-test";
|
||||
let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1);
|
||||
let result = update_doc(&initial_bin, &markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_ydoc_rejects_unsupported_markdown() {
|
||||
let initial_md = "# Title\n\nContent.";
|
||||
let doc_id = "unsupported-test";
|
||||
let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc");
|
||||
|
||||
let markdown = "# Title\n\n<div>HTML</div>";
|
||||
let result = update_doc(&initial_bin, markdown, doc_id);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
+20
@@ -131,6 +131,16 @@ export class ChatContentStreamObjects extends WithDisposable(
|
||||
.data=${streamObject}
|
||||
.width=${this.width}
|
||||
></doc-read-result>`;
|
||||
case 'doc_create':
|
||||
case 'doc_update':
|
||||
case 'doc_update_meta':
|
||||
return html`<doc-write-tool
|
||||
.data=${streamObject}
|
||||
.width=${this.width}
|
||||
.peekViewService=${this.peekViewService}
|
||||
.docDisplayService=${this.docDisplayService}
|
||||
.onOpenDoc=${this.onOpenDoc}
|
||||
></doc-write-tool>`;
|
||||
case 'section_edit':
|
||||
return html`
|
||||
<section-edit-tool
|
||||
@@ -223,6 +233,16 @@ export class ChatContentStreamObjects extends WithDisposable(
|
||||
.peekViewService=${this.peekViewService}
|
||||
.onOpenDoc=${this.onOpenDoc}
|
||||
></doc-read-result>`;
|
||||
case 'doc_create':
|
||||
case 'doc_update':
|
||||
case 'doc_update_meta':
|
||||
return html`<doc-write-tool
|
||||
.data=${streamObject}
|
||||
.width=${this.width}
|
||||
.peekViewService=${this.peekViewService}
|
||||
.docDisplayService=${this.docDisplayService}
|
||||
.onOpenDoc=${this.onOpenDoc}
|
||||
></doc-write-tool>`;
|
||||
case 'section_edit':
|
||||
return html`
|
||||
<section-edit-tool
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
import type { PeekViewService } from '@affine/core/modules/peek-view';
|
||||
import { WithDisposable } from '@blocksuite/global/lit';
|
||||
import { PageIcon, PenIcon } from '@blocksuite/icons/lit';
|
||||
import { ShadowlessElement } from '@blocksuite/std';
|
||||
import type { Signal } from '@preact/signals-core';
|
||||
import { html, nothing } from 'lit';
|
||||
import { property } from 'lit/decorators.js';
|
||||
|
||||
import type { DocDisplayConfig } from '../ai-chat-chips';
|
||||
import type { ToolError } from './type';
|
||||
|
||||
type DocWriteToolName = 'doc_create' | 'doc_update' | 'doc_update_meta';
|
||||
|
||||
type DocWriteToolArgs = {
|
||||
doc_id?: string;
|
||||
title?: string;
|
||||
content?: string;
|
||||
};
|
||||
|
||||
interface DocWriteToolCall {
|
||||
type: 'tool-call';
|
||||
toolCallId: string;
|
||||
toolName: DocWriteToolName;
|
||||
args: DocWriteToolArgs;
|
||||
}
|
||||
|
||||
interface DocWriteToolResult {
|
||||
type: 'tool-result';
|
||||
toolCallId: string;
|
||||
toolName: DocWriteToolName;
|
||||
args: DocWriteToolArgs;
|
||||
result:
|
||||
| {
|
||||
success?: boolean;
|
||||
docId?: string;
|
||||
message?: string;
|
||||
}
|
||||
| ToolError
|
||||
| null;
|
||||
}
|
||||
|
||||
const isToolError = (result: unknown): result is ToolError =>
|
||||
!!result &&
|
||||
typeof result === 'object' &&
|
||||
'type' in result &&
|
||||
(result as ToolError).type === 'error';
|
||||
|
||||
export class DocWriteTool extends WithDisposable(ShadowlessElement) {
|
||||
@property({ attribute: false })
|
||||
accessor data!: DocWriteToolCall | DocWriteToolResult;
|
||||
|
||||
@property({ attribute: false })
|
||||
accessor width: Signal<number | undefined> | undefined;
|
||||
|
||||
@property({ attribute: false })
|
||||
accessor peekViewService!: PeekViewService;
|
||||
|
||||
@property({ attribute: false })
|
||||
accessor docDisplayService!: DocDisplayConfig;
|
||||
|
||||
@property({ attribute: false })
|
||||
accessor onOpenDoc!: (docId: string, sessionId?: string) => void;
|
||||
|
||||
private getDocId() {
|
||||
const { data } = this;
|
||||
if (
|
||||
data.type === 'tool-result' &&
|
||||
data.result &&
|
||||
!isToolError(data.result)
|
||||
) {
|
||||
const docId =
|
||||
typeof data.result.docId === 'string' ? data.result.docId : undefined;
|
||||
if (docId) return docId;
|
||||
}
|
||||
const docId = data.args.doc_id;
|
||||
return typeof docId === 'string' && docId.trim() ? docId : undefined;
|
||||
}
|
||||
|
||||
private getDocTitle(docId?: string) {
|
||||
const { data } = this;
|
||||
if (data.toolName === 'doc_create' || data.toolName === 'doc_update_meta') {
|
||||
const title = data.args.title;
|
||||
if (title) return title;
|
||||
}
|
||||
if (docId && this.docDisplayService) {
|
||||
const title = this.docDisplayService.getTitle(docId);
|
||||
if (title) return title;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private getToolIcon() {
|
||||
return this.data.toolName === 'doc_create' ? PageIcon() : PenIcon();
|
||||
}
|
||||
|
||||
private getCallLabel(title?: string) {
|
||||
switch (this.data.toolName) {
|
||||
case 'doc_create':
|
||||
return title ? `Creating "${title}"` : 'Creating document';
|
||||
case 'doc_update':
|
||||
return title ? `Updating "${title}"` : 'Updating document';
|
||||
case 'doc_update_meta':
|
||||
return title ? `Renaming to "${title}"` : 'Updating document title';
|
||||
default:
|
||||
return 'Updating document';
|
||||
}
|
||||
}
|
||||
|
||||
private getResultLabel(title?: string) {
|
||||
switch (this.data.toolName) {
|
||||
case 'doc_create':
|
||||
return title ? `Created "${title}"` : 'Document created';
|
||||
case 'doc_update':
|
||||
return title ? `Updated "${title}"` : 'Document updated';
|
||||
case 'doc_update_meta':
|
||||
return title ? `Renamed "${title}"` : 'Document title updated';
|
||||
default:
|
||||
return 'Document updated';
|
||||
}
|
||||
}
|
||||
|
||||
private openDoc(docId?: string) {
|
||||
if (!docId) return;
|
||||
if (this.peekViewService) {
|
||||
this.peekViewService.peekView
|
||||
.open({ type: 'doc', docRef: { docId } })
|
||||
.catch(console.error);
|
||||
return;
|
||||
}
|
||||
this.onOpenDoc?.(docId);
|
||||
}
|
||||
|
||||
renderToolCall() {
|
||||
const docId = this.getDocId();
|
||||
const title = this.getDocTitle(docId);
|
||||
return html`<tool-call-card
|
||||
.name=${this.getCallLabel(title)}
|
||||
.icon=${this.getToolIcon()}
|
||||
.width=${this.width}
|
||||
></tool-call-card>`;
|
||||
}
|
||||
|
||||
renderToolResult() {
|
||||
if (this.data.type !== 'tool-result') {
|
||||
return nothing;
|
||||
}
|
||||
|
||||
const result = this.data.result;
|
||||
if (!result || isToolError(result)) {
|
||||
const name = isToolError(result) ? result.name : 'Document action failed';
|
||||
return html`<tool-call-failed
|
||||
.name=${name}
|
||||
.icon=${this.getToolIcon()}
|
||||
></tool-call-failed>`;
|
||||
}
|
||||
|
||||
const docId = this.getDocId();
|
||||
const title = this.getDocTitle(docId) ?? 'Document';
|
||||
const parts: string[] = [];
|
||||
if (result.message) parts.push(result.message);
|
||||
if (docId) parts.push(`Doc ID: ${docId}`);
|
||||
const content = parts.length ? parts.join('\n') : undefined;
|
||||
|
||||
return html`<tool-result-card
|
||||
.name=${this.getResultLabel(title)}
|
||||
.icon=${this.getToolIcon()}
|
||||
.width=${this.width}
|
||||
.results=${[
|
||||
{
|
||||
title,
|
||||
icon: PageIcon(),
|
||||
content,
|
||||
onClick: () => this.openDoc(docId),
|
||||
},
|
||||
]}
|
||||
></tool-result-card>`;
|
||||
}
|
||||
|
||||
protected override render() {
|
||||
if (this.data.type === 'tool-call') {
|
||||
return this.renderToolCall();
|
||||
}
|
||||
if (this.data.type === 'tool-result') {
|
||||
return this.renderToolResult();
|
||||
}
|
||||
return nothing;
|
||||
}
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface HTMLElementTagNameMap {
|
||||
'doc-write-tool': DocWriteTool;
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,7 @@ import { DocEditTool } from './components/ai-tools/doc-edit';
|
||||
import { DocKeywordSearchResult } from './components/ai-tools/doc-keyword-search-result';
|
||||
import { DocReadResult } from './components/ai-tools/doc-read-result';
|
||||
import { DocSemanticSearchResult } from './components/ai-tools/doc-semantic-search-result';
|
||||
import { DocWriteTool } from './components/ai-tools/doc-write';
|
||||
import { SectionEditTool } from './components/ai-tools/section-edit';
|
||||
import { ToolCallCard } from './components/ai-tools/tool-call-card';
|
||||
import { ToolFailedCard } from './components/ai-tools/tool-failed-card';
|
||||
@@ -222,6 +223,7 @@ export function registerAIEffects() {
|
||||
customElements.define('doc-semantic-search-result', DocSemanticSearchResult);
|
||||
customElements.define('doc-keyword-search-result', DocKeywordSearchResult);
|
||||
customElements.define('doc-read-result', DocReadResult);
|
||||
customElements.define('doc-write-tool', DocWriteTool);
|
||||
customElements.define('web-crawl-tool', WebCrawlTool);
|
||||
customElements.define('web-search-tool', WebSearchTool);
|
||||
customElements.define('section-edit-tool', SectionEditTool);
|
||||
|
||||
@@ -102,6 +102,7 @@ export function setupAIProvider(
|
||||
selectedSnapshot: contexts?.selectedSnapshot,
|
||||
selectedMarkdown: contexts?.selectedMarkdown,
|
||||
html: contexts?.html,
|
||||
...(options.docId ? { currentDocId: options.docId } : {}),
|
||||
},
|
||||
endpoint: Endpoint.StreamObject,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user