Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-12 23:56:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): improve markdown parse (#14580)

Browse Source 
#### PR Dependency Tree


* **PR #14580** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Markdown conversion now reports lists of known-unsupported and unknown
block identifiers encountered during parsing, and separates them from
the main markdown output.

* **Bug Fixes**
  * Improved error handling and logging around markdown parsing.

* **Tests**
* Updated tests and snapshots to reflect the new block-list fields and
the adjusted markdown output.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-03-07 00:14:27 +08:00
committed by GitHub
parent 09fa1a8e4e
commit b5d5b71f95
15 changed files with 305 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md
+11 -27
View File
@@ -9,6 +9,16 @@ Generated by [AVA](https://avajs.dev).
> Snapshot 1
{
knownUnsupportedBlocks: [
'RX4CG2zsBk:affine:note',
'S1mkc8zUoU:affine:note',
'yGlBdshAqN:affine:note',
'6lDiuDqZGL:affine:note',
'cauvaHOQmh:affine:note',
'2jwCeO8Yot:affine:note',
'c9MF_JiRgx:affine:note',
'6x7ALjUDjj:affine:surface',
],
markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊
␊
␊
@@ -70,35 +80,9 @@ Generated by [AVA](https://avajs.dev).
␊
␊
␊
␊
[](Bookmark,https://affine.pro/)␊
␊
␊
[](Bookmark,https://www.youtube.com/@affinepro)␊
␊
<img␊
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
alt=""␊
width="1302"␊
height="728"␊
/>␊
␊
<img␊
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
alt=""␊
width="1463"␊
height="374"␊
/>␊
␊
<img␊
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
alt=""␊
width="862"␊
height="1388"␊
/>␊
␊
`,
title: 'Write, Draw, Plan all at Once.',
unknownBlocks: [],
}
## should get doc markdown return null when doc not exists
packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap
BIN
View File
Binary file not shown.
packages/backend/server/src/core/doc-renderer/__tests__/controller.spec.ts
+2
View File
@@ -129,6 +129,8 @@ test('should return markdown content and skip page view when accept is text/mark
const markdown = Sinon.stub(docReader, 'getDocMarkdown').resolves({
title: 'markdown-doc',
markdown: '# markdown-doc',
knownUnsupportedBlocks: [],
unknownBlocks: [],
});
const docContent = Sinon.stub(docReader, 'getDocContent');
const record = Sinon.stub(
packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts
+4
View File
@@ -402,6 +402,8 @@ test('should get doc markdown in json format', async t => {
return {
title: 'test title',
markdown: 'test markdown',
knownUnsupportedBlocks: [],
unknownBlocks: [],
};
});
@@ -418,6 +420,8 @@ test('should get doc markdown in json format', async t => {
.expect({
title: 'test title',
markdown: 'test markdown',
knownUnsupportedBlocks: [],
unknownBlocks: [],
});
t.pass();
});
packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.md
+11 -27
View File
@@ -9,6 +9,16 @@ Generated by [AVA](https://avajs.dev).
> Snapshot 1
{
knownUnsupportedBlocks: [
'RX4CG2zsBk:affine:note',
'S1mkc8zUoU:affine:note',
'yGlBdshAqN:affine:note',
'6lDiuDqZGL:affine:note',
'cauvaHOQmh:affine:note',
'2jwCeO8Yot:affine:note',
'c9MF_JiRgx:affine:note',
'6x7ALjUDjj:affine:surface',
],
markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊
␊
␊
@@ -70,33 +80,7 @@ Generated by [AVA](https://avajs.dev).
␊
␊
␊
␊
[](Bookmark,https://affine.pro/)␊
␊
␊
[](Bookmark,https://www.youtube.com/@affinepro)␊
␊
<img␊
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
alt=""␊
width="1302"␊
height="728"␊
/>␊
␊
<img␊
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
alt=""␊
width="1463"␊
height="374"␊
/>␊
␊
<img␊
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
alt=""␊
width="862"␊
height="1388"␊
/>␊
␊
`,
title: 'Write, Draw, Plan all at Once.',
unknownBlocks: [],
}
packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.snap
BIN
View File
Binary file not shown.
packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md
+11 -27
View File
@@ -9,6 +9,16 @@ Generated by [AVA](https://avajs.dev).
> Snapshot 1
{
knownUnsupportedBlocks: [
'RX4CG2zsBk:affine:note',
'S1mkc8zUoU:affine:note',
'yGlBdshAqN:affine:note',
'6lDiuDqZGL:affine:note',
'cauvaHOQmh:affine:note',
'2jwCeO8Yot:affine:note',
'c9MF_JiRgx:affine:note',
'6x7ALjUDjj:affine:surface',
],
markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊
␊
␊
@@ -70,33 +80,7 @@ Generated by [AVA](https://avajs.dev).
␊
␊
␊
␊
[](Bookmark,https://affine.pro/)␊
␊
␊
[](Bookmark,https://www.youtube.com/@affinepro)␊
␊
<img␊
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
alt=""␊
width="1302"␊
height="728"␊
/>␊
␊
<img␊
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
alt=""␊
width="1463"␊
height="374"␊
/>␊
␊
<img␊
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
alt=""␊
width="862"␊
height="1388"␊
/>␊
␊
`,
title: 'Write, Draw, Plan all at Once.',
unknownBlocks: [],
}
packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap
BIN
View File
Binary file not shown.
packages/backend/server/src/core/doc/reader.ts
+23 -6
View File
@@ -32,6 +32,8 @@ export interface WorkspaceDocInfo {
export interface DocMarkdown {
title: string;
markdown: string;
knownUnsupportedBlocks: string[];
unknownBlocks: string[];
}
export abstract class DocReader {
@@ -185,12 +187,27 @@ export class DatabaseDocReader extends DocReader {
if (!doc) {
return null;
}
return parseDocToMarkdownFromDocSnapshot(
workspaceId,
docId,
doc.bin,
aiEditable
);
try {
const markdown = parseDocToMarkdownFromDocSnapshot(
workspaceId,
docId,
doc.bin,
aiEditable
);
const unknownBlocks = markdown.unknownBlocks ?? [];
if (unknownBlocks.length > 0) {
this.logger.warn(
`Unknown blocks found when parsing markdown for ${workspaceId}/${docId}.`,
{ unknownBlocks }
);
}
return markdown;
} catch (error) {
this.logger.error(`Failed to parse ${workspaceId}/${docId}.`, error);
throw error;
}
}
async getDocDiff(
packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md
+22 -59
View File
@@ -1379,6 +1379,16 @@ Generated by [AVA](https://avajs.dev).
> Snapshot 1
{
knownUnsupportedBlocks: [
'RX4CG2zsBk:affine:note',
'S1mkc8zUoU:affine:note',
'yGlBdshAqN:affine:note',
'6lDiuDqZGL:affine:note',
'cauvaHOQmh:affine:note',
'2jwCeO8Yot:affine:note',
'c9MF_JiRgx:affine:note',
'6x7ALjUDjj:affine:surface',
],
markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊
␊
␊
@@ -1440,35 +1450,9 @@ Generated by [AVA](https://avajs.dev).
␊
␊
␊
␊
[](Bookmark,https://affine.pro/)␊
␊
␊
[](Bookmark,https://www.youtube.com/@affinepro)␊
␊
<img␊
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
alt=""␊
width="1302"␊
height="728"␊
/>␊
␊
<img␊
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
alt=""␊
width="1463"␊
height="374"␊
/>␊
␊
<img␊
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
alt=""␊
width="862"␊
height="1388"␊
/>␊
␊
`,
title: 'Write, Draw, Plan all at Once.',
unknownBlocks: [],
}
## can parse doc to markdown from doc snapshot with ai editable
@@ -1476,6 +1460,16 @@ Generated by [AVA](https://avajs.dev).
> Snapshot 1
{
knownUnsupportedBlocks: [
'RX4CG2zsBk:affine:note',
'S1mkc8zUoU:affine:note',
'yGlBdshAqN:affine:note',
'6lDiuDqZGL:affine:note',
'cauvaHOQmh:affine:note',
'2jwCeO8Yot:affine:note',
'c9MF_JiRgx:affine:note',
'6x7ALjUDjj:affine:surface',
],
markdown: `<!-- block_id=FoPQcAyV_m flavour=affine:paragraph -->␊
AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊
␊
@@ -1565,38 +1559,7 @@ Generated by [AVA](https://avajs.dev).
<!-- block_id=faFteK9eG- flavour=affine:paragraph -->␊
␊
␊
<!-- block_id=ECrtbvW6xx flavour=affine:bookmark -->␊
␊
[](Bookmark,https://affine.pro/)␊
␊
<!-- block_id=5W--UQLN11 flavour=affine:bookmark -->␊
␊
[](Bookmark,https://www.youtube.com/@affinepro)␊
␊
<!-- block_id=lcZphIJe63 flavour=affine:image -->␊
<img␊
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
alt=""␊
width="1302"␊
height="728"␊
/>␊
␊
<!-- block_id=JlgVJdWU12 flavour=affine:image -->␊
<img␊
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
alt=""␊
width="1463"␊
height="374"␊
/>␊
␊
<!-- block_id=lht7AqBqnF flavour=affine:image -->␊
<img␊
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
alt=""␊
width="862"␊
height="1388"␊
/>␊
␊
`,
title: 'Write, Draw, Plan all at Once.',
unknownBlocks: [],
}
packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap
BIN
View File
Binary file not shown.
packages/backend/server/src/core/utils/blocksuite.ts
+10 -1
View File
@@ -16,6 +16,13 @@ export interface WorkspaceDocContent {
avatarKey: string;
}
export interface DocMarkdownContent {
title: string;
markdown: string;
knownUnsupportedBlocks: string[];
unknownBlocks: string[];
}
export interface ParsePageOptions {
maxSummaryLength?: number;
}
@@ -74,7 +81,7 @@ export function parseDocToMarkdownFromDocSnapshot(
docId: string,
docSnapshot: Uint8Array,
aiEditable = false
) {
): DocMarkdownContent {
const docUrlPrefix = workspaceId ? `/workspace/${workspaceId}` : undefined;
const parsed = parseYDocToMarkdown(
Buffer.from(docSnapshot),
@@ -86,5 +93,7 @@ export function parseDocToMarkdownFromDocSnapshot(
return {
title: parsed.title,
markdown: parsed.markdown,
knownUnsupportedBlocks: parsed.knownUnsupportedBlocks ?? [],
unknownBlocks: parsed.unknownBlocks ?? [],
};
}