diff --git a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md index 258c0c252e..de7fb65f3e 100644 --- a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md +++ b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md @@ -1467,3 +1467,122 @@ Generated by [AVA](https://avajs.dev). `, title: 'Write, Draw, Plan all at Once.', } + +## can parse doc to markdown from doc snapshot with ai editable + +> Snapshot 1 + + { + markdown: `␊ + AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + # You own your data, with no compromises␊ + ␊ + ␊ + ␊ + ## Local-first & Real-time collaborative␊ + ␊ + ␊ + ␊ + We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊ + ␊ + ␊ + ␊ + AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ### Blocks that assemble your next docs, tasks kanban or whiteboard␊ + ␊ + ␊ + ␊ + There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊ + ␊ + ␊ + ␊ + We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊ + ␊ + ␊ + ␊ + If you want to learn more about the product design of AFFiNE, here goes the concepts:␊ + ␊ + ␊ + ␊ + To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊ + ␊ + ␊ + ␊ + ## A true canvas for blocks in any form␊ + ␊ + ␊ + ␊ + [Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + "We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊ + ␊ + ␊ + ␊ + * Quip & Notion with their great concept of "everything is a block"␊ + ␊ + ␊ + ␊ + * Trello with their Kanban␊ + ␊ + ␊ + ␊ + * Airtable & Miro with their no-code programable datasheets␊ + ␊ + ␊ + ␊ + * Miro & Whimiscal with their edgeless visual whiteboard␊ + ␊ + ␊ + ␊ + * Remnote & Capacities with their object-based tag system␊ + ␊ + ␊ + ␊ + For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊ + ␊ + ␊ + ␊ + ## Self Host␊ + ␊ + ␊ + ␊ + Self host AFFiNE␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ## Affine Development␊ + ␊ + ␊ + ␊ + For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + `, + title: 'Write, Draw, Plan all at Once.', + } diff --git a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap index 7c77db4c2a..4d48699869 100644 Binary files a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap and b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap differ diff --git a/packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts b/packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts index 90e2a0947f..e337a07ce5 100644 --- a/packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts +++ b/packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts @@ -99,3 +99,14 @@ test('can parse doc to markdown from doc snapshot', async t => { t.snapshot(result); }); + +test('can parse doc to markdown from doc snapshot with ai editable', async t => { + const result = parseDocToMarkdownFromDocSnapshot( + workspace.id, + docSnapshot.id, + docSnapshot.blob, + true + ); + + t.snapshot(result); +}); diff --git a/packages/backend/server/src/core/utils/blocksuite.ts b/packages/backend/server/src/core/utils/blocksuite.ts index ff3f64597c..d522660dac 100644 --- a/packages/backend/server/src/core/utils/blocksuite.ts +++ b/packages/backend/server/src/core/utils/blocksuite.ts @@ -201,7 +201,8 @@ export async function readAllBlocksFromDocSnapshot( export function parseDocToMarkdownFromDocSnapshot( workspaceId: string, docId: string, - docSnapshot: Uint8Array + docSnapshot: Uint8Array, + aiEditable = false ) { const ydoc = new YDoc({ guid: docId, @@ -217,6 +218,7 @@ export function parseDocToMarkdownFromDocSnapshot( buildDocUrl: (docId: string) => { return `/workspace/${workspaceId}/${docId}`; }, + aiEditable, }); return { diff --git a/packages/common/reader/__tests__/__fixtures__/test-doc-with-ai-editable.snapshot.bin b/packages/common/reader/__tests__/__fixtures__/test-doc-with-ai-editable.snapshot.bin new file mode 100644 index 0000000000..96bedb2424 Binary files /dev/null and b/packages/common/reader/__tests__/__fixtures__/test-doc-with-ai-editable.snapshot.bin differ diff --git a/packages/common/reader/__tests__/__snapshots__/reader.spec.ts.snap b/packages/common/reader/__tests__/__snapshots__/reader.spec.ts.snap index 24d325711e..47f8b54d53 100644 --- a/packages/common/reader/__tests__/__snapshots__/reader.spec.ts.snap +++ b/packages/common/reader/__tests__/__snapshots__/reader.spec.ts.snap @@ -555,6 +555,450 @@ For developer or installation guides, please go to [AFFiNE Development](https:// } `; +exports[`should parse page doc work with ai editable 1`] = ` +" +AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. + + + + + + + +# You own your data, with no compromises + + + +## Local-first & Real-time collaborative + + + +We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience. + + + +AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time. + + + + + + + +### Blocks that assemble your next docs, tasks kanban or whiteboard + + + +There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. + + + +We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too. + + + +If you want to learn more about the product design of AFFiNE, here goes the concepts: + + + +To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools. + + + +## A true canvas for blocks in any form + + + +[Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers. + + + + + + + +"We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.: + + + +* Quip & Notion with their great concept of "everything is a block" + + + +* Trello with their Kanban + + + +* Airtable & Miro with their no-code programable datasheets + + + +* Miro & Whimiscal with their edgeless visual whiteboard + + + +* Remnote & Capacities with their object-based tag system + + + +For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap) + + + +## Self Host + + + +Self host AFFiNE + + + + + +## Affine Development + + + +For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start) + + + + + + +" +`; + +exports[`should parse page full doc work with ai editable 1`] = ` +" +# H1 text + + + +List all flavours in one document. + + + +## H2 ~ H6 + + + +### H3 + + + +#### H4 with emoji 😄 + + + +##### H5 + + + +###### H6 + + + +max is H6 + + + +## List + + + +* item 1 + + + +* item 2 + + + * sub item 1 + + + * sub item 2 + + + * super sub item 1 + + + * sub item 3 + + + +* item 3 + + + + + + + + + + + + + + + +sort list + + + +1. item 1 + + + +1. item 2 + + + +1. item 3 + + + 1. sub item 1 + + + 1. sub item 2 + + + 1. super item 1 + + + 1. super item 2 + + + 1. sub item 3 + + + +1. item 4 + + + + + + + + + + + +Table + + + +|c1|c2|c3|c4| +|---|---|---|---| +|v1|v2|v3|| +||||v4| +||v6||v5| + + + + + + + + + + + +Database + + + + + +Code + + + +\`\`\`javascript +console.log('hello world'); +\`\`\` + + + + + + + +Image + + + + +![-HjsQksaVuEaM0KeTEbBYDgWVOmoVzrAeVK0kn4Jfr0=](blob://-HjsQksaVuEaM0KeTEbBYDgWVOmoVzrAeVK0kn4Jfr0=) + + + + + + + +File + + + + +![IrsiJ9XonFXF8fw9tLQXbSsBbUYFfzLgoFpodeidQOU=](blob://IrsiJ9XonFXF8fw9tLQXbSsBbUYFfzLgoFpodeidQOU=) + + + + + + + +> foo bar quote text + + + + + + + + + + + + +--- + + + + + + + +TeX + + + + + + + + + + + +2025-06-18 13:15 + + + + + + + + + +Mind Map + + + + + + + + + + + +A Link + + + + +[null](doc://FmHFPAPzp51JjFP89aZ-b) + + + +Todo List + + + +- [ ] abc + + + +- [ ] edf + + + - [x] done1 + + + +- [ ] end + + + + + + + +~~delete text~~ + + + + + + + +**Bold text** + + + + + + + +Underline + + + + + + + +Youtube + + + + + + + + + + + + +## end + + + +this is end + + + + + + +" +`; + exports[`should read all doc ids from root doc snapshot work 1`] = ` [ "5nS9BSp3Px", diff --git a/packages/common/reader/__tests__/reader.spec.ts b/packages/common/reader/__tests__/reader.spec.ts index 5323915051..6ca3a81d46 100644 --- a/packages/common/reader/__tests__/reader.spec.ts +++ b/packages/common/reader/__tests__/reader.spec.ts @@ -17,6 +17,12 @@ const rootDocSnapshot = readFileSync( const docSnapshot = readFileSync( path.join(import.meta.dirname, './__fixtures__/test-doc.snapshot.bin') ); +const docSnapshotWithAiEditable = readFileSync( + path.join( + import.meta.dirname, + './__fixtures__/test-doc-with-ai-editable.snapshot.bin' + ) +); test('should read doc blocks work', async () => { const rootDoc = new YDoc({ @@ -118,3 +124,39 @@ test('should parse page doc work', () => { expect(result).toMatchSnapshot(); }); + +test('should parse page doc work with ai editable', () => { + const doc = new YDoc({ + guid: 'test-doc', + }); + applyUpdate(doc, docSnapshot); + + const result = parsePageDoc({ + workspaceId: 'test-space', + doc, + buildBlobUrl: id => `blob://${id}`, + buildDocUrl: id => `doc://${id}`, + renderDocTitle: id => `Doc Title ${id}`, + aiEditable: true, + }); + + expect(result.md).toMatchSnapshot(); +}); + +test('should parse page full doc work with ai editable', () => { + const doc = new YDoc({ + guid: 'test-doc', + }); + applyUpdate(doc, docSnapshotWithAiEditable); + + const result = parsePageDoc({ + workspaceId: 'test-space', + doc, + buildBlobUrl: id => `blob://${id}`, + buildDocUrl: id => `doc://${id}`, + renderDocTitle: id => `Doc Title ${id}`, + aiEditable: true, + }); + + expect(result.md).toMatchSnapshot(); +}); diff --git a/packages/common/reader/src/doc-parser/parser.ts b/packages/common/reader/src/doc-parser/parser.ts index d3e92cd380..3f36f779cb 100644 --- a/packages/common/reader/src/doc-parser/parser.ts +++ b/packages/common/reader/src/doc-parser/parser.ts @@ -34,7 +34,9 @@ export const parseBlockToMd = ( export function parseBlock( context: ParserContext, yBlock: YBlock | undefined, - yBlocks: YBlocks // all blocks + yBlocks: YBlocks, // all blocks + aiEditable = false, + blockLevel = 0 ): ParsedBlock | null { if (!yBlock) { return null; @@ -73,6 +75,8 @@ export function parseBlock( return result; } + let placeholder = false; + try { switch (flavour) { case 'affine:paragraph': { @@ -100,7 +104,12 @@ export function parseBlock( break; } case 'affine:list': { - result.content = (type === 'bulleted' ? '* ' : '1. ') + toMd() + '\n'; + let prefix = type === 'bulleted' ? '* ' : '1. '; + if (type === 'todo') { + const checked = yBlock.get('prop:checked') as boolean; + prefix = checked ? '- [x] ' : '- [ ] '; + } + result.content = prefix + toMd() + '\n'; break; } case 'affine:code': { @@ -218,7 +227,9 @@ export function parseBlock( const child = parseBlock( context, yBlocks.get(cid) as YBlock | undefined, - yBlocks + yBlocks, + aiEditable, + blockLevel + 1 ); if (!child) { return [cid, '']; @@ -375,6 +386,7 @@ export function parseBlock( } default: { // console.warn("Unknown or unsupported flavour", flavour); + placeholder = true; } } @@ -385,7 +397,9 @@ export function parseBlock( parseBlock( context, yBlocks.get(cid) as YBlock | undefined, - yBlocks + yBlocks, + aiEditable, + blockLevel + 1 ) ) .filter( @@ -397,6 +411,16 @@ export function parseBlock( } catch (e) { console.warn('Error converting block to md', e); } + + if (result.content && aiEditable && blockLevel === 2) { + // add a placeholder comment for the block level 2 + if (flavour === 'affine:database' || placeholder) { + result.content = `\n`; + result.children = []; + } else { + result.content = `\n${result.content}`; + } + } return result; } @@ -416,7 +440,7 @@ export const parsePageDoc = (ctx: ParserContext): ParsedDoc => { } else { const yPage = yBlocks.get(maybePageBlock[0]) as YBlock; const title = yPage.get('prop:title') as YText; - const rootBlock = parseBlock(ctx, yPage, yBlocks); + const rootBlock = parseBlock(ctx, yPage, yBlocks, ctx.aiEditable); if (!rootBlock) { return { title: '', diff --git a/packages/common/reader/src/doc-parser/types.ts b/packages/common/reader/src/doc-parser/types.ts index c88ce3e7bb..e95ba6b5fa 100644 --- a/packages/common/reader/src/doc-parser/types.ts +++ b/packages/common/reader/src/doc-parser/types.ts @@ -149,4 +149,5 @@ export interface ParserContext { buildBlobUrl: (blobId: string) => string; buildDocUrl: (docId: string) => string; renderDocTitle?: (docId: string) => string; + aiEditable?: boolean; }