From 6792c3e6569b1b9c7a5461ee8e21535ce60e0b45 Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Wed, 14 May 2025 14:52:41 +0000 Subject: [PATCH] feat(common): add blocksuite reader lib (#11955) close CLOUD-202 --- packages/common/nbstore/package.json | 1 + .../nbstore/src/sync/indexer/crawler.ts | 824 +------- .../common/nbstore/src/sync/indexer/index.ts | 34 +- packages/common/nbstore/tsconfig.json | 1 + packages/common/reader/README.md | 25 + .../__fixtures__/test-doc.snapshot.bin | Bin 0 -> 36447 bytes .../__fixtures__/test-root-doc.snapshot.bin | Bin 0 -> 296 bytes .../__snapshots__/reader.spec.ts.snap | 1758 +++++++++++++++++ .../common/reader/__tests__/reader.spec.ts | 90 + packages/common/reader/esbuild.config.js | 23 + packages/common/reader/package.json | 27 + .../sync/indexer => reader/src}/bs-store.ts | 0 packages/common/reader/src/index.ts | 1 + packages/common/reader/src/reader.ts | 905 +++++++++ packages/common/reader/tsconfig.json | 10 + tools/utils/src/workspace.gen.ts | 7 + tsconfig.json | 1 + yarn.lock | 14 + 18 files changed, 2877 insertions(+), 844 deletions(-) create mode 100644 packages/common/reader/README.md create mode 100644 packages/common/reader/__tests__/__fixtures__/test-doc.snapshot.bin create mode 100644 packages/common/reader/__tests__/__fixtures__/test-root-doc.snapshot.bin create mode 100644 packages/common/reader/__tests__/__snapshots__/reader.spec.ts.snap create mode 100644 packages/common/reader/__tests__/reader.spec.ts create mode 100644 packages/common/reader/esbuild.config.js create mode 100644 packages/common/reader/package.json rename packages/common/{nbstore/src/sync/indexer => reader/src}/bs-store.ts (100%) create mode 100644 packages/common/reader/src/index.ts create mode 100644 packages/common/reader/src/reader.ts create mode 100644 packages/common/reader/tsconfig.json diff --git a/packages/common/nbstore/package.json b/packages/common/nbstore/package.json index d7b44a97d4..db70d3e3bc 100644 --- a/packages/common/nbstore/package.json +++ b/packages/common/nbstore/package.json @@ -18,6 +18,7 @@ "./frontend": "./src/frontend/index.ts" }, "dependencies": { + "@affine/reader": "workspace:*", "@toeverything/infra": "workspace:*", "eventemitter2": "^6.4.9", "graphemer": "^1.4.0", diff --git a/packages/common/nbstore/src/sync/indexer/crawler.ts b/packages/common/nbstore/src/sync/indexer/crawler.ts index 13c6a4e41f..aedbba1095 100644 --- a/packages/common/nbstore/src/sync/indexer/crawler.ts +++ b/packages/common/nbstore/src/sync/indexer/crawler.ts @@ -1,440 +1,7 @@ -import { Container } from '@blocksuite/affine/global/di'; -import type { - AttachmentBlockModel, - BookmarkBlockModel, - EmbedBlockModel, - ImageBlockModel, - TableBlockModel, -} from '@blocksuite/affine/model'; -import { AffineSchemas } from '@blocksuite/affine/schemas'; -import { MarkdownAdapter } from '@blocksuite/affine/shared/adapters'; -import type { AffineTextAttributes } from '@blocksuite/affine/shared/types'; -import { - createYProxy, - type DeltaInsert, - type DraftModel, - Schema, - Transformer, - type TransformerMiddleware, - type YBlock, -} from '@blocksuite/affine/store'; -import { uniq } from 'lodash-es'; -import { - Array as YArray, - type Doc as YDoc, - Map as YMap, - Text as YText, -} from 'yjs'; +import { readAllBlocksFromDoc } from '@affine/reader'; +import { type Doc as YDoc } from 'yjs'; import { IndexerDocument } from '../../storage'; -import { getStoreManager } from './bs-store'; - -const blocksuiteSchema = new Schema(); -blocksuiteSchema.register([...AffineSchemas]); - -interface BlockDocumentInfo { - docId: string; - blockId: string; - content?: string | string[]; - flavour: string; - blob?: string[]; - refDocId?: string[]; - ref?: string[]; - parentFlavour?: string; - parentBlockId?: string; - additional?: { - databaseName?: string; - displayMode?: string; - noteBlockId?: string; - }; - yblock: YMap; - markdownPreview?: string; -} - -const bookmarkFlavours = new Set([ - 'affine:bookmark', - 'affine:embed-youtube', - 'affine:embed-figma', - 'affine:embed-github', - 'affine:embed-loom', -]); - -function generateMarkdownPreviewBuilder( - yRootDoc: YDoc, - workspaceId: string, - blocks: BlockDocumentInfo[] -) { - function yblockToDraftModal(yblock: YBlock): DraftModel | null { - const flavour = yblock.get('sys:flavour') as string; - const blockSchema = blocksuiteSchema.flavourSchemaMap.get(flavour); - if (!blockSchema) { - return null; - } - const keys = Array.from(yblock.keys()) - .filter(key => key.startsWith('prop:')) - .map(key => key.substring(5)); - - const props = Object.fromEntries( - keys.map(key => [key, createYProxy(yblock.get(`prop:${key}`))]) - ); - - return { - props, - id: yblock.get('sys:id') as string, - flavour, - children: [], - role: blockSchema.model.role, - version: (yblock.get('sys:version') as number) ?? blockSchema.version, - keys: Array.from(yblock.keys()) - .filter(key => key.startsWith('prop:')) - .map(key => key.substring(5)), - } as unknown as DraftModel; - } - - const titleMiddleware: TransformerMiddleware = ({ adapterConfigs }) => { - const pages = yRootDoc.getMap('meta').get('pages'); - if (!(pages instanceof YArray)) { - return; - } - for (const meta of pages.toArray()) { - adapterConfigs.set( - 'title:' + meta.get('id'), - meta.get('title')?.toString() ?? 'Untitled' - ); - } - }; - - const baseUrl = `/workspace/${workspaceId}`; - - function getDocLink(docId: string, blockId: string) { - const searchParams = new URLSearchParams(); - searchParams.set('blockIds', blockId); - return `${baseUrl}/${docId}?${searchParams.toString()}`; - } - - const docLinkBaseURLMiddleware: TransformerMiddleware = ({ - adapterConfigs, - }) => { - adapterConfigs.set('docLinkBaseUrl', baseUrl); - }; - - const container = new Container(); - getStoreManager() - .get('store') - .forEach(ext => { - ext.setup(container); - }); - - const provider = container.provider(); - const markdownAdapter = new MarkdownAdapter( - new Transformer({ - schema: blocksuiteSchema, - blobCRUD: { - delete: () => Promise.resolve(), - get: () => Promise.resolve(null), - list: () => Promise.resolve([]), - set: () => Promise.resolve(''), - }, - docCRUD: { - create: () => { - throw new Error('Not implemented'); - }, - get: () => null, - delete: () => {}, - }, - middlewares: [docLinkBaseURLMiddleware, titleMiddleware], - }), - provider - ); - - const markdownPreviewCache = new WeakMap(); - - function trimCodeBlock(markdown: string) { - const lines = markdown.split('\n').filter(line => line.trim() !== ''); - if (lines.length > 5) { - return [...lines.slice(0, 4), '...', lines.at(-1), ''].join('\n'); - } - return [...lines, ''].join('\n'); - } - - function trimParagraph(markdown: string) { - const lines = markdown.split('\n').filter(line => line.trim() !== ''); - - if (lines.length > 3) { - return [...lines.slice(0, 3), '...', lines.at(-1), ''].join('\n'); - } - - return [...lines, ''].join('\n'); - } - - function getListDepth(block: BlockDocumentInfo) { - let parentBlockCount = 0; - let currentBlock: BlockDocumentInfo | undefined = block; - do { - currentBlock = blocks.find( - b => b.blockId === currentBlock?.parentBlockId - ); - - // reach the root block. do not count it. - if (!currentBlock || currentBlock.flavour !== 'affine:list') { - break; - } - parentBlockCount++; - } while (currentBlock); - return parentBlockCount; - } - - // only works for list block - function indentMarkdown(markdown: string, depth: number) { - if (depth <= 0) { - return markdown; - } - - return ( - markdown - .split('\n') - .map(line => ' '.repeat(depth) + line) - .join('\n') + '\n' - ); - } - - const generateDatabaseMarkdownPreview = (block: BlockDocumentInfo) => { - const isDatabaseBlock = (block: BlockDocumentInfo) => { - return block.flavour === 'affine:database'; - }; - - const model = yblockToDraftModal(block.yblock); - - if (!model) { - return null; - } - - let dbBlock: BlockDocumentInfo | null = null; - - if (isDatabaseBlock(block)) { - dbBlock = block; - } else { - const parentBlock = blocks.find(b => b.blockId === block.parentBlockId); - - if (parentBlock && isDatabaseBlock(parentBlock)) { - dbBlock = parentBlock; - } - } - - if (!dbBlock) { - return null; - } - - const url = getDocLink(block.docId, dbBlock.blockId); - const title = dbBlock.additional?.databaseName; - - return `[database · ${title || 'Untitled'}][](${url})\n`; - }; - - const generateImageMarkdownPreview = (block: BlockDocumentInfo) => { - const isImageModel = ( - model: DraftModel | null - ): model is DraftModel => { - return model?.flavour === 'affine:image'; - }; - - const model = yblockToDraftModal(block.yblock); - - if (!isImageModel(model)) { - return null; - } - - const info = ['an image block']; - - if (model.props.sourceId) { - info.push(`file id ${model.props.sourceId}`); - } - - if (model.props.caption) { - info.push(`with caption ${model.props.caption}`); - } - - return info.join(', ') + '\n'; - }; - - const generateEmbedMarkdownPreview = (block: BlockDocumentInfo) => { - const isEmbedModel = ( - model: DraftModel | null - ): model is DraftModel => { - return ( - model?.flavour === 'affine:embed-linked-doc' || - model?.flavour === 'affine:embed-synced-doc' - ); - }; - - const draftModel = yblockToDraftModal(block.yblock); - if (!isEmbedModel(draftModel)) { - return null; - } - - const url = getDocLink(block.docId, draftModel.id); - - return `[](${url})\n`; - }; - - const generateLatexMarkdownPreview = (block: BlockDocumentInfo) => { - let content = - typeof block.content === 'string' - ? block.content.trim() - : block.content?.join('').trim(); - - content = content?.split('\n').join(' ') ?? ''; - - return `LaTeX, with value ${content}\n`; - }; - - const generateBookmarkMarkdownPreview = (block: BlockDocumentInfo) => { - const isBookmarkModel = ( - model: DraftModel | null - ): model is DraftModel => { - return bookmarkFlavours.has(model?.flavour ?? ''); - }; - - const draftModel = yblockToDraftModal(block.yblock); - if (!isBookmarkModel(draftModel)) { - return null; - } - const title = draftModel.props.title; - const url = draftModel.props.url; - return `[${title}](${url})\n`; - }; - - const generateAttachmentMarkdownPreview = (block: BlockDocumentInfo) => { - const isAttachmentModel = ( - model: DraftModel | null - ): model is DraftModel => { - return model?.flavour === 'affine:attachment'; - }; - - const draftModel = yblockToDraftModal(block.yblock); - if (!isAttachmentModel(draftModel)) { - return null; - } - - return `[${draftModel.props.name}](${draftModel.props.sourceId})\n`; - }; - - const generateTableMarkdownPreview = (block: BlockDocumentInfo) => { - const isTableModel = ( - model: DraftModel | null - ): model is DraftModel => { - return model?.flavour === 'affine:table'; - }; - - const draftModel = yblockToDraftModal(block.yblock); - if (!isTableModel(draftModel)) { - return null; - } - - const url = getDocLink(block.docId, draftModel.id); - - return `[table][](${url})\n`; - }; - - const generateMarkdownPreview = async (block: BlockDocumentInfo) => { - if (markdownPreviewCache.has(block)) { - return markdownPreviewCache.get(block); - } - const flavour = block.flavour; - let markdown: string | null = null; - - if ( - flavour === 'affine:paragraph' || - flavour === 'affine:list' || - flavour === 'affine:code' - ) { - const draftModel = yblockToDraftModal(block.yblock); - markdown = - block.parentFlavour === 'affine:database' - ? generateDatabaseMarkdownPreview(block) - : ((draftModel ? await markdownAdapter.fromBlock(draftModel) : null) - ?.file ?? null); - - if (markdown) { - if (flavour === 'affine:code') { - markdown = trimCodeBlock(markdown); - } else if (flavour === 'affine:paragraph') { - markdown = trimParagraph(markdown); - } - } - } else if (flavour === 'affine:database') { - markdown = generateDatabaseMarkdownPreview(block); - } else if ( - flavour === 'affine:embed-linked-doc' || - flavour === 'affine:embed-synced-doc' - ) { - markdown = generateEmbedMarkdownPreview(block); - } else if (flavour === 'affine:attachment') { - markdown = generateAttachmentMarkdownPreview(block); - } else if (flavour === 'affine:image') { - markdown = generateImageMarkdownPreview(block); - } else if (flavour === 'affine:surface' || flavour === 'affine:page') { - // skip - } else if (flavour === 'affine:latex') { - markdown = generateLatexMarkdownPreview(block); - } else if (bookmarkFlavours.has(flavour)) { - markdown = generateBookmarkMarkdownPreview(block); - } else if (flavour === 'affine:table') { - markdown = generateTableMarkdownPreview(block); - } else { - console.warn(`unknown flavour: ${flavour}`); - } - - if (markdown && flavour === 'affine:list') { - const blockDepth = getListDepth(block); - markdown = indentMarkdown(markdown, Math.max(0, blockDepth)); - } - - markdownPreviewCache.set(block, markdown); - return markdown; - }; - - return generateMarkdownPreview; -} - -// remove the indent of the first line of list -// e.g., -// ``` -// - list item 1 -// - list item 2 -// ``` -// becomes -// ``` -// - list item 1 -// - list item 2 -// ``` -function unindentMarkdown(markdown: string) { - const lines = markdown.split('\n'); - const res: string[] = []; - let firstListFound = false; - let baseIndent = 0; - - for (let current of lines) { - const indent = current.match(/^\s*/)?.[0]?.length ?? 0; - - if (indent > 0) { - if (!firstListFound) { - // For the first list item, remove all indentation - firstListFound = true; - baseIndent = indent; - current = current.trimStart(); - } else { - // For subsequent list items, maintain relative indentation - current = - ' '.repeat(Math.max(0, indent - baseIndent)) + current.trimStart(); - } - } - - res.push(current); - } - - return res.join('\n'); -} export async function crawlingDocData({ ydoc, @@ -453,391 +20,18 @@ export async function crawlingDocData({ } | undefined > { - let docTitle = ''; - let summaryLenNeeded = 1000; - let summary = ''; - const blockDocuments: BlockDocumentInfo[] = []; - - const generateMarkdownPreview = generateMarkdownPreviewBuilder( + const result = await readAllBlocksFromDoc({ + ydoc, rootYDoc, spaceId, - blockDocuments - ); - - const blocks = ydoc.getMap('blocks'); - - // build a parent map for quick lookup - // for each block, record its parent id - const parentMap: Record = {}; - for (const [id, block] of blocks.entries()) { - const children = block.get('sys:children') as YArray | undefined; - if (children instanceof YArray && children.length) { - for (const child of children) { - parentMap[child] = id; - } - } - } - - if (blocks.size === 0) { + maxSummaryLength: 1000, + }); + if (!result) { return undefined; } - // find the nearest block that satisfies the predicate - const nearest = ( - blockId: string, - predicate: (block: YMap) => boolean - ) => { - let current: string | null = blockId; - while (current) { - const block = blocks.get(current); - if (block && predicate(block)) { - return block; - } - current = parentMap[current] ?? null; - } - return null; - }; - - const nearestByFlavour = (blockId: string, flavour: string) => - nearest(blockId, block => block.get('sys:flavour') === flavour); - - let rootBlockId: string | null = null; - for (const block of blocks.values()) { - const flavour = block.get('sys:flavour')?.toString(); - const blockId = block.get('sys:id')?.toString(); - if (flavour === 'affine:page' && blockId) { - rootBlockId = blockId; - } - } - - if (!rootBlockId) { - return undefined; - } - - const queue: { parent?: string; id: string }[] = [{ id: rootBlockId }]; - const visited = new Set(); // avoid loop - - const pushChildren = (id: string, block: YMap) => { - const children = block.get('sys:children'); - if (children instanceof YArray && children.length) { - for (let i = children.length - 1; i >= 0; i--) { - const childId = children.get(i); - if (childId && !visited.has(childId)) { - queue.push({ parent: id, id: childId }); - visited.add(childId); - } - } - } - }; - - // #region first loop - generate block base info - while (queue.length) { - const next = queue.pop(); - if (!next) { - break; - } - - const { parent: parentBlockId, id: blockId } = next; - const block = blockId ? blocks.get(blockId) : null; - const parentBlock = parentBlockId ? blocks.get(parentBlockId) : null; - if (!block) { - break; - } - - const flavour = block.get('sys:flavour')?.toString(); - const parentFlavour = parentBlock?.get('sys:flavour')?.toString(); - const noteBlock = nearestByFlavour(blockId, 'affine:note'); - - // display mode: - // - both: page and edgeless -> fallback to page - // - page: only page -> page - // - edgeless: only edgeless -> edgeless - // - undefined: edgeless (assuming it is a normal element on the edgeless) - let displayMode = noteBlock?.get('prop:displayMode') ?? 'edgeless'; - - if (displayMode === 'both') { - displayMode = 'page'; - } - - const noteBlockId: string | undefined = noteBlock - ?.get('sys:id') - ?.toString(); - - pushChildren(blockId, block); - - const commonBlockProps = { - docId: ydoc.guid, - flavour, - blockId, - yblock: block, - additional: { displayMode, noteBlockId }, - }; - - if (flavour === 'affine:page') { - docTitle = block.get('prop:title').toString(); - blockDocuments.push({ ...commonBlockProps, content: docTitle }); - } else if ( - flavour === 'affine:paragraph' || - flavour === 'affine:list' || - flavour === 'affine:code' - ) { - const text = block.get('prop:text') as YText; - - if (!text) { - continue; - } - - const deltas: DeltaInsert[] = text.toDelta(); - const refs = uniq( - deltas - .flatMap(delta => { - if ( - delta.attributes && - delta.attributes.reference && - delta.attributes.reference.pageId - ) { - const { pageId: refDocId, params = {} } = - delta.attributes.reference; - return { - refDocId, - ref: JSON.stringify({ docId: refDocId, ...params }), - }; - } - return null; - }) - .filter(ref => !!ref) - ); - - const databaseName = - flavour === 'affine:paragraph' && parentFlavour === 'affine:database' // if block is a database row - ? parentBlock?.get('prop:title')?.toString() - : undefined; - - blockDocuments.push({ - ...commonBlockProps, - content: text.toString(), - ...refs.reduce<{ refDocId: string[]; ref: string[] }>( - (prev, curr) => { - prev.refDocId.push(curr.refDocId); - prev.ref.push(curr.ref); - return prev; - }, - { refDocId: [], ref: [] } - ), - parentFlavour, - parentBlockId, - additional: { ...commonBlockProps.additional, databaseName }, - }); - - if (summaryLenNeeded > 0) { - summary += text.toString(); - summaryLenNeeded -= text.length; - } - } else if ( - flavour === 'affine:embed-linked-doc' || - flavour === 'affine:embed-synced-doc' - ) { - const pageId = block.get('prop:pageId'); - if (typeof pageId === 'string') { - // reference info - const params = block.get('prop:params') ?? {}; - blockDocuments.push({ - ...commonBlockProps, - refDocId: [pageId], - ref: [JSON.stringify({ docId: pageId, ...params })], - parentFlavour, - parentBlockId, - }); - } - } else if (flavour === 'affine:attachment' || flavour === 'affine:image') { - const blobId = block.get('prop:sourceId'); - if (typeof blobId === 'string') { - blockDocuments.push({ - ...commonBlockProps, - blob: [blobId], - parentFlavour, - parentBlockId, - }); - } - } else if (flavour === 'affine:surface') { - const texts = []; - - const elementsObj = block.get('prop:elements'); - if ( - !( - elementsObj instanceof YMap && - elementsObj.get('type') === '$blocksuite:internal:native$' - ) - ) { - continue; - } - const elements = elementsObj.get('value') as YMap; - if (!(elements instanceof YMap)) { - continue; - } - - for (const element of elements.values()) { - if (!(element instanceof YMap)) { - continue; - } - const text = element.get('text') as YText; - if (!text) { - continue; - } - - texts.push(text.toString()); - } - - blockDocuments.push({ - ...commonBlockProps, - content: texts, - parentFlavour, - parentBlockId, - }); - } else if (flavour === 'affine:database') { - const texts = []; - const columnsObj = block.get('prop:columns'); - const databaseTitle = block.get('prop:title'); - if (databaseTitle instanceof YText) { - texts.push(databaseTitle.toString()); - } - if (columnsObj instanceof YArray) { - for (const column of columnsObj) { - if (!(column instanceof YMap)) { - continue; - } - if (typeof column.get('name') === 'string') { - texts.push(column.get('name')); - } - - const data = column.get('data'); - if (!(data instanceof YMap)) { - continue; - } - const options = data.get('options'); - if (!(options instanceof YArray)) { - continue; - } - for (const option of options) { - if (!(option instanceof YMap)) { - continue; - } - const value = option.get('value'); - if (typeof value === 'string') { - texts.push(value); - } - } - } - } - - blockDocuments.push({ - ...commonBlockProps, - content: texts, - additional: { - ...commonBlockProps.additional, - databaseName: databaseTitle?.toString(), - }, - }); - } else if (flavour === 'affine:latex') { - blockDocuments.push({ - ...commonBlockProps, - content: block.get('prop:latex')?.toString() ?? '', - }); - } else if (flavour === 'affine:table') { - const contents = Array.from(block.keys()) - .map(key => { - if (key.startsWith('prop:cells.') && key.endsWith('.text')) { - return block.get(key)?.toString() ?? ''; - } - return ''; - }) - .filter(Boolean); - blockDocuments.push({ - ...commonBlockProps, - content: contents, - }); - } else if (bookmarkFlavours.has(flavour)) { - blockDocuments.push({ ...commonBlockProps }); - } - } - // #endregion - - // #region second loop - generate markdown preview - const TARGET_PREVIEW_CHARACTER = 500; - const TARGET_PREVIOUS_BLOCK = 1; - const TARGET_FOLLOW_BLOCK = 4; - for (const block of blockDocuments) { - if (block.ref?.length) { - const target = block; - - // should only generate the markdown preview belong to the same affine:note - const noteBlock = nearestByFlavour(block.blockId, 'affine:note'); - - const sameNoteBlocks = noteBlock - ? blockDocuments.filter( - candidate => - nearestByFlavour(candidate.blockId, 'affine:note') === noteBlock - ) - : []; - - // only generate markdown preview for reference blocks - let previewText = (await generateMarkdownPreview(target)) ?? ''; - let previousBlock = 0; - let followBlock = 0; - let previousIndex = sameNoteBlocks.findIndex( - block => block.blockId === target.blockId - ); - let followIndex = previousIndex; - - while ( - !( - ( - previewText.length > TARGET_PREVIEW_CHARACTER || // stop if preview text reaches the limit - ((previousBlock >= TARGET_PREVIOUS_BLOCK || previousIndex < 0) && - (followBlock >= TARGET_FOLLOW_BLOCK || - followIndex >= sameNoteBlocks.length)) - ) // stop if no more blocks, or preview block reaches the limit - ) - ) { - if (previousBlock < TARGET_PREVIOUS_BLOCK) { - previousIndex--; - const block = - previousIndex >= 0 ? sameNoteBlocks.at(previousIndex) : null; - const markdown = block ? await generateMarkdownPreview(block) : null; - if ( - markdown && - !previewText.startsWith( - markdown - ) /* A small hack to skip blocks with the same content */ - ) { - previewText = markdown + '\n' + previewText; - previousBlock++; - } - } - - if (followBlock < TARGET_FOLLOW_BLOCK) { - followIndex++; - const block = sameNoteBlocks.at(followIndex); - const markdown = block ? await generateMarkdownPreview(block) : null; - if ( - markdown && - !previewText.endsWith( - markdown - ) /* A small hack to skip blocks with the same content */ - ) { - previewText = previewText + '\n' + markdown; - followBlock++; - } - } - } - - block.markdownPreview = unindentMarkdown(previewText); - } - } - // #endregion - return { - blocks: blockDocuments.map(block => + blocks: result.blocks.map(block => IndexerDocument.from<'block'>(`${docId}:${block.blockId}`, { docId: block.docId, blockId: block.blockId, @@ -854,6 +48,6 @@ export async function crawlingDocData({ markdownPreview: block.markdownPreview, }) ), - preview: summary, + preview: result.summary, }; } diff --git a/packages/common/nbstore/src/sync/indexer/index.ts b/packages/common/nbstore/src/sync/indexer/index.ts index 9dfe446c8a..0e6f5a3f8c 100644 --- a/packages/common/nbstore/src/sync/indexer/index.ts +++ b/packages/common/nbstore/src/sync/indexer/index.ts @@ -1,3 +1,4 @@ +import { readAllDocsFromRootDoc } from '@affine/reader'; import { filter, first, @@ -7,12 +8,7 @@ import { Subject, throttleTime, } from 'rxjs'; -import { - applyUpdate, - type Array as YArray, - Doc as YDoc, - type Map as YMap, -} from 'yjs'; +import { applyUpdate, Doc as YDoc } from 'yjs'; import { type DocStorage, @@ -432,29 +428,9 @@ export class IndexerSyncImpl implements IndexerSync { * Get all docs from the root doc, without deleted docs */ private getAllDocsFromRootDoc() { - const docs = this.status.rootDoc.getMap('meta').get('pages') as - | YArray> - | undefined; - const availableDocs = new Map(); - - if (docs) { - for (const page of docs) { - const docId = page.get('id'); - - if (typeof docId !== 'string') { - continue; - } - - const inTrash = page.get('trash') ?? false; - const title = page.get('title'); - - if (!inTrash) { - availableDocs.set(docId, { title }); - } - } - } - - return availableDocs; + return readAllDocsFromRootDoc(this.status.rootDoc, { + includeTrash: false, + }); } private async getAllDocsFromIndexer() { diff --git a/packages/common/nbstore/tsconfig.json b/packages/common/nbstore/tsconfig.json index a03bd33485..515ba78b71 100644 --- a/packages/common/nbstore/tsconfig.json +++ b/packages/common/nbstore/tsconfig.json @@ -7,6 +7,7 @@ "tsBuildInfoFile": "./dist/tsconfig.tsbuildinfo" }, "references": [ + { "path": "../reader" }, { "path": "../infra" }, { "path": "../error" }, { "path": "../graphql" }, diff --git a/packages/common/reader/README.md b/packages/common/reader/README.md new file mode 100644 index 0000000000..79a83f9dd9 --- /dev/null +++ b/packages/common/reader/README.md @@ -0,0 +1,25 @@ +# Affine Blocksuite format YDoc reader + +## Usage + +### read rootYDoc + +```ts +import { readAllDocsFromRootDoc } from '@affine/reader'; + +const docs = readAllDocsFromRootDoc(rootDoc); +console.log(Array.from(docsWithTrash.entries())); + +// [ +// 'doc-id-1', { title: 'test doc title' }, +// // ... +// ] +``` + +### read YDoc + +```ts +import { readAllBlocksFromDoc } from '@affine/reader'; + +const blocks = readAllBlocksFromDoc(doc); +``` diff --git a/packages/common/reader/__tests__/__fixtures__/test-doc.snapshot.bin b/packages/common/reader/__tests__/__fixtures__/test-doc.snapshot.bin new file mode 100644 index 0000000000000000000000000000000000000000..7c59832629a7757109a3b15ae19d626fe0862230 GIT binary patch literal 36447 zcmb__36NaXm8G(fsZ||&Rc^) zztb1;`uu*6%jNQfJZ+9Z(C2jfLVlk|3VPg<&!c?n4*BWZUoI0hOCGn|84O7wp9hVQ zd_Es)b_WCaEhPCQugm9gdDWV|UKf4+{Ay8iC>V5lQ95c41^sT(?2yy%bGiK?ugB*` zn_OPC)dAM(?=;(r(yb0D;B)#VuiG2&1cO0OND828)a-S;UHB^?h1@>BPx;*Mkv{#x z!ZK#_{=(K6Nu;Awxy8e&k%|21M4+o+`@}pwy)jqH-5ihEiY%QaFA>S(iSTr~khK+; zgvZC@Df#A1ctWnsG%aUy@pQ^|u;DML^rhnS=wv(*%gU*FEMajbo6g*vkLMF|!+bse zf{kO@cwTO^cV@%IHv5i5IAsqf683Q3zC9I{oz$7ZUEYo!_krBzDb|@AsWX&L?Yz;v zGpTf*bw)q^;<`)$0j5AtqPI|i(+FXU_o7g7^ki`Ar|_yO5=a6U^|q9rJ06&K{v zi8$-DYjJKe97`96OBu9nosN!>q(e<|jK>p6Dsa=nXgZMyXL7P_W}!~E`CvT}g~-*N z-mx>&bxhS1#2pGw=LLVvF@I*G8nXJrd#;|}D!vZS9@=AZ_=jE9^$)cQw zAK6IyyN$iba~LPVDJ%dkN*)^Kv+8FUF_hZT6{Dx|k58uqVSQsEDMT z%A?$vJrWM<2yiHZTzpJdo4>zFq)Di0aL@?~=3_14g8e+Y_hxMK^P)+yL-+L{qS?}+$?EO84 z-ZxB2#AQ9nNap(+ZrYtL*we+7O1W4#A8rHZ@{{&d+8#|O!HZ-(C+B!4tl_#_z(qYL zxV8opz1yUo!M#2P*I5P^WuThQ)(@AK)eNq)mT;Y|fXhvA=}AVo&Ncjbe>xgYKnG@X zdHYTFK^gy(k0)hd1uY2Osth;Y2WzyR7HCn&39Xrq!dPs;HQIlGp>>I&MH#53OZB6r zWi>wqZ=RC(=R#LDR)!a@a0ZO*$vX?2(dvOKJ)z4uNpd zNjo&B$TS{KrVCJI*?c^jfKH913wgm#!BU|m72T*3H74gWuxgagqKR}N=CpSgvM3~* zOlP5aNl_~WD?)?S1wF;*N}Knj;KP)&yx)U9x}tY)k@w7^Q? z9$HXC;p&M-RvcdN2%kbh4;oV8OePVJVh{nwLI(fn80;L*sRFQqJ6FhL(%HPz4po-5 z3uY)&bD|h7C=rQGs-|RJKb2f%(Sd#GCSR9C9~6wHDsnI3YqaX zS@-@ic#n&zBh>pLDLEAl9vDfFFvgx?y{8OR)0v9impvn+Goj@&>-d?9j{m7{m7dW$ zuJ{wY#%jl4wQl+f$Dkx966qrUqQW^C;c2zK1GHQ*wyNEOGmW$R2WJ{B9fWg@RUX3k z8m(P~?=@Qa2;Xb0?IhsK`n&|)jPipZBx@o_=T6sPDdHH)j4^_oX9S@P1n~I^g4A#b zw5(<H2cUq)?_*IPBc}GystD_kG!umS&h7}G}Rt?_;QXT?>WI3 z530M>k>wzJrT~rONPOjMNmSx(y;vE@P+g6W1{vODtzr5gs)tb;lU$Y4#s9C zlXKcpTnY9k%mW#o5Ikw_G2 z3a~(O!U2yKV2mUYk#X8#6c@vA;qj|dRxX{C^OM55RV-2QB`gy32O&DNz&anDW)r)TSAPW3?#AVmekD%Evp$Lo?BwYi078nV1%A% zWW>KL8QG!)*Yweski%I(3LimpG{tDcEDq{TC4X5oxg3E1it`r3qj)gOD&d3)RqbILuNCem8)A1NaM-E{C5$}YCn*@i& zm{lEr;0*Px#ifvxmB$NE)YEb*hCwH+E_m}I#u`(;G-y>t0jDvt(Q~0jQ4nJ480~Qk zM~sU^j*X13F1422BTFGU5&}uipXiJiI`{AC>1UF=xQrx68Ax&$ml-8zZg*+9%!GDv znNetDcU@ej3a!HKy11;`?pmeWUBbhe$YK)YO{_4p26$R+GYbv3FtaW$o88R1xXjYb zdT&{knf04x)@IgkmRXruzgbq>%)*y*G_y{M$&_SG1li;6jYs+pv}YZRAXgYcC<7sU zrGg+eCQ~h|8B4BIu!NV-hjzmx3w&PD2vlKY{chQf?e=`O0G$_3O^0*J=+!1aSU}-a zN&Lc=g{~~C8chpzH|F24=)0IYPH2r3r^jNLkPi$nv`#Fi;Y1mzHz$@Ghf@u-w5(=m zomg%hQ@nhRR)tY|V!4(m6k(&@!;{PBC*rB8RX0rL^O>7BZGtL9M9Y~=-*9JnY2o4J zD+dH_B2WlTCM-|;;d0`))bHn~3WoG&j2^N2*myou0E z-^jknt@|X#yn9v<^C$z=bkB;qnWtqnW8OV0teAJtiWYKq2%fv+tSu9E*i{hjS|E<(! zmz@(%X8k>|!W#VJ0(`=ez@O>PW@GUk9>*x7-qDprJ<34fA6;prUfG67#B=$oTxPf* zU1`O-qboV<7Dk|gh5ll~pM7?^Mf0*%@K`D}ChOyAtpdk_7*7CNxvp z=G{J(VDP`m;HL~!%bRt|4mEy-<%D8J%bVce_OeBI4k1Rs>i6}-lxPGdAlu9 zeX|y-Z>}^)^$me40Ys?w?e$KM9O&EAy_2DO`zj(eWuVsGzN#KlYq`u&y?vDxsc&C3 z3#s)CMpTci3bn_xd1d~($u9OyaClQ`#3vE$G0bvUf67hD*p0zEW(oW4t7?$>_EqMv zA6f<22|wOnSHA6{)m^~0-Yp}L;Ii22{H4h+gkvCIY)-Vw&$C9GhJkfYcQqNAi&`!-(4VaUogD9HHP9;@?`Kz6JKl3!@OV~3M%#yYO?X#=cc`{2j;<(U zpn9$G+10bV;?J(Obj5$Xy2=%QX|=U0{?ckISNx^bwOw(1IY(FgvT!a*`b3%C`%^u> z?f&*IKcmcRj53sg5Pq$KGBun_Evp%CUaQ~@FP{(A3g_~*@<=ug)QhWc>W0Ut1TAAS z777w@*RY!bqf&&&M64^V*toh2ceq(`N4<(Wnu9+og4IrSBc&4zQT~5Z6j4yjXKGAy zQ#KurCBqqVSdTO}4dSvYLtKXtR|rJ=)C0V~CXKNk-uuZ(bz!1KWGk21C5L3KFtR zu@QYSb&e3(ISOB1&W%U*F+?6>h)@PXLo$r-dF69w+DBp^9-7&7&MfDYI>>ynzGMiRR3BoGgO|cK!w!* zQyMB2>i?-~{Y-jmvTAAPXU`%g8` zuJ@m6w$%I2HdpEWA2eI*{U0=2>HQxx*Vg;^a*lfcCql81*oiA6dwbHo-JPL!&XpG# zS11D^{9*-HuGMy(i-ibb{#51E`Em_8$}d*%g{P>}osP@J+`J}TJHFJ+`22EngE&62 zrY9fljT~@IZ!UQ}V?zmFaJ1LGn;P(P^ZXRnFwy&A#Z4JSnTFJ)a?oIb;>1uBUT$u} ziew>~;`RT8*Z-PY|Kez3TQCCY)84yD0>B;PeX8k%vWH}CKy zy#vM3Qg^Skhbp|-yf}`+%h{D?9Vy3FU*>&$OT*afk+)2Fa#MXf$GQ{YWN9L}WeZym zd8>@^64j%&8$m|03UZ^mxrci5mKn;w;w^YbYr!mCdZ&3ltT@?L();laFXcV0lw3zB zw55HsduH#>0oQ@>j;?)fzH#lnaw)up)sp^M12-eTi6O^N}eyOsV6FY_ya7_%|s-n>cbEMG+CN$zTT&r|%2lg||*sLsb z7Qd*e@5*yn|F3yjzgTh>OaI;2TTjK6zlb7+le5v(a@U$QTiZtmGnwRWX=iem&$}ZU zNlvsUZS&?EG7a_w`%_)fcyF?6+tzGXQVQ)33|D6IdS!X{Ol146QmJEeHZkS*rejyH_G`S_)8%Rnyrm+MuGW6daCn01 z#?MOK^vgHaG^I1D@@&%QkH>jc-_$_so#@T=Z6DdPjcLelu30c0P82X}EE*IpgV>-> zjX)|!VjAZXubU2MKcQPJVx>5Oul`c++c$ZIXS51k`zAvZ+lt=Z%vC+3SGc$nvEPIk zGb}=eWUa=vRW;&J1=g3S&NIBur?ooUXNF{l&oi@c7j@%lz0O8;_kkhFvWM80Gxe90 z^|V@pxk6f~L;ughRwYw{1a;S%#raHW`<}>-sI-F}==<&(^CB7T&#jqNBT_x*wBF96 zHhq5$+C<1uUbJa+U?!C4EqaCrS(|<^dz*eZmp1*dx=rUrn}{KXHtqEHr$$SC>7F6h zrWa;!(~EOy(~H$@`WMkA8YYG|bxn2mW;Q#LTjQ)vKc2l!FU_S*FIBhcl^SjGga;Ct zN!gvqur|Fqdz*eTmp1*Rx=pXwXwyJ^Pktmm;TZ2{ZF*z&HoZBQHoaNhri&GA(obcx zK}Uo67i%b>KpA*?-2sJPtWmP+hCl1}FVb7FvlqN?s8g{#4Xeh7M)m>2;%X)f%0vmgroqg^r$RMCY%syG2C9 zV)~>acGhFf7TX=LiBmWi_K5;4XNr+cq(y*C+mF_nj`IFhL8*sn{vdcppb(NwX;De% zt|OjN2A*CwlIO0o)>wY`I$%k#5SC2Y5>|I# zPgqh0o?bVWcVBOb<=xj?VR`rUHL%nZjac4yeI2W@cVBM}(HE`-@N{hv9$VF4Gh07FeEq#LpE0tMpqbyl!2$$jo}q* z46j&Xc%>GGdZH1--(CM7UMIa$1Hj7yK*E9ml(W0L+#P{v)W@vV(`#v*QU(I>^xC=s z)Uuk*t*6&ojnmU>Ym8Gp(FowT*8ayAJx{N-9>$NY1%d$?f%egCQ@j z^>GKiZ2?@qg1>!Uapg}SfO~dum5!|qEwN_E_t7~gV#qni*QE>w04{O;4 zExIxDf-nn-VMNPqrQTiHfnAW=*X6+@P~N!WxOecn(yAL*E8O78^2hB6Q~C+f#W%W8(r z2}^8F7_g~uDD*@lIw#k83?_4>#cXac>#M%6u2WW%{k2zpf+lntoT& z`{mm5SFVs9$DKx18TGy|pxDXpD*Db-G*y~VE;7oo;dFJK7R+tEQ9WLOJ>l}r@sx6t zQQ~IxHlrJ&tOsC2UV099qAye32S!$sOw# zC!%{YlUugRevfU2x{L(hn8Pu$vwxc;*=Be-3o66?f3v=+N34a}CHnr*dNr0!jpS(y z^b5zl{O9>O8Jl1ae(~>r|F=sGzpL|gKQ6L!c|Ewv&L0W}ytphcE} zK9>|kt9??)=@Pg0!Hfum+v*|p}1=w!8n){G{PlwtLrWNm`M0?h(*G(>hgon;Y5*@=Z*p#7%Oy-SWY> z@`txbWg^L#rOyuyngwZxDp6k(<@gF9$a5fM|r~+j^jADdK2Fvrp+H*zgQ%6hm-L{$yVIJ^qr<= zZ{Cg#-SR|udy3IW`)o{(PfX^qs1DPcSGGv0OeJ|E&Z;NFjN=ckUnDx(p1`qITs@eO z$Jq#ZaQzaT#~BFEsQhFX8a}9LED{*6vCNZ(@K*k_Mw2fb|Jg2s)Nw$ebIRAbePBCN zC}+!3Ur?b?kf4fv;8u^%;|#*F5Z;C-;PLpy?T@%H)Cqm(#;>??RPwY*L9f^85f?(b z{NnCUm%9yDmxi2PKYW&eR|-Jug%~5wGDcAI&#tebO3tpIMU{v$t5peptyU$dh^dn5 zp`)r2vF64bXQ@ceemq5@e`cXb(8`*MB#956{#0B0vBGeZCvqq@T{OZ^L*l1u> zPch=%voBT}*vc4nL1$FOY=6OYw*S%k#nc$C>ybdTuSlR9E-Mn~!i|aqIwHyqhYRbg z42M_ChJ%M7xS*j%{pD$d+BW{PhT4gbF1^hbC@`__33Ph)XNL;ePKMfBWz;IPFMd~t z;inK72;geffE(^VCUD%CE9P~NM}8G>GqP{e**-pc3l zx7f%dd5RVob@8v(uNagO%Ey-TcJ)p;wr>2^9GUmldZvxPZRq*O(#CJ8O59u(J0}uSLHzkkca^kAzU?3Rq2M`4cw0SD0;5%B% zCLEuFMTJJOA8xU+L!EcD%x%`aqlL+t&$@TCd@MO@pP9?~j+UBoMzXb>?`W}_b?<0V ztuqo3$r*5voY@2q7*#oc{U1L1XSU4-a_*Yizq4N&81ZqR=%|I9@q5|YGPzw_3c|9N zLVjEhj@y{Qfnb{t;;$OvM@>a7>NJY_{uX7=v_tFqsWhPk@KN_^y7(K`^;SA zM{A25$<`u2Y7{whEstsYBHuHPX zMBoBH4NhM+XDb-KUp1aZT30HTNh2hobz=TDoE9cIooZQ$!>-wiObgnxS-uhTRLcUz zqT=UTPifL8vGG#N1HK_V#(&m?|MY9$`S~J)@b?dPOgW~rJ-dobub(LkztYY|f;!-P zZ$(_d>BS@;z?crftcK%`dEFU+VI7cMaIbLDy0;Bi>pKH3Ul8vExZIK-UOV0n2udDY z#18-7CovQI3{wNt^fN6rOzbl)vzXYT%xV)GzgC;rsEC=^8>@J#n%GI)s*P))c>9dB zXP2m-DUXP`x$F987VJkuYg*Z7=BN-ws2{_$e3DF?LKK%i_7SR^+`J9LXczjr+bOrI! zN1M;y26p>y6nImMF7YVd2+mg@uZk!mE9LEc~lVeu=-RVB#jA6@w`2FREOW^C{7J=SM9?x1{JOvepiRce-dl?At&C}kq`k3 zf%14sCn(4t1o82_jVFM0{7@T0W^B^_kc|Oqgs9ADyWi2;q^;sn&pw>vMD-!J+_|&% z;m5Pwwa?5g_YZ4Z?nt(_+&{Fm+&|O=O(HVDp$nQB7{F2$^ka9P`bHCB3}0oN&o8<0d{Zc7BI;wUW*)hy&M4~RtxMQSE%onM z>)d?vW33jx`LR}mNl|Y^T>H#i+{apL`sPTs7Wc7MgKy3h#<5n_Hzz?E;Lyb#;XiBQ zKK1kazitf0CBma!{RnnDi`HVMriSRwPjKjR|YK zfr&tF^YpgS;UQOkYj`NN+mmv0{r6~VmHvCYRnvby>hz-;PSiu5M*5HK!9~9{8{wg^ zovZL?J%jnZo84R5`Psl{$~c)>k4UVt9-nD78urY3e5Q47{;Xb<#SHmOYwa0Q)Mv0B zX%=F(-$7f2(TC5_qw9U#ge5SP?%kY`jdK}qjTB{w|XEdqjb$Rh-kSL3Mt7nri-zt(VO zFSO3$%!)FrZA1K8Z5v*II-{j$yM;5G7YC-YSaszsG_szVhZo8N!qPl6J)y3DW5 zwWfJ^VUE+r1&vk22i`kVR)GvEtL}dMu~%4lM6ky=xBa};iPuAvo0ydw$Hav$vMp9N``53=WmCWs3`Z3cQi`Ln%w2%KUtBSIEA`asrYo zr!tx}Fpm0rCqujTm3l(ldc1FgW>lDIPb5}p&-*qQdBn8meH-ScJ@uk22z1|u+5{5y z8MG&@;4$ra+mrWxx9`91`?K3BlqkL~D^b<&?&`?qBhzF4nHl!=LF}8jSn0B(;Z~~k zz76%L*84WpqgwCVP_0_;+n^dABupX?^6Q2N8~Ko;Itb2OI)1;=&dhD=?C#jywfn#z zGdxb#q68AFC~?x75+~i$8f3hy& zpEMEvNlo}9Hr^}}{y6_x6aLv#C;tz-ngaTu+tITt9|;yBd_m~(vhXXsN&K!(tN$50 z&4L~$-c|KrS=a5t)2ZUASC0pqGw=edgq`A88p4Wj8`g!Kc*`~5#`doO77KlCAwH2l zzQN4v5cL~1^FM7^)u~;&VK+Id{P>2sIUkR2V9v)~jP{RjFsSkCP1JZlP9hVu+6U&k z`tc1lbv2SJbu};8%I$buBLInxe?|G3yNA?zO8KAsKk$YduN46l?4 zQ0aEOvcb&lcy$Bk5%Z9cK$o#_&*7B~Ri4A^8>&5rSG2BDBdJd)LRWPA_|IBb|KX*_ zUNCY!+2;s#wfiQ#QP$PB%Uv~_`;b_r=-#$gbZ^g5(dk85D7v?2QFNj{gQ7F{9OhL_ zdicz!;Ydb=Bid%A;oh!G!@XUXhI`vY!@aFZo#e%vMN*IRpEaqU_`K4tH#*maBDGI&HATW~`GI2riE9EuN^9^}>5a45tl)&2s0 zt@al_fF2@$p?Y7by7;BspUEpX+IVc$=Y#SPn47Ple`Z1V4>ru=BB38Ooe79ZyL(9K z5$}s?FcBAcT0L`zfl@uAJ7W8@`~AcG62T)IiJ6R2M>cBA{Gb2&@Xy%B_#+$5H0ftI zE~Yl}a10W*CMHI=#>Pe@LktGhR_-_d{6eX^P@fDR{qbq~G&6DS5{-dD#uJS6VJ3-Q>j=68DBe(Vm#qw#Kp#M$sXzLCrnUTuxx zDL$zlp7^zTcpl&QF^1>yjh4exPnz5C)IYNrp2s)NGCa|bn!^*~GasJEH4YLNILP%3 zPyGbUUyBnkbeM%sz>s6dCjV0#=}a4Cpvq4H>PXwEjp{+hoVqt~A-=|Qw|uYmsf`Pj(~5;`!d6@@-jNU&cd|P>c?+~l zl^ zmir7|9>eAcaZF5k+(EYQ!&7G313h@dLb=A>$tzXQ>&0P40J%-$;c4-oj*#D(Nio~B z3<6$ym=Mp+n{+ZgVg8At_-w^L(bk?E_IA8nu_a~4R(X40`LXovL%dCflcNn`b|UB0 zMzy`FCh5D?;df!V7@OPRyZGH$9~N61aR%1u4Z2NM4vh)o4mD5;9oHt&*SN;STUxKYSn_K2feO$2B8zN`3pJgj>Dv0HL^R)R7NQ;r=)A&iYhbywE0|1#Q#rGRO0P zF+6&b5_j*a4_9cTPrb_ouT+5OqP2LTBBs2G0*J(GWy-@ZN}II@Snv*vIKLvg_os_F zh-sq-ya-uwx_j=qjIC)_h&1K1Jqxp!fTy!b# zaZaSu6A2kFcV&fZ0cD~Hxi{n(OegjZkL>r3BnN}hSjo{pIn$jSizN^E`nM$a_6_e& zkM3}H2Bj^IVrW=$=kr^8#|OInQ)45kE~uq#qiJt67;FDz4qHGRQOA@c>Y8!6)RRsQ z+|Qq74q~~5n)0Z+6s(-CJD2Wi*i;?yM?G)HE2*s{Fd~ggLZAH3Y=n-W;Z8b&onfSU z&NCfB8F;#J@>{AM|5QXqo{+oE zhnWPwz^;_A+5~6O(+0;8M=;$!Dd53(I3(d^=H~fXrit^F4B#D82X+oB#CGub=a%c_ zcutUmV;OTXd<8^hOF(b$8mgGQ+B{^QNCP(ksbjOzbP$1ZOU`J1+ z7%hZ*qT^GU(aw?S{!nnsbgyq{OJC5@9_? zRag*Av%GE~NB^yEtTN&z3^*OwVCDrs#3ovth{Z`;K@aS?afbqJZkJ1Jqz5ywLqiJs zy*_NC6~BlbX?`4sb-8$?;PqL!0mO~kfw!G=<4a#`)VcAR;0B4AxRIF52io^<-k<7b z#>Kmg80YC z5DZce-?aGdEbIX0oE^7)>d!CkJ^#o@@AFP@l05z9LY*YPRul*kqI{KdqWC~@G)pG{MmWu^~SwV*_pUY3sMqGQd3-j1Y@;RWo+W4e-juuVe$;> zj4ZjSC5fyBiRr1u3>pv~OI~7bDr32DNNQrPLU?{rc5y*sa;kz$YHmI|+-L?yBx{&5 zQy9y+O!I;*oq`LD11c2Z%9vP7GD~t&kri_#7o`H7gXE+&8yM6PI#@~)(~B828QH+r z79$yp#4KZ20rLb4W4T;-QD#Z1j)F^3V!4h&Ku%(wLSjyiLSl)6e_nE`9wSUCBMTz{ D-uQP( literal 0 HcmV?d00001 diff --git a/packages/common/reader/__tests__/__snapshots__/reader.spec.ts.snap b/packages/common/reader/__tests__/__snapshots__/reader.spec.ts.snap new file mode 100644 index 0000000000..fcea084a15 --- /dev/null +++ b/packages/common/reader/__tests__/__snapshots__/reader.spec.ts.snap @@ -0,0 +1,1758 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`should get all docs from root doc work 1`] = ` +[ + [ + "test-doc-1", + { + "title": "Test Doc 1", + }, + ], + [ + "test-doc-2", + { + "title": "Test Doc 2", + }, + ], + [ + "test-doc-4", + { + "title": undefined, + }, + ], +] +`; + +exports[`should get all docs from root doc work 2`] = ` +[ + [ + "test-doc-1", + { + "title": "Test Doc 1", + }, + ], + [ + "test-doc-2", + { + "title": "Test Doc 2", + }, + ], + [ + "test-doc-3", + { + "title": "Test Doc 3", + }, + ], + [ + "test-doc-4", + { + "title": undefined, + }, + ], +] +`; + +exports[`should read all doc ids from root doc snapshot work 1`] = ` +[ + "5nS9BSp3Px", +] +`; + +exports[`should read all docs from root doc snapshot work 1`] = ` +[ + [ + "5nS9BSp3Px", + { + "title": "Write, Draw, Plan all at Once.", + }, + ], +] +`; + +exports[`should read doc blocks work 1`] = ` +{ + "blocks": [ + { + "additional": { + "displayMode": "edgeless", + "noteBlockId": undefined, + }, + "blockId": "TnUgtVg7Eu", + "content": "Write, Draw, Plan all at Once.", + "docId": "test-doc", + "flavour": "affine:page", + "yblock": { + "prop:title": "Write, Draw, Plan all at Once.", + "sys:children": [ + "RX4CG2zsBk", + "S1mkc8zUoU", + "yGlBdshAqN", + "6lDiuDqZGL", + "cauvaHOQmh", + "2jwCeO8Yot", + "c9MF_JiRgx", + "6x7ALjUDjj", + ], + "sys:flavour": "affine:page", + "sys:id": "TnUgtVg7Eu", + "sys:version": 2, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "RX4CG2zsBk", + }, + "blockId": "FoPQcAyV_m", + "content": "AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. ", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "RX4CG2zsBk", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. ", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "FoPQcAyV_m", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "RX4CG2zsBk", + }, + "blockId": "oz48nn_zp8", + "content": "", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "RX4CG2zsBk", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "oz48nn_zp8", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "RX4CG2zsBk", + }, + "blockId": "g8a-D9-jXS", + "content": "You own your data, with no compromises", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "RX4CG2zsBk", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "You own your data, with no compromises", + "prop:type": "h1", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "g8a-D9-jXS", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "RX4CG2zsBk", + }, + "blockId": "J8lHN1GR_5", + "content": "Local-first & Real-time collaborative", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "RX4CG2zsBk", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Local-first & Real-time collaborative", + "prop:type": "h2", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "J8lHN1GR_5", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "RX4CG2zsBk", + }, + "blockId": "xCuWdM0VLz", + "content": "We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "RX4CG2zsBk", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "xCuWdM0VLz", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "RX4CG2zsBk", + }, + "blockId": "zElMi0tViK", + "content": "AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "RX4CG2zsBk", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "zElMi0tViK", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "RX4CG2zsBk", + }, + "blockId": "Z4rK0OF9Wk", + "content": "", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "RX4CG2zsBk", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "Z4rK0OF9Wk", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "S1mkc8zUoU", + }, + "blockId": "DQ0Ryb-SpW", + "content": "Blocks that assemble your next docs, tasks kanban or whiteboard", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "S1mkc8zUoU", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Blocks that assemble your next docs, tasks kanban or whiteboard", + "prop:type": "h3", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "DQ0Ryb-SpW", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "yGlBdshAqN", + }, + "blockId": "HAZC3URZp_", + "content": "There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "yGlBdshAqN", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "HAZC3URZp_", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "yGlBdshAqN", + }, + "blockId": "0H87ypiuv8", + "content": "We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "yGlBdshAqN", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "0H87ypiuv8", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "yGlBdshAqN", + }, + "blockId": "Sp4G1KD0Wn", + "content": "If you want to learn more about the product design of AFFiNE, here goes the concepts:", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "yGlBdshAqN", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "If you want to learn more about the product design of AFFiNE, here goes the concepts:", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "Sp4G1KD0Wn", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "yGlBdshAqN", + }, + "blockId": "RsUhDuEqXa", + "content": "To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "yGlBdshAqN", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "RsUhDuEqXa", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "6lDiuDqZGL", + }, + "blockId": "Z2HibKzAr-", + "content": "A true canvas for blocks in any form", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "6lDiuDqZGL", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "A true canvas for blocks in any form", + "prop:type": "h2", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "Z2HibKzAr-", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "6lDiuDqZGL", + }, + "blockId": "UwvWddamzM", + "content": "Many editor apps claimed to be a canvas for productivity. Since the Mother of All Demos, Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers. ", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "6lDiuDqZGL", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Many editor apps claimed to be a canvas for productivity. Since the Mother of All Demos, Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers. ", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "UwvWddamzM", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "6lDiuDqZGL", + }, + "blockId": "g9xKUjhJj1", + "content": "", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "6lDiuDqZGL", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "g9xKUjhJj1", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "6lDiuDqZGL", + }, + "blockId": "wDTn4YJ4pm", + "content": ""We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "6lDiuDqZGL", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": ""We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "wDTn4YJ4pm", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "6lDiuDqZGL", + }, + "blockId": "xFrrdiP3-V", + "content": "Quip & Notion with their great concept of "everything is a block"", + "docId": "test-doc", + "flavour": "affine:list", + "parentBlockId": "6lDiuDqZGL", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:checked": false, + "prop:collapsed": false, + "prop:order": null, + "prop:text": "Quip & Notion with their great concept of "everything is a block"", + "prop:type": "bulleted", + "sys:children": [], + "sys:flavour": "affine:list", + "sys:id": "xFrrdiP3-V", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "6lDiuDqZGL", + }, + "blockId": "Tp9xyN4Okl", + "content": "Trello with their Kanban", + "docId": "test-doc", + "flavour": "affine:list", + "parentBlockId": "6lDiuDqZGL", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:checked": false, + "prop:collapsed": false, + "prop:order": null, + "prop:text": "Trello with their Kanban", + "prop:type": "bulleted", + "sys:children": [], + "sys:flavour": "affine:list", + "sys:id": "Tp9xyN4Okl", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "6lDiuDqZGL", + }, + "blockId": "K_4hUzKZFQ", + "content": "Airtable & Miro with their no-code programable datasheets", + "docId": "test-doc", + "flavour": "affine:list", + "parentBlockId": "6lDiuDqZGL", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:checked": false, + "prop:collapsed": false, + "prop:order": null, + "prop:text": "Airtable & Miro with their no-code programable datasheets", + "prop:type": "bulleted", + "sys:children": [], + "sys:flavour": "affine:list", + "sys:id": "K_4hUzKZFQ", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "6lDiuDqZGL", + }, + "blockId": "QwMzON2s7x", + "content": "Miro & Whimiscal with their edgeless visual whiteboard", + "docId": "test-doc", + "flavour": "affine:list", + "parentBlockId": "6lDiuDqZGL", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:checked": false, + "prop:collapsed": false, + "prop:order": null, + "prop:text": "Miro & Whimiscal with their edgeless visual whiteboard", + "prop:type": "bulleted", + "sys:children": [], + "sys:flavour": "affine:list", + "sys:id": "QwMzON2s7x", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "6lDiuDqZGL", + }, + "blockId": "FFVmit6u1T", + "content": "Remnote & Capacities with their object-based tag system", + "docId": "test-doc", + "flavour": "affine:list", + "parentBlockId": "6lDiuDqZGL", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:checked": false, + "prop:collapsed": false, + "prop:order": null, + "prop:text": "Remnote & Capacities with their object-based tag system", + "prop:type": "bulleted", + "sys:children": [], + "sys:flavour": "affine:list", + "sys:id": "FFVmit6u1T", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "cauvaHOQmh", + }, + "blockId": "YqnG5O6AE6", + "content": "For more details, please refer to our RoadMap", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "cauvaHOQmh", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "For more details, please refer to our RoadMap", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "YqnG5O6AE6", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "cauvaHOQmh", + }, + "blockId": "sbDTmZMZcq", + "content": "Self Host", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "cauvaHOQmh", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Self Host", + "prop:type": "h2", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "sbDTmZMZcq", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "cauvaHOQmh", + }, + "blockId": "QVvitesfbj", + "content": "Self host AFFiNE", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "cauvaHOQmh", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Self host AFFiNE", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "QVvitesfbj", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": "Learning From", + "displayMode": "page", + "noteBlockId": "2jwCeO8Yot", + }, + "blockId": "U_GoHFD9At", + "content": [ + "Learning From", + "Title", + "Tag", + "Reference", + "Developers", + "AFFiNE", + ], + "docId": "test-doc", + "flavour": "affine:database", + "yblock": { + "prop:cells": { + "3aMlphe2lp": { + "sC99IAB2x_QM0zaPEj2ow": { + "columnId": "sC99IAB2x_QM0zaPEj2ow", + "value": [ + "HgHsKOUINZ", + ], + }, + }, + "EkFHpB-mJi": { + "sC99IAB2x_QM0zaPEj2ow": { + "columnId": "sC99IAB2x_QM0zaPEj2ow", + "value": [ + "HgHsKOUINZ", + ], + }, + }, + "MiZtUig-fL": { + "sC99IAB2x_QM0zaPEj2ow": { + "columnId": "sC99IAB2x_QM0zaPEj2ow", + "value": [ + "HgHsKOUINZ", + ], + }, + }, + "Q6LnVyKoGS": { + "sC99IAB2x_QM0zaPEj2ow": { + "columnId": "sC99IAB2x_QM0zaPEj2ow", + "value": [ + "HgHsKOUINZ", + ], + }, + }, + "VMx9lHw3TR": { + "sC99IAB2x_QM0zaPEj2ow": { + "columnId": "sC99IAB2x_QM0zaPEj2ow", + "value": [ + "0jh9gNw4Yl", + ], + }, + }, + "tpyOZbPc1P": { + "sC99IAB2x_QM0zaPEj2ow": { + "columnId": "sC99IAB2x_QM0zaPEj2ow", + "value": [ + "AxSe-53xjX", + ], + }, + }, + }, + "prop:columns": [ + { + "data": {}, + "id": "clNriIlPoh", + "type": "title", + }, + { + "data": {}, + "id": "tEvV9x-oBP3m4MwVyFH1Z", + "name": "Title", + "type": "title", + }, + { + "data": { + "options": [ + { + "color": "var(--affine-tag-blue)", + "id": "HgHsKOUINZ", + "value": "Reference", + }, + { + "color": "var(--affine-tag-orange)", + "id": "0jh9gNw4Yl", + "value": "Developers", + }, + { + "color": "var(--affine-tag-pink)", + "id": "AxSe-53xjX", + "value": "AFFiNE", + }, + ], + }, + "id": "sC99IAB2x_QM0zaPEj2ow", + "name": "Tag", + "type": "multi-select", + }, + ], + "prop:title": "Learning From", + "prop:views": [ + { + "columns": [ + { + "id": "clNriIlPoh", + }, + ], + "filter": { + "conditions": [], + "op": "and", + "type": "group", + }, + "header": { + "iconColumn": "type", + "titleColumn": "tEvV9x-oBP3m4MwVyFH1Z", + }, + "id": "Gt8Hbz0vBy33WSl58VH2Y", + "mode": "table", + "name": "Table View", + }, + { + "columns": [ + { + "hide": false, + "id": "tEvV9x-oBP3m4MwVyFH1Z", + }, + { + "hide": false, + "id": "sC99IAB2x_QM0zaPEj2ow", + }, + ], + "filter": { + "conditions": [], + "op": "and", + "type": "group", + }, + "groupBy": { + "columnId": "sC99IAB2x_QM0zaPEj2ow", + "name": "multi-select", + "type": "groupBy", + }, + "groupProperties": [], + "header": { + "iconColumn": "type", + "titleColumn": "tEvV9x-oBP3m4MwVyFH1Z", + }, + "id": "43eIk3skKQWFlamyg8IIn", + "mode": "kanban", + "name": "Kanban View", + }, + ], + "sys:children": [ + "tpyOZbPc1P", + "VMx9lHw3TR", + "Q6LnVyKoGS", + "EkFHpB-mJi", + "3aMlphe2lp", + "MiZtUig-fL", + "erYE2C7cc5", + ], + "sys:flavour": "affine:database", + "sys:id": "U_GoHFD9At", + "sys:version": 3, + }, + }, + { + "additional": { + "databaseName": "Learning From", + "displayMode": "page", + "noteBlockId": "2jwCeO8Yot", + }, + "blockId": "tpyOZbPc1P", + "content": "Affine Development", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "U_GoHFD9At", + "parentFlavour": "affine:database", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Affine Development", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "tpyOZbPc1P", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": "Learning From", + "displayMode": "page", + "noteBlockId": "2jwCeO8Yot", + }, + "blockId": "VMx9lHw3TR", + "content": "For developers or installations guides, please go to AFFiNE Doc", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "U_GoHFD9At", + "parentFlavour": "affine:database", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "For developers or installations guides, please go to AFFiNE Doc", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "VMx9lHw3TR", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": "Learning From", + "displayMode": "page", + "noteBlockId": "2jwCeO8Yot", + }, + "blockId": "Q6LnVyKoGS", + "content": "Quip & Notion with their great concept of "everything is a block"", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "U_GoHFD9At", + "parentFlavour": "affine:database", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Quip & Notion with their great concept of "everything is a block"", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "Q6LnVyKoGS", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": "Learning From", + "displayMode": "page", + "noteBlockId": "2jwCeO8Yot", + }, + "blockId": "EkFHpB-mJi", + "content": "Trello with their Kanban", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "U_GoHFD9At", + "parentFlavour": "affine:database", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Trello with their Kanban", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "EkFHpB-mJi", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": "Learning From", + "displayMode": "page", + "noteBlockId": "2jwCeO8Yot", + }, + "blockId": "3aMlphe2lp", + "content": "Airtable & Miro with their no-code programable datasheets", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "U_GoHFD9At", + "parentFlavour": "affine:database", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Airtable & Miro with their no-code programable datasheets", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "3aMlphe2lp", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": "Learning From", + "displayMode": "page", + "noteBlockId": "2jwCeO8Yot", + }, + "blockId": "MiZtUig-fL", + "content": "Miro & Whimiscal with their edgeless visual whiteboard", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "U_GoHFD9At", + "parentFlavour": "affine:database", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Miro & Whimiscal with their edgeless visual whiteboard", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "MiZtUig-fL", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": "Learning From", + "displayMode": "page", + "noteBlockId": "2jwCeO8Yot", + }, + "blockId": "erYE2C7cc5", + "content": "Remnote & Capacities with their object-based tag system", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "U_GoHFD9At", + "parentFlavour": "affine:database", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Remnote & Capacities with their object-based tag system", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "erYE2C7cc5", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "c9MF_JiRgx", + }, + "blockId": "NyHXrMX3R1", + "content": "Affine Development", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "c9MF_JiRgx", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "Affine Development", + "prop:type": "h2", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "NyHXrMX3R1", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "c9MF_JiRgx", + }, + "blockId": "9-K49otbCv", + "content": "For developer or installation guides, please go to AFFiNE Development", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "c9MF_JiRgx", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "For developer or installation guides, please go to AFFiNE Development", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "9-K49otbCv", + "sys:version": 1, + }, + }, + { + "additional": { + "databaseName": undefined, + "displayMode": "page", + "noteBlockId": "c9MF_JiRgx", + }, + "blockId": "faFteK9eG-", + "content": "", + "docId": "test-doc", + "flavour": "affine:paragraph", + "parentBlockId": "c9MF_JiRgx", + "parentFlavour": "affine:note", + "ref": [], + "refDocId": [], + "yblock": { + "prop:collapsed": false, + "prop:text": "", + "prop:type": "text", + "sys:children": [], + "sys:flavour": "affine:paragraph", + "sys:id": "faFteK9eG-", + "sys:version": 1, + }, + }, + { + "additional": { + "displayMode": "edgeless", + "noteBlockId": undefined, + }, + "blockId": "6x7ALjUDjj", + "content": [ + "What is AFFiNE", + "Related Articles", + " ", + "Self-host", + "", + "AFFiNE ", + "Development", + "You can check these URLs to learn about AFFiNE", + "Database Reference", + ], + "docId": "test-doc", + "flavour": "affine:surface", + "parentBlockId": "TnUgtVg7Eu", + "parentFlavour": "affine:page", + "yblock": { + "prop:elements": { + "type": "$blocksuite:internal:native$", + "value": { + "EkqQL1MU5m": { + "color": "--affine-palette-line-black", + "fontFamily": "blocksuite:surface:OrelegaOne", + "fontSize": 120, + "fontStyle": "normal", + "fontWeight": "400", + "hasMaxWidth": true, + "id": "EkqQL1MU5m", + "index": "aXd", + "rotate": 0, + "seed": 1588752895, + "text": "Self-host", + "textAlign": "left", + "type": "text", + "xywh": "[988.4153663986212,-181.71961053807448,563,168.75]", + }, + "F-GXtb8ubm": { + "color": "--affine-palette-line-black", + "fontFamily": "blocksuite:surface:Lora", + "fontSize": 72, + "fontStyle": "normal", + "fontWeight": "600", + "hasMaxWidth": true, + "id": "F-GXtb8ubm", + "index": "ag", + "rotate": 0, + "seed": 1394803222, + "text": "Database Reference", + "textAlign": "left", + "type": "text", + "xywh": "[-183.35234372014293,-333.83661977719123,744.8157214875225,92]", + }, + "GVPdqrq6T6": { + "children": { + "JlgVJdWU12": true, + "R2MK4ZzUb3": true, + }, + "id": "GVPdqrq6T6", + "index": "a0", + "seed": 771020267, + "title": "Group 6", + "type": "group", + "xywh": "[126.65897893082821,-2338.7483936405356,2375.5289713541665,722.5748697916665]", + }, + "Gwb4ZjdyMJ": { + "children": { + "6lDiuDqZGL": true, + "RX4CG2zsBk": true, + "istDk5DOMO": true, + "saGXC7nPOk": true, + "yGlBdshAqN": true, + }, + "id": "Gwb4ZjdyMJ", + "index": "aN", + "seed": 242073567, + "title": "Group 3", + "type": "group", + }, + "LHh9XjyG9P": { + "controllers": [], + "frontEndpointStyle": "None", + "id": "LHh9XjyG9P", + "index": "ae", + "mode": 2, + "rearEndpointStyle": "Arrow", + "rough": false, + "roughness": 1.4, + "seed": 962796410, + "source": { + "id": "istDk5DOMO", + "position": [ + -6.164088349722365e-17, + 0.5, + ], + }, + "stroke": "--affine-palette-line-grey", + "strokeStyle": "solid", + "strokeWidth": 8, + "target": { + "id": "ECrtbvW6xx", + "position": [ + 1, + 0.5000000000000001, + ], + }, + "type": "connector", + }, + "Nb_9OXyIT3": { + "fillColor": "--affine-palette-shape-yellow", + "filled": true, + "id": "Nb_9OXyIT3", + "index": "a0", + "radius": 0, + "rotate": 0, + "roughness": 1.4, + "seed": 97408177, + "shapeStyle": "General", + "shapeType": "rect", + "strokeColor": "--affine-palette-line-yellow", + "strokeStyle": "solid", + "strokeWidth": 4, + "type": "shape", + "xywh": "[2403.9004515323168,57.24116579067933,955.1393577008388,542.1082071559455]", + }, + "R2MK4ZzUb3": { + "color": "--affine-palette-line-black", + "fillColor": "--affine-palette-shape-white", + "filled": true, + "id": "R2MK4ZzUb3", + "index": "a0", + "radius": 0, + "rotate": 0, + "roughness": 1.4, + "seed": 1558899337, + "shapeStyle": "General", + "shapeType": "rect", + "strokeColor": "--affine-palette-transparent", + "strokeStyle": "solid", + "strokeWidth": 2, + "type": "shape", + "xywh": "[126.65897893082821,-2338.7483936405356,2375.5289713541665,722.5748697916666]", + }, + "TRRWjtvWJm": { + "children": { + "5W--UQLN11": true, + "lcZphIJe63": true, + }, + "id": "TRRWjtvWJm", + "index": "a4", + "seed": 1624205143, + "title": "Group 1", + "type": "group", + "xywh": "[-1043.6169874048903,-641.7901035486665,622.6248931928226,667.0980998494529]", + }, + "UloPoCxt6P": { + "color": "--affine-palette-line-black", + "fontFamily": "blocksuite:surface:OrelegaOne", + "fontSize": 128, + "fontStyle": "normal", + "fontWeight": "400", + "hasMaxWidth": false, + "id": "UloPoCxt6P", + "index": "aTl", + "rotate": 0, + "seed": 606071663, + "text": " ", + "textAlign": "left", + "type": "text", + "xywh": "[2522.7441912768363,-1310.8328787672467,35,128]", + }, + "XWYKw-kpYn": { + "children": { + "EkqQL1MU5m": true, + "cauvaHOQmh": true, + "qRCk-vrGXw": true, + }, + "id": "XWYKw-kpYn", + "index": "aZ", + "seed": 1740232717, + "title": "Group 4", + "type": "group", + }, + "YWOfr8Pprg": { + "children": { + "2jwCeO8Yot": true, + "F-GXtb8ubm": true, + }, + "id": "YWOfr8Pprg", + "index": "aj", + "seed": 484840749, + "title": "Group 5", + "type": "group", + }, + "Z7D3qrSurD": { + "color": "--affine-palette-line-black", + "fontFamily": "blocksuite:surface:OrelegaOne", + "fontSize": 120, + "fontStyle": "normal", + "fontWeight": "400", + "hasMaxWidth": false, + "id": "Z7D3qrSurD", + "index": "aTV", + "rotate": 0, + "seed": 2100869690, + "text": "Related Articles", + "textAlign": "left", + "type": "text", + "xywh": "[2478.972284676839,-1231.0379248118952,879.4798583984375,168.75]", + }, + "_nC65-wkSP": { + "color": "--affine-palette-line-black", + "fontFamily": "blocksuite:surface:OrelegaOne", + "fontSize": 128, + "fontStyle": "normal", + "fontWeight": "400", + "hasMaxWidth": true, + "id": "_nC65-wkSP", + "index": "aa", + "rotate": 0, + "seed": 994394416, + "text": "AFFiNE ", + "textAlign": "left", + "type": "text", + "xywh": "[2459.953125798208,88.108859492537,779,147]", + }, + "f3x6HbuyUQ": { + "controllers": [], + "frontEndpointStyle": "None", + "id": "f3x6HbuyUQ", + "index": "aY", + "mode": 2, + "rearEndpointStyle": "Arrow", + "rough": false, + "roughness": 1.4, + "seed": 668321843, + "source": { + "id": "uzfdAcEDxu", + "position": [ + 0.4999999999993948, + 1.0000000000000002, + ], + }, + "stroke": "--affine-palette-line-grey", + "strokeStyle": "solid", + "strokeWidth": 8, + "target": { + "id": "qRCk-vrGXw", + "position": [ + 1.0000000000000002, + 0.5, + ], + }, + "type": "connector", + }, + "gPvT0nfbcw": { + "color": "--affine-palette-line-black", + "fontFamily": "blocksuite:surface:OrelegaOne", + "fontSize": 128, + "fontStyle": "normal", + "fontWeight": "400", + "hasMaxWidth": true, + "id": "gPvT0nfbcw", + "index": "ab", + "rotate": 0, + "seed": 183687274, + "text": "Development", + "textAlign": "left", + "type": "text", + "xywh": "[2459.953125798208,208.4260321250699,779,147]", + }, + "hLAqby4WpD": { + "color": "--affine-palette-line-black", + "fillColor": "--affine-palette-shape-white", + "filled": true, + "id": "hLAqby4WpD", + "index": "Zz", + "radius": 0, + "rotate": 0, + "roughness": 1.4, + "seed": 2025695507, + "shapeStyle": "General", + "shapeType": "rect", + "strokeColor": "--affine-palette-transparent", + "strokeStyle": "solid", + "strokeWidth": 2, + "type": "shape", + "xywh": "[-259.87318248973133,-371.99627945163166,948.0569720935435,876.3267415022492]", + }, + "istDk5DOMO": { + "color": "--affine-palette-line-black", + "fillColor": "--affine-palette-shape-blue", + "filled": true, + "id": "istDk5DOMO", + "index": "aI", + "radius": 0, + "rotate": 0, + "roughness": 1.4, + "seed": 858601929, + "shapeStyle": "General", + "shapeType": "rect", + "strokeColor": "--affine-palette-transparent", + "strokeStyle": "solid", + "strokeWidth": 4, + "type": "shape", + "xywh": "[353.84627540373367,-1532.740227539313,1844.3414706529402,1139.4642574172829]", + }, + "laVEftUZ5b": { + "children": { + "47g7sBvNVTS0tJaSnY3n2": true, + "Nb_9OXyIT3": true, + "_nC65-wkSP": true, + "gPvT0nfbcw": true, + }, + "id": "laVEftUZ5b", + "index": "ac", + "seed": 1549109555, + "title": "Group 5", + "type": "group", + }, + "mK-9EA5g4c": { + "controllers": [], + "frontEndpointStyle": "None", + "id": "mK-9EA5g4c", + "index": "ai", + "mode": 2, + "rearEndpointStyle": "Arrow", + "rough": false, + "roughness": 1.4, + "seed": 1095574431, + "source": { + "id": "qRCk-vrGXw", + "position": [ + 0, + 0.5, + ], + }, + "stroke": "--affine-palette-line-grey", + "strokeStyle": "solid", + "strokeWidth": 8, + "target": { + "id": "2jwCeO8Yot", + }, + "type": "connector", + }, + "qRCk-vrGXw": { + "color": "--affine-palette-line-black", + "fillColor": "--affine-palette-shape-magenta", + "filled": true, + "fontFamily": "blocksuite:surface:Inter", + "id": "qRCk-vrGXw", + "index": "aXG", + "radius": 0, + "rotate": 0, + "roughness": 1.4, + "seed": 2015167989, + "shapeStyle": "General", + "shapeType": "rect", + "strokeColor": "--affine-palette-transparent", + "strokeStyle": "solid", + "strokeWidth": 4, + "text": "", + "type": "shape", + "xywh": "[928.4478756397534,-248.78447101666404,943.70584160216,514.081347775151]", + }, + "sNDFCBEYzR": { + "controllers": [], + "frontEndpointStyle": "None", + "id": "sNDFCBEYzR", + "index": "af", + "mode": 2, + "rearEndpointStyle": "Arrow", + "rough": false, + "roughness": 1.4, + "seed": 524593855, + "source": { + "id": "istDk5DOMO", + "position": [ + -6.164088349722365e-17, + 0.5, + ], + }, + "stroke": "--affine-palette-line-grey", + "strokeStyle": "solid", + "strokeWidth": 8, + "target": { + "id": "5W--UQLN11", + }, + "type": "connector", + }, + "saGXC7nPOk": { + "color": "--affine-palette-line-black", + "fontFamily": "blocksuite:surface:OrelegaOne", + "fontSize": 128, + "fontStyle": "normal", + "fontWeight": "400", + "hasMaxWidth": false, + "id": "saGXC7nPOk", + "index": "aJ", + "rotate": 0, + "seed": 93054478, + "text": "What is AFFiNE", + "textAlign": "left", + "type": "text", + "xywh": "[438.73809335123497,-1467.165331669401,913.75,128]", + }, + "t3Rt_B2IAr": { + "controllers": [], + "frontEndpointStyle": "None", + "id": "t3Rt_B2IAr", + "index": "ad", + "mode": 2, + "rearEndpointStyle": "Arrow", + "rough": false, + "roughness": 1.4, + "seed": 306364128, + "source": { + "id": "qRCk-vrGXw", + "position": [ + 1, + 0.5, + ], + }, + "stroke": "--affine-palette-line-grey", + "strokeStyle": "solid", + "strokeWidth": 8, + "target": { + "id": "Nb_9OXyIT3", + }, + "type": "connector", + }, + "tCpJR12_hu": { + "controllers": [], + "frontEndpointStyle": "None", + "id": "tCpJR12_hu", + "index": "aW", + "mode": 2, + "rearEndpointStyle": "Arrow", + "rough": false, + "roughness": 1.4, + "seed": 1310217918, + "source": { + "id": "istDk5DOMO", + "position": [ + 0.9999999999999999, + 0.5, + ], + }, + "stroke": "--affine-palette-line-grey", + "strokeStyle": "solid", + "strokeWidth": 8, + "target": { + "id": "uzfdAcEDxu", + }, + "type": "connector", + }, + "uzfdAcEDxu": { + "color": "--affine-palette-line-black", + "fillColor": "--affine-palette-shape-purple", + "filled": true, + "id": "uzfdAcEDxu", + "index": "aTG", + "radius": 0, + "rotate": 0.062196392871315495, + "roughness": 1.4, + "seed": 462401908, + "shapeStyle": "General", + "shapeType": "rect", + "strokeColor": "--affine-palette-transparent", + "strokeStyle": "solid", + "strokeWidth": 4, + "type": "shape", + "xywh": "[2421.751354148884,-1283.6175127158676,1085.7995880501803,519.0294552479265]", + }, + "w86OKmzMtn": { + "color": "--affine-palette-line-black", + "fillColor": "--affine-palette-shape-tangerine", + "filled": true, + "fontFamily": "blocksuite:surface:Poppins", + "fontSize": 36, + "fontStyle": "normal", + "fontWeight": "600", + "id": "w86OKmzMtn", + "index": "ad", + "radius": 0, + "rotate": 0, + "roughness": 1.4, + "seed": 1298533711, + "shapeStyle": "General", + "shapeType": "rect", + "strokeColor": "--affine-palette-transparent", + "strokeStyle": "solid", + "strokeWidth": 4, + "text": "You can check these URLs to learn about AFFiNE", + "textAlign": "left", + "type": "shape", + "xywh": "[-1055.7017064945487,-981.0008617000956,702.1470991695091,295.7649942194072]", + }, + }, + }, + "sys:children": [ + "ECrtbvW6xx", + "5W--UQLN11", + "lcZphIJe63", + "JlgVJdWU12", + "lht7AqBqnF", + ], + "sys:flavour": "affine:surface", + "sys:id": "6x7ALjUDjj", + "sys:version": 5, + }, + }, + { + "additional": { + "displayMode": "edgeless", + "noteBlockId": undefined, + }, + "blockId": "ECrtbvW6xx", + "docId": "test-doc", + "flavour": "affine:bookmark", + "yblock": { + "prop:caption": null, + "prop:description": "The universal editor that lets you work, play, present or create just about anything.", + "prop:icon": "/favicon-96.png", + "prop:image": "https://affine.pro/og.png", + "prop:index": "a1", + "prop:lockedBySelf": false, + "prop:rotate": 0, + "prop:style": "vertical", + "prop:title": "AFFiNE - All In One KnowledgeOS", + "prop:url": "https://affine.pro/", + "prop:xywh": "[-605.1053721621553,-1497.696057920688,606.4771127476911,649.7969065153833]", + "sys:children": [], + "sys:flavour": "affine:bookmark", + "sys:id": "ECrtbvW6xx", + "sys:version": 1, + }, + }, + { + "additional": { + "displayMode": "edgeless", + "noteBlockId": undefined, + }, + "blockId": "5W--UQLN11", + "docId": "test-doc", + "flavour": "affine:bookmark", + "yblock": { + "prop:caption": null, + "prop:description": "AFFiNE is the all-in-one workspace where you can write, draw and plan just about anything - Blend the power of Notion and Miro to enable dynamic note-taking, wikis, tasks, visualized mindmaps and presentations. All you need for productivity and creativity is here! +", + "prop:icon": "https://www.youtube.com/s/desktop/8b6c1f4c/img/favicon_32x32.png", + "prop:image": "https://yt3.googleusercontent.com/H9-Rol_TUq4UmR8cdy-LhxFmWdmz5LIm_KTYoVP2D81I-w9T12ttJHfME6kWUnEresNVo4c8dA=s900-c-k-c0x00ffffff-no-rj", + "prop:index": "a2", + "prop:lockedBySelf": false, + "prop:rotate": 0, + "prop:style": "vertical", + "prop:title": "AFFiNE", + "prop:url": "https://www.youtube.com/@affinepro", + "prop:xywh": "[-1043.6169874048903,-641.7901035486665,622.6248931928226,667.0980998494529]", + "sys:children": [], + "sys:flavour": "affine:bookmark", + "sys:id": "5W--UQLN11", + "sys:version": 1, + }, + }, + { + "additional": { + "displayMode": "edgeless", + "noteBlockId": undefined, + }, + "blob": [ + "BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA=", + ], + "blockId": "lcZphIJe63", + "docId": "test-doc", + "flavour": "affine:image", + "parentBlockId": "6x7ALjUDjj", + "parentFlavour": "affine:surface", + "yblock": { + "prop:caption": "", + "prop:height": 728, + "prop:index": "a3", + "prop:lockedBySelf": false, + "prop:rotate": 0, + "prop:size": 115416, + "prop:sourceId": "BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA=", + "prop:width": 1302, + "prop:xywh": "[-991.0766514037751,-616.4591370809392,505.45950210093383,282.62251730374794]", + "sys:children": [], + "sys:flavour": "affine:image", + "sys:id": "lcZphIJe63", + "sys:version": 1, + }, + }, + { + "additional": { + "displayMode": "edgeless", + "noteBlockId": undefined, + }, + "blob": [ + "HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8=", + ], + "blockId": "JlgVJdWU12", + "docId": "test-doc", + "flavour": "affine:image", + "parentBlockId": "6x7ALjUDjj", + "parentFlavour": "affine:surface", + "yblock": { + "prop:caption": "", + "prop:height": 374, + "prop:index": "aj", + "prop:lockedBySelf": false, + "prop:rotate": 0, + "prop:size": 50651, + "prop:sourceId": "HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8=", + "prop:width": 1463, + "prop:xywh": "[281.13294520744034,-2242.095913253297,2001.7110026018645,511.7155946501006]", + "sys:children": [], + "sys:flavour": "affine:image", + "sys:id": "JlgVJdWU12", + "sys:version": 1, + }, + }, + { + "additional": { + "displayMode": "edgeless", + "noteBlockId": undefined, + }, + "blob": [ + "ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0=", + ], + "blockId": "lht7AqBqnF", + "docId": "test-doc", + "flavour": "affine:image", + "parentBlockId": "6x7ALjUDjj", + "parentFlavour": "affine:surface", + "yblock": { + "prop:caption": "", + "prop:height": 1388, + "prop:index": "acV", + "prop:lockedBySelf": false, + "prop:rotate": 10.938230891828395, + "prop:size": 71614, + "prop:sourceId": "ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0=", + "prop:width": 862, + "prop:xywh": "[3446.6044934637607,-1339.9783466050073,122.45568062094182,197.1792165914935]", + "sys:children": [], + "sys:flavour": "affine:image", + "sys:id": "lht7AqBqnF", + "sys:version": 1, + }, + }, + ], + "summary": "AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. You own your data, with no compromisesLocal-first & Real-time collaborativeWe love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.Blocks that assemble your next docs, tasks kanban or whiteboardThere is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ", + "title": "Write, Draw, Plan all at Once.", +} +`; diff --git a/packages/common/reader/__tests__/reader.spec.ts b/packages/common/reader/__tests__/reader.spec.ts new file mode 100644 index 0000000000..bdfa56d7f5 --- /dev/null +++ b/packages/common/reader/__tests__/reader.spec.ts @@ -0,0 +1,90 @@ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; + +import { expect, test } from 'vitest'; +import { applyUpdate, Array as YArray, Doc as YDoc, Map as YMap } from 'yjs'; + +import { + readAllBlocksFromDoc, + readAllDocIdsFromRootDoc, + readAllDocsFromRootDoc, +} from '../src'; + +const rootDocSnapshot = readFileSync( + path.join(import.meta.dirname, './__fixtures__/test-root-doc.snapshot.bin') +); +const docSnapshot = readFileSync( + path.join(import.meta.dirname, './__fixtures__/test-doc.snapshot.bin') +); + +test('should read doc blocks work', async () => { + const rootDoc = new YDoc({ + guid: 'test-root-doc', + }); + applyUpdate(rootDoc, rootDocSnapshot); + + const doc1 = new YDoc({ + guid: 'test-doc', + }); + applyUpdate(doc1, docSnapshot); + const result = await readAllBlocksFromDoc({ + ydoc: doc1, + rootYDoc: rootDoc, + spaceId: 'test-space', + }); + expect(result).toMatchSnapshot(); +}); + +test('should get all docs from root doc work', async () => { + const rootDoc = new YDoc({ + guid: 'test-root-doc', + }); + rootDoc.getMap('meta').set( + 'pages', + YArray.from([ + new YMap([ + ['id', 'test-doc-1'], + ['title', 'Test Doc 1'], + ]), + new YMap([ + ['id', 'test-doc-2'], + ['title', 'Test Doc 2'], + ]), + new YMap([ + ['id', 'test-doc-3'], + ['title', 'Test Doc 3'], + ['trash', true], + ]), + new YMap([['id', 'test-doc-4']]), + ]) + ); + + const docs = readAllDocsFromRootDoc(rootDoc); + expect(Array.from(docs.entries())).toMatchSnapshot(); + + // include trash + const docsWithTrash = readAllDocsFromRootDoc(rootDoc, { + includeTrash: true, + }); + expect(Array.from(docsWithTrash.entries())).toMatchSnapshot(); +}); + +test('should read all docs from root doc snapshot work', async () => { + const rootDoc = new YDoc({ + guid: 'test-root-doc', + }); + applyUpdate(rootDoc, rootDocSnapshot); + const docsWithTrash = readAllDocsFromRootDoc(rootDoc, { + includeTrash: true, + }); + expect(Array.from(docsWithTrash.entries())).toMatchSnapshot(); +}); + +test('should read all doc ids from root doc snapshot work', async () => { + const rootDoc = new YDoc({ + guid: 'test-root-doc', + }); + applyUpdate(rootDoc, rootDocSnapshot); + const docIds = readAllDocIdsFromRootDoc(rootDoc); + expect(docIds).toMatchSnapshot(); +}); diff --git a/packages/common/reader/esbuild.config.js b/packages/common/reader/esbuild.config.js new file mode 100644 index 0000000000..416fc537f2 --- /dev/null +++ b/packages/common/reader/esbuild.config.js @@ -0,0 +1,23 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import { build } from 'esbuild'; + +const result = await build({ + entryPoints: ['./src/index.ts'], + bundle: true, + platform: 'node', + outdir: 'dist', + target: 'es2024', + sourcemap: true, + format: 'esm', + external: ['yjs'], + metafile: true, +}); + +if (process.env.METAFILE) { + await fs.writeFile( + path.resolve(`metafile-${Date.now()}.json`), + JSON.stringify(result.metafile, null, 2) + ); +} diff --git a/packages/common/reader/package.json b/packages/common/reader/package.json new file mode 100644 index 0000000000..14529f9aa2 --- /dev/null +++ b/packages/common/reader/package.json @@ -0,0 +1,27 @@ +{ + "name": "@affine/reader", + "type": "module", + "version": "0.21.0", + "private": true, + "sideEffects": false, + "module": "./src/index.ts", + "exports": { + ".": "./src/index.ts", + "./dist": "./dist/index.js" + }, + "scripts": { + "build": "yarn bundle", + "bundle": "node esbuild.config.js" + }, + "dependencies": { + "lodash-es": "^4.17.21", + "yjs": "^13.6.21" + }, + "devDependencies": { + "@blocksuite/affine": "workspace:*", + "vitest": "3.1.3" + }, + "peerDependencies": { + "@blocksuite/affine": "workspace:*" + } +} diff --git a/packages/common/nbstore/src/sync/indexer/bs-store.ts b/packages/common/reader/src/bs-store.ts similarity index 100% rename from packages/common/nbstore/src/sync/indexer/bs-store.ts rename to packages/common/reader/src/bs-store.ts diff --git a/packages/common/reader/src/index.ts b/packages/common/reader/src/index.ts new file mode 100644 index 0000000000..ed95948d94 --- /dev/null +++ b/packages/common/reader/src/index.ts @@ -0,0 +1 @@ +export * from './reader'; diff --git a/packages/common/reader/src/reader.ts b/packages/common/reader/src/reader.ts new file mode 100644 index 0000000000..888125289e --- /dev/null +++ b/packages/common/reader/src/reader.ts @@ -0,0 +1,905 @@ +import { Container } from '@blocksuite/affine/global/di'; +import type { + AttachmentBlockModel, + BookmarkBlockModel, + EmbedBlockModel, + ImageBlockModel, + TableBlockModel, +} from '@blocksuite/affine/model'; +import { AffineSchemas } from '@blocksuite/affine/schemas'; +import { MarkdownAdapter } from '@blocksuite/affine/shared/adapters'; +import type { AffineTextAttributes } from '@blocksuite/affine/shared/types'; +import { + createYProxy, + type DeltaInsert, + type DraftModel, + Schema, + Transformer, + type TransformerMiddleware, + type YBlock, +} from '@blocksuite/affine/store'; +import { uniq } from 'lodash-es'; +import { + Array as YArray, + type Doc as YDoc, + Map as YMap, + Text as YText, +} from 'yjs'; + +import { getStoreManager } from './bs-store'; + +const blocksuiteSchema = new Schema(); +blocksuiteSchema.register([...AffineSchemas]); + +export interface BlockDocumentInfo { + docId: string; + blockId: string; + content?: string | string[]; + flavour: string; + blob?: string[]; + refDocId?: string[]; + ref?: string[]; + parentFlavour?: string; + parentBlockId?: string; + additional?: { + databaseName?: string; + displayMode?: string; + noteBlockId?: string; + }; + yblock: YMap; + markdownPreview?: string; +} + +const bookmarkFlavours = new Set([ + 'affine:bookmark', + 'affine:embed-youtube', + 'affine:embed-figma', + 'affine:embed-github', + 'affine:embed-loom', +]); + +function generateMarkdownPreviewBuilder( + yRootDoc: YDoc, + workspaceId: string, + blocks: BlockDocumentInfo[] +) { + function yblockToDraftModal(yblock: YBlock): DraftModel | null { + const flavour = yblock.get('sys:flavour') as string; + const blockSchema = blocksuiteSchema.flavourSchemaMap.get(flavour); + if (!blockSchema) { + return null; + } + const keys = Array.from(yblock.keys()) + .filter(key => key.startsWith('prop:')) + .map(key => key.substring(5)); + + const props = Object.fromEntries( + keys.map(key => [key, createYProxy(yblock.get(`prop:${key}`))]) + ); + + return { + props, + id: yblock.get('sys:id') as string, + flavour, + children: [], + role: blockSchema.model.role, + version: (yblock.get('sys:version') as number) ?? blockSchema.version, + keys: Array.from(yblock.keys()) + .filter(key => key.startsWith('prop:')) + .map(key => key.substring(5)), + } as unknown as DraftModel; + } + + const titleMiddleware: TransformerMiddleware = ({ adapterConfigs }) => { + const pages = yRootDoc.getMap('meta').get('pages'); + if (!(pages instanceof YArray)) { + return; + } + for (const meta of pages.toArray()) { + adapterConfigs.set( + 'title:' + meta.get('id'), + meta.get('title')?.toString() ?? 'Untitled' + ); + } + }; + + const baseUrl = `/workspace/${workspaceId}`; + + function getDocLink(docId: string, blockId: string) { + const searchParams = new URLSearchParams(); + searchParams.set('blockIds', blockId); + return `${baseUrl}/${docId}?${searchParams.toString()}`; + } + + const docLinkBaseURLMiddleware: TransformerMiddleware = ({ + adapterConfigs, + }) => { + adapterConfigs.set('docLinkBaseUrl', baseUrl); + }; + + const container = new Container(); + getStoreManager() + .get('store') + .forEach(ext => { + ext.setup(container); + }); + + const provider = container.provider(); + const markdownAdapter = new MarkdownAdapter( + new Transformer({ + schema: blocksuiteSchema, + blobCRUD: { + delete: () => Promise.resolve(), + get: () => Promise.resolve(null), + list: () => Promise.resolve([]), + set: () => Promise.resolve(''), + }, + docCRUD: { + create: () => { + throw new Error('Not implemented'); + }, + get: () => null, + delete: () => {}, + }, + middlewares: [docLinkBaseURLMiddleware, titleMiddleware], + }), + provider + ); + + const markdownPreviewCache = new WeakMap(); + + function trimCodeBlock(markdown: string) { + const lines = markdown.split('\n').filter(line => line.trim() !== ''); + if (lines.length > 5) { + return [...lines.slice(0, 4), '...', lines.at(-1), ''].join('\n'); + } + return [...lines, ''].join('\n'); + } + + function trimParagraph(markdown: string) { + const lines = markdown.split('\n').filter(line => line.trim() !== ''); + + if (lines.length > 3) { + return [...lines.slice(0, 3), '...', lines.at(-1), ''].join('\n'); + } + + return [...lines, ''].join('\n'); + } + + function getListDepth(block: BlockDocumentInfo) { + let parentBlockCount = 0; + let currentBlock: BlockDocumentInfo | undefined = block; + do { + currentBlock = blocks.find( + b => b.blockId === currentBlock?.parentBlockId + ); + + // reach the root block. do not count it. + if (!currentBlock || currentBlock.flavour !== 'affine:list') { + break; + } + parentBlockCount++; + } while (currentBlock); + return parentBlockCount; + } + + // only works for list block + function indentMarkdown(markdown: string, depth: number) { + if (depth <= 0) { + return markdown; + } + + return ( + markdown + .split('\n') + .map(line => ' '.repeat(depth) + line) + .join('\n') + '\n' + ); + } + + const generateDatabaseMarkdownPreview = (block: BlockDocumentInfo) => { + const isDatabaseBlock = (block: BlockDocumentInfo) => { + return block.flavour === 'affine:database'; + }; + + const model = yblockToDraftModal(block.yblock); + + if (!model) { + return null; + } + + let dbBlock: BlockDocumentInfo | null = null; + + if (isDatabaseBlock(block)) { + dbBlock = block; + } else { + const parentBlock = blocks.find(b => b.blockId === block.parentBlockId); + + if (parentBlock && isDatabaseBlock(parentBlock)) { + dbBlock = parentBlock; + } + } + + if (!dbBlock) { + return null; + } + + const url = getDocLink(block.docId, dbBlock.blockId); + const title = dbBlock.additional?.databaseName; + + return `[database · ${title || 'Untitled'}][](${url})\n`; + }; + + const generateImageMarkdownPreview = (block: BlockDocumentInfo) => { + const isImageModel = ( + model: DraftModel | null + ): model is DraftModel => { + return model?.flavour === 'affine:image'; + }; + + const model = yblockToDraftModal(block.yblock); + + if (!isImageModel(model)) { + return null; + } + + const info = ['an image block']; + + if (model.props.sourceId) { + info.push(`file id ${model.props.sourceId}`); + } + + if (model.props.caption) { + info.push(`with caption ${model.props.caption}`); + } + + return info.join(', ') + '\n'; + }; + + const generateEmbedMarkdownPreview = (block: BlockDocumentInfo) => { + const isEmbedModel = ( + model: DraftModel | null + ): model is DraftModel => { + return ( + model?.flavour === 'affine:embed-linked-doc' || + model?.flavour === 'affine:embed-synced-doc' + ); + }; + + const draftModel = yblockToDraftModal(block.yblock); + if (!isEmbedModel(draftModel)) { + return null; + } + + const url = getDocLink(block.docId, draftModel.id); + + return `[](${url})\n`; + }; + + const generateLatexMarkdownPreview = (block: BlockDocumentInfo) => { + let content = + typeof block.content === 'string' + ? block.content.trim() + : block.content?.join('').trim(); + + content = content?.split('\n').join(' ') ?? ''; + + return `LaTeX, with value ${content}\n`; + }; + + const generateBookmarkMarkdownPreview = (block: BlockDocumentInfo) => { + const isBookmarkModel = ( + model: DraftModel | null + ): model is DraftModel => { + return bookmarkFlavours.has(model?.flavour ?? ''); + }; + + const draftModel = yblockToDraftModal(block.yblock); + if (!isBookmarkModel(draftModel)) { + return null; + } + const title = draftModel.props.title; + const url = draftModel.props.url; + return `[${title}](${url})\n`; + }; + + const generateAttachmentMarkdownPreview = (block: BlockDocumentInfo) => { + const isAttachmentModel = ( + model: DraftModel | null + ): model is DraftModel => { + return model?.flavour === 'affine:attachment'; + }; + + const draftModel = yblockToDraftModal(block.yblock); + if (!isAttachmentModel(draftModel)) { + return null; + } + + return `[${draftModel.props.name}](${draftModel.props.sourceId})\n`; + }; + + const generateTableMarkdownPreview = (block: BlockDocumentInfo) => { + const isTableModel = ( + model: DraftModel | null + ): model is DraftModel => { + return model?.flavour === 'affine:table'; + }; + + const draftModel = yblockToDraftModal(block.yblock); + if (!isTableModel(draftModel)) { + return null; + } + + const url = getDocLink(block.docId, draftModel.id); + + return `[table][](${url})\n`; + }; + + const generateMarkdownPreview = async (block: BlockDocumentInfo) => { + if (markdownPreviewCache.has(block)) { + return markdownPreviewCache.get(block); + } + const flavour = block.flavour; + let markdown: string | null = null; + + if ( + flavour === 'affine:paragraph' || + flavour === 'affine:list' || + flavour === 'affine:code' + ) { + const draftModel = yblockToDraftModal(block.yblock); + markdown = + block.parentFlavour === 'affine:database' + ? generateDatabaseMarkdownPreview(block) + : ((draftModel ? await markdownAdapter.fromBlock(draftModel) : null) + ?.file ?? null); + + if (markdown) { + if (flavour === 'affine:code') { + markdown = trimCodeBlock(markdown); + } else if (flavour === 'affine:paragraph') { + markdown = trimParagraph(markdown); + } + } + } else if (flavour === 'affine:database') { + markdown = generateDatabaseMarkdownPreview(block); + } else if ( + flavour === 'affine:embed-linked-doc' || + flavour === 'affine:embed-synced-doc' + ) { + markdown = generateEmbedMarkdownPreview(block); + } else if (flavour === 'affine:attachment') { + markdown = generateAttachmentMarkdownPreview(block); + } else if (flavour === 'affine:image') { + markdown = generateImageMarkdownPreview(block); + } else if (flavour === 'affine:surface' || flavour === 'affine:page') { + // skip + } else if (flavour === 'affine:latex') { + markdown = generateLatexMarkdownPreview(block); + } else if (bookmarkFlavours.has(flavour)) { + markdown = generateBookmarkMarkdownPreview(block); + } else if (flavour === 'affine:table') { + markdown = generateTableMarkdownPreview(block); + } else { + console.warn(`unknown flavour: ${flavour}`); + } + + if (markdown && flavour === 'affine:list') { + const blockDepth = getListDepth(block); + markdown = indentMarkdown(markdown, Math.max(0, blockDepth)); + } + + markdownPreviewCache.set(block, markdown); + return markdown; + }; + + return generateMarkdownPreview; +} + +// remove the indent of the first line of list +// e.g., +// ``` +// - list item 1 +// - list item 2 +// ``` +// becomes +// ``` +// - list item 1 +// - list item 2 +// ``` +function unindentMarkdown(markdown: string) { + const lines = markdown.split('\n'); + const res: string[] = []; + let firstListFound = false; + let baseIndent = 0; + + for (let current of lines) { + const indent = current.match(/^\s*/)?.[0]?.length ?? 0; + + if (indent > 0) { + if (!firstListFound) { + // For the first list item, remove all indentation + firstListFound = true; + baseIndent = indent; + current = current.trimStart(); + } else { + // For subsequent list items, maintain relative indentation + current = + ' '.repeat(Math.max(0, indent - baseIndent)) + current.trimStart(); + } + } + + res.push(current); + } + + return res.join('\n'); +} + +export async function readAllBlocksFromDoc({ + ydoc, + rootYDoc, + spaceId, + maxSummaryLength, +}: { + ydoc: YDoc; + rootYDoc: YDoc; + spaceId: string; + maxSummaryLength?: number; +}): Promise< + | { + blocks: BlockDocumentInfo[]; + title: string; + summary: string; + } + | undefined +> { + let docTitle = ''; + let summary = ''; + maxSummaryLength ??= 1000; + const blockDocuments: BlockDocumentInfo[] = []; + + const generateMarkdownPreview = generateMarkdownPreviewBuilder( + rootYDoc, + spaceId, + blockDocuments + ); + + const blocks = ydoc.getMap('blocks'); + if (blocks.size === 0) { + return undefined; + } + + // build a parent map for quick lookup + // for each block, record its parent id + const parentMap: Record = {}; + for (const [id, block] of blocks.entries()) { + const children = block.get('sys:children') as YArray | undefined; + if (children instanceof YArray && children.length) { + for (const child of children) { + parentMap[child] = id; + } + } + } + + // find the nearest block that satisfies the predicate + const nearest = ( + blockId: string, + predicate: (block: YMap) => boolean + ) => { + let current: string | null = blockId; + while (current) { + const block = blocks.get(current); + if (block && predicate(block)) { + return block; + } + current = parentMap[current] ?? null; + } + return null; + }; + + const nearestByFlavour = (blockId: string, flavour: string) => + nearest(blockId, block => block.get('sys:flavour') === flavour); + + let rootBlockId: string | null = null; + for (const block of blocks.values()) { + const flavour = block.get('sys:flavour')?.toString(); + const blockId = block.get('sys:id')?.toString(); + if (flavour === 'affine:page' && blockId) { + rootBlockId = blockId; + } + } + + if (!rootBlockId) { + return undefined; + } + + const queue: { parent?: string; id: string }[] = [{ id: rootBlockId }]; + const visited = new Set(); // avoid loop + + const pushChildren = (id: string, block: YMap) => { + const children = block.get('sys:children'); + if (children instanceof YArray && children.length) { + for (let i = children.length - 1; i >= 0; i--) { + const childId = children.get(i); + if (childId && !visited.has(childId)) { + queue.push({ parent: id, id: childId }); + visited.add(childId); + } + } + } + }; + + // #region first loop - generate block base info + while (queue.length) { + const next = queue.pop(); + if (!next) { + break; + } + + const { parent: parentBlockId, id: blockId } = next; + const block = blockId ? blocks.get(blockId) : null; + const parentBlock = parentBlockId ? blocks.get(parentBlockId) : null; + if (!block) { + break; + } + + const flavour = block.get('sys:flavour')?.toString(); + const parentFlavour = parentBlock?.get('sys:flavour')?.toString(); + const noteBlock = nearestByFlavour(blockId, 'affine:note'); + + // display mode: + // - both: page and edgeless -> fallback to page + // - page: only page -> page + // - edgeless: only edgeless -> edgeless + // - undefined: edgeless (assuming it is a normal element on the edgeless) + let displayMode = noteBlock?.get('prop:displayMode') ?? 'edgeless'; + + if (displayMode === 'both') { + displayMode = 'page'; + } + + const noteBlockId: string | undefined = noteBlock + ?.get('sys:id') + ?.toString(); + + pushChildren(blockId, block); + + const commonBlockProps = { + docId: ydoc.guid, + flavour, + blockId, + yblock: block, + additional: { displayMode, noteBlockId }, + }; + + if (flavour === 'affine:page') { + docTitle = block.get('prop:title').toString(); + blockDocuments.push({ ...commonBlockProps, content: docTitle }); + } else if ( + flavour === 'affine:paragraph' || + flavour === 'affine:list' || + flavour === 'affine:code' + ) { + const text = block.get('prop:text') as YText; + + if (!text) { + continue; + } + + const deltas: DeltaInsert[] = text.toDelta(); + const refs = uniq( + deltas + .flatMap(delta => { + if ( + delta.attributes && + delta.attributes.reference && + delta.attributes.reference.pageId + ) { + const { pageId: refDocId, params = {} } = + delta.attributes.reference; + return { + refDocId, + ref: JSON.stringify({ docId: refDocId, ...params }), + }; + } + return null; + }) + .filter(ref => !!ref) + ); + + const databaseName = + flavour === 'affine:paragraph' && parentFlavour === 'affine:database' // if block is a database row + ? parentBlock?.get('prop:title')?.toString() + : undefined; + + blockDocuments.push({ + ...commonBlockProps, + content: text.toString(), + ...refs.reduce<{ refDocId: string[]; ref: string[] }>( + (prev, curr) => { + prev.refDocId.push(curr.refDocId); + prev.ref.push(curr.ref); + return prev; + }, + { refDocId: [], ref: [] } + ), + parentFlavour, + parentBlockId, + additional: { ...commonBlockProps.additional, databaseName }, + }); + + if (maxSummaryLength > 0) { + summary += text.toString(); + maxSummaryLength -= text.length; + } + } else if ( + flavour === 'affine:embed-linked-doc' || + flavour === 'affine:embed-synced-doc' + ) { + const pageId = block.get('prop:pageId'); + if (typeof pageId === 'string') { + // reference info + const params = block.get('prop:params') ?? {}; + blockDocuments.push({ + ...commonBlockProps, + refDocId: [pageId], + ref: [JSON.stringify({ docId: pageId, ...params })], + parentFlavour, + parentBlockId, + }); + } + } else if (flavour === 'affine:attachment' || flavour === 'affine:image') { + const blobId = block.get('prop:sourceId'); + if (typeof blobId === 'string') { + blockDocuments.push({ + ...commonBlockProps, + blob: [blobId], + parentFlavour, + parentBlockId, + }); + } + } else if (flavour === 'affine:surface') { + const texts = []; + + const elementsObj = block.get('prop:elements'); + if ( + !( + elementsObj instanceof YMap && + elementsObj.get('type') === '$blocksuite:internal:native$' + ) + ) { + continue; + } + const elements = elementsObj.get('value') as YMap; + if (!(elements instanceof YMap)) { + continue; + } + + for (const element of elements.values()) { + if (!(element instanceof YMap)) { + continue; + } + const text = element.get('text') as YText; + if (!text) { + continue; + } + + texts.push(text.toString()); + } + + blockDocuments.push({ + ...commonBlockProps, + content: texts, + parentFlavour, + parentBlockId, + }); + } else if (flavour === 'affine:database') { + const texts = []; + const columnsObj = block.get('prop:columns'); + const databaseTitle = block.get('prop:title'); + if (databaseTitle instanceof YText) { + texts.push(databaseTitle.toString()); + } + if (columnsObj instanceof YArray) { + for (const column of columnsObj) { + if (!(column instanceof YMap)) { + continue; + } + if (typeof column.get('name') === 'string') { + texts.push(column.get('name')); + } + + const data = column.get('data'); + if (!(data instanceof YMap)) { + continue; + } + const options = data.get('options'); + if (!(options instanceof YArray)) { + continue; + } + for (const option of options) { + if (!(option instanceof YMap)) { + continue; + } + const value = option.get('value'); + if (typeof value === 'string') { + texts.push(value); + } + } + } + } + + blockDocuments.push({ + ...commonBlockProps, + content: texts, + additional: { + ...commonBlockProps.additional, + databaseName: databaseTitle?.toString(), + }, + }); + } else if (flavour === 'affine:latex') { + blockDocuments.push({ + ...commonBlockProps, + content: block.get('prop:latex')?.toString() ?? '', + }); + } else if (flavour === 'affine:table') { + const contents = Array.from(block.keys()) + .map(key => { + if (key.startsWith('prop:cells.') && key.endsWith('.text')) { + return block.get(key)?.toString() ?? ''; + } + return ''; + }) + .filter(Boolean); + blockDocuments.push({ + ...commonBlockProps, + content: contents, + }); + } else if (bookmarkFlavours.has(flavour)) { + blockDocuments.push({ ...commonBlockProps }); + } + } + // #endregion + + // #region second loop - generate markdown preview + const TARGET_PREVIEW_CHARACTER = 500; + const TARGET_PREVIOUS_BLOCK = 1; + const TARGET_FOLLOW_BLOCK = 4; + for (const block of blockDocuments) { + if (block.ref?.length) { + const target = block; + + // should only generate the markdown preview belong to the same affine:note + const noteBlock = nearestByFlavour(block.blockId, 'affine:note'); + + const sameNoteBlocks = noteBlock + ? blockDocuments.filter( + candidate => + nearestByFlavour(candidate.blockId, 'affine:note') === noteBlock + ) + : []; + + // only generate markdown preview for reference blocks + let previewText = (await generateMarkdownPreview(target)) ?? ''; + let previousBlock = 0; + let followBlock = 0; + let previousIndex = sameNoteBlocks.findIndex( + block => block.blockId === target.blockId + ); + let followIndex = previousIndex; + + while ( + !( + ( + previewText.length > TARGET_PREVIEW_CHARACTER || // stop if preview text reaches the limit + ((previousBlock >= TARGET_PREVIOUS_BLOCK || previousIndex < 0) && + (followBlock >= TARGET_FOLLOW_BLOCK || + followIndex >= sameNoteBlocks.length)) + ) // stop if no more blocks, or preview block reaches the limit + ) + ) { + if (previousBlock < TARGET_PREVIOUS_BLOCK) { + previousIndex--; + const block = + previousIndex >= 0 ? sameNoteBlocks.at(previousIndex) : null; + const markdown = block ? await generateMarkdownPreview(block) : null; + if ( + markdown && + !previewText.startsWith( + markdown + ) /* A small hack to skip blocks with the same content */ + ) { + previewText = markdown + '\n' + previewText; + previousBlock++; + } + } + + if (followBlock < TARGET_FOLLOW_BLOCK) { + followIndex++; + const block = sameNoteBlocks.at(followIndex); + const markdown = block ? await generateMarkdownPreview(block) : null; + if ( + markdown && + !previewText.endsWith( + markdown + ) /* A small hack to skip blocks with the same content */ + ) { + previewText = previewText + '\n' + markdown; + followBlock++; + } + } + } + + block.markdownPreview = unindentMarkdown(previewText); + } + } + // #endregion + + return { + blocks: blockDocuments, + title: docTitle, + summary, + }; +} + +/** + * Get all docs from the root doc + */ +export function readAllDocsFromRootDoc( + rootDoc: YDoc, + options?: { + includeTrash?: boolean; + } +) { + const docs = rootDoc.getMap('meta').get('pages') as + | YArray> + | undefined; + const availableDocs = new Map(); + + if (docs) { + for (const page of docs) { + const docId = page.get('id'); + + if (typeof docId !== 'string') { + continue; + } + + const inTrash = page.get('trash') ?? false; + const title = page.get('title'); + + if (!options?.includeTrash && inTrash) { + continue; + } + + availableDocs.set(docId, { title }); + } + } + + return availableDocs; +} + +export function readAllDocIdsFromRootDoc( + rootDoc: YDoc, + options?: { + includeTrash?: boolean; + } +) { + const docs = rootDoc.getMap('meta').get('pages') as + | YArray> + | undefined; + const docIds = new Set(); + if (docs) { + for (const page of docs) { + const docId = page.get('id'); + if (typeof docId !== 'string') { + continue; + } + const inTrash = page.get('trash') ?? false; + if (!options?.includeTrash && inTrash) { + continue; + } + docIds.add(docId); + } + } + return Array.from(docIds); +} diff --git a/packages/common/reader/tsconfig.json b/packages/common/reader/tsconfig.json new file mode 100644 index 0000000000..6819e2c960 --- /dev/null +++ b/packages/common/reader/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../../tsconfig.web.json", + "include": ["./src"], + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist", + "tsBuildInfoFile": "./dist/tsconfig.tsbuildinfo" + }, + "references": [{ "path": "../../../blocksuite/affine/all" }] +} diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts index 9364d9cfc4..b7387e5970 100644 --- a/tools/utils/src/workspace.gen.ts +++ b/tools/utils/src/workspace.gen.ts @@ -1128,12 +1128,18 @@ export const PackageList = [ location: 'packages/common/nbstore', name: '@affine/nbstore', workspaceDependencies: [ + 'packages/common/reader', 'packages/common/infra', 'packages/common/error', 'packages/common/graphql', 'blocksuite/affine/all', ], }, + { + location: 'packages/common/reader', + name: '@affine/reader', + workspaceDependencies: ['blocksuite/affine/all'], + }, { location: 'packages/common/y-octo/node', name: '@y-octo/node', @@ -1470,6 +1476,7 @@ export type PackageName = | '@affine/graphql' | '@toeverything/infra' | '@affine/nbstore' + | '@affine/reader' | '@y-octo/node' | '@affine/admin' | '@affine/android' diff --git a/tsconfig.json b/tsconfig.json index 2f0a8d6e09..9838a18bb5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -126,6 +126,7 @@ { "path": "./packages/common/graphql" }, { "path": "./packages/common/infra" }, { "path": "./packages/common/nbstore" }, + { "path": "./packages/common/reader" }, { "path": "./packages/common/y-octo/node" }, { "path": "./packages/frontend/admin" }, { "path": "./packages/frontend/apps/android" }, diff --git a/yarn.lock b/yarn.lock index 1c60194b25..6ad19327af 100644 --- a/yarn.lock +++ b/yarn.lock @@ -822,6 +822,7 @@ __metadata: dependencies: "@affine/error": "workspace:*" "@affine/graphql": "workspace:*" + "@affine/reader": "workspace:*" "@blocksuite/affine": "workspace:*" "@toeverything/infra": "workspace:*" eventemitter2: "npm:^6.4.9" @@ -858,6 +859,19 @@ __metadata: languageName: unknown linkType: soft +"@affine/reader@workspace:*, @affine/reader@workspace:packages/common/reader": + version: 0.0.0-use.local + resolution: "@affine/reader@workspace:packages/common/reader" + dependencies: + "@blocksuite/affine": "workspace:*" + lodash-es: "npm:^4.17.21" + vitest: "npm:3.1.3" + yjs: "npm:^13.6.21" + peerDependencies: + "@blocksuite/affine": "workspace:*" + languageName: unknown + linkType: soft + "@affine/routes@workspace:*, @affine/routes@workspace:packages/frontend/routes": version: 0.0.0-use.local resolution: "@affine/routes@workspace:packages/frontend/routes"