feat: enhance markdown preview for backlinks (#8956)

fix AF-1770
fix AF-1771

---

fix: doc link middlewares
feat: markdown renderer
feat: allow multiple backlink for a single doc
feat: showing correct doc ref link
feat: trim long para & ident lists
feat: list indentition fix
feat: database/latext handling
feat: other block types handling
fix: lint
This commit is contained in:
pengx17
2024-12-02 10:29:44 +00:00
parent b8467a91b8
commit 31bab2507e
19 changed files with 942 additions and 186 deletions

View File

@@ -1,4 +1,9 @@
import { BlockStdScope, type EditorHost } from '@blocksuite/affine/block-std';
import {
BlockStdScope,
type EditorHost,
type ExtensionType,
ShadowlessElement,
} from '@blocksuite/affine/block-std';
import type {
AffineAIPanelState,
AffineAIPanelWidgetConfig,
@@ -10,8 +15,14 @@ import {
ParagraphBlockComponent,
} from '@blocksuite/affine/blocks';
import { WithDisposable } from '@blocksuite/affine/global/utils';
import { BlockViewType, type Doc, type Query } from '@blocksuite/affine/store';
import { css, html, LitElement, nothing, type PropertyValues } from 'lit';
import {
BlockViewType,
type Doc,
type JobMiddleware,
type Query,
type Schema,
} from '@blocksuite/affine/store';
import { css, html, nothing, type PropertyValues } from 'lit';
import { property, query } from 'lit/decorators.js';
import { classMap } from 'lit/directives/class-map.js';
import { keyed } from 'lit/directives/keyed.js';
@@ -70,9 +81,12 @@ const customHeadingStyles = css`
export type TextRendererOptions = {
maxHeight?: number;
customHeading?: boolean;
extensions?: ExtensionType[];
additionalMiddlewares?: JobMiddleware[];
};
export class TextRenderer extends WithDisposable(LitElement) {
// todo: refactor it for more general purpose usage instead of AI only?
export class TextRenderer extends WithDisposable(ShadowlessElement) {
static override styles = css`
.ai-answer-text-editor.affine-page-viewport {
background: transparent;
@@ -177,8 +191,9 @@ export class TextRenderer extends WithDisposable(LitElement) {
if (this._answers.length > 0) {
const latestAnswer = this._answers.pop();
this._answers = [];
if (latestAnswer) {
markDownToDoc(this.host, latestAnswer)
const schema = this.schema ?? this.host?.std.doc.collection.schema;
if (latestAnswer && schema) {
markDownToDoc(schema, latestAnswer, this.options.additionalMiddlewares)
.then(doc => {
this._doc = doc.blockCollection.getDoc({
query: this._query,
@@ -245,7 +260,7 @@ export class TextRenderer extends WithDisposable(LitElement) {
html`<div class="ai-answer-text-editor affine-page-viewport">
${new BlockStdScope({
doc: this._doc,
extensions: CustomPageEditorBlockSpecs,
extensions: this.options.extensions ?? CustomPageEditorBlockSpecs,
}).render()}
</div>`
)}
@@ -277,7 +292,10 @@ export class TextRenderer extends WithDisposable(LitElement) {
accessor answer!: string;
@property({ attribute: false })
accessor host!: EditorHost;
accessor host: EditorHost | null = null;
@property({ attribute: false })
accessor schema: Schema | null = null;
@property({ attribute: false })
accessor options!: TextRendererOptions;

View File

@@ -12,6 +12,7 @@ import {
PlainTextAdapter,
titleMiddleware,
} from '@blocksuite/affine/blocks';
import type { JobMiddleware, Schema } from '@blocksuite/affine/store';
import { DocCollection, Job } from '@blocksuite/affine/store';
import { assertExists } from '@blocksuite/global/utils';
import type {
@@ -184,16 +185,23 @@ export async function replaceFromMarkdown(
await job.snapshotToSlice(snapshot, host.doc, parent, index);
}
export async function markDownToDoc(host: EditorHost, answer: string) {
const schema = host.std.doc.collection.schema;
export async function markDownToDoc(
schema: Schema,
answer: string,
additionalMiddlewares?: JobMiddleware[]
) {
// Should not create a new doc in the original collection
const collection = new DocCollection({
schema,
});
collection.meta.initialize();
const middlewares = [defaultImageProxyMiddleware];
if (additionalMiddlewares) {
middlewares.push(...additionalMiddlewares);
}
const job = new Job({
collection,
middlewares: [defaultImageProxyMiddleware],
middlewares,
});
const mdAdapter = new MarkdownAdapter(job);
const doc = await mdAdapter.toDoc({

View File

@@ -142,7 +142,7 @@ export const copyTextAnswer = async (panel: AffineAIPanelWidget) => {
};
export const copyText = async (host: EditorHost, text: string) => {
const previewDoc = await markDownToDoc(host, text);
const previewDoc = await markDownToDoc(host.std.doc.schema, text);
const models = previewDoc
.getBlocksByFlavour('affine:note')
.map(b => b.model)

View File

@@ -0,0 +1 @@
export * from './_common/components/text-renderer';

View File

@@ -7,7 +7,12 @@ import { useI18n } from '@affine/i18n';
import { track } from '@affine/track';
import type { DocMode } from '@blocksuite/affine/blocks';
import type { DocCollection } from '@blocksuite/affine/store';
import { LiveData, useLiveData, useService } from '@toeverything/infra';
import {
DocsService,
LiveData,
useLiveData,
useService,
} from '@toeverything/infra';
import clsx from 'clsx';
import { nanoid } from 'nanoid';
import {
@@ -36,6 +41,7 @@ function AffinePageReferenceInner({
Icon: UserIcon,
}: AffinePageReferenceProps) {
const docDisplayMetaService = useService(DocDisplayMetaService);
const docsService = useService(DocsService);
const i18n = useI18n();
let linkWithMode: DocMode | null = null;
@@ -62,15 +68,19 @@ function AffinePageReferenceInner({
);
})
);
const title = useLiveData(
const notFound = !useLiveData(docsService.list.doc$(pageId));
let title = useLiveData(
docDisplayMetaService.title$(pageId, { reference: true })
);
title = notFound ? i18n.t('com.affine.notFoundPage.title') : title;
return (
<>
<span className={notFound ? styles.notFound : ''}>
<Icon className={styles.pageReferenceIcon} />
<span className="affine-reference-title">{i18n.t(title)}</span>
</>
</span>
);
}

View File

@@ -1,9 +1,11 @@
import { style } from '@vanilla-extract/css';
import { cssVarV2 } from '@toeverything/theme/v2';
import { globalStyle, style } from '@vanilla-extract/css';
export const pageReferenceIcon = style({
verticalAlign: 'middle',
fontSize: '1.1em',
transform: 'translate(2px, -1px)',
color: cssVarV2('icon/primary'),
});
export const pageReferenceLink = style({
@@ -12,3 +14,13 @@ export const pageReferenceLink = style({
wordBreak: 'break-word',
hyphens: 'auto',
});
export const notFound = style({
color: cssVarV2('text/secondary'),
textDecoration: 'line-through',
});
globalStyle('affine-reference .affine-reference', {
color: 'inherit !important',
textDecoration: 'none !important',
});

View File

@@ -1,5 +1,6 @@
import { cssVar } from '@toeverything/theme';
import { style } from '@vanilla-extract/css';
import { cssVarV2 } from '@toeverything/theme/v2';
import { globalStyle, style } from '@vanilla-extract/css';
export const container = style({
width: '100%',
@@ -49,7 +50,6 @@ export const title = style({
});
export const showButton = style({
width: '56px',
height: '28px',
borderRadius: '8px',
border: '1px solid ' + cssVar('--affine-border-color'),
@@ -74,9 +74,45 @@ export const linksTitles = style({
export const link = style({
width: '100%',
height: '32px',
height: '30px',
display: 'flex',
alignItems: 'center',
gap: '4px',
whiteSpace: 'nowrap',
});
globalStyle(`${link} .affine-reference-title`, {
borderBottom: 'none',
});
export const linkPreviewContainer = style({
display: 'flex',
flexDirection: 'column',
gap: '8px',
});
export const linkPreview = style({
border: `1px solid ${cssVarV2('layer/insideBorder/border')}`,
borderRadius: '8px',
padding: '8px',
color: cssVarV2('text/primary'),
':hover': {
backgroundColor: cssVarV2('layer/background/hoverOverlay'),
},
});
export const linkPreviewRenderer = style({
cursor: 'pointer',
});
export const collapsedIcon = style({
transition: 'all 0.2s ease-in-out',
color: cssVarV2('icon/primary'),
fontSize: 20,
selectors: {
'&[data-collapsed="true"]': {
transform: 'rotate(90deg)',
color: cssVarV2('icon/secondary'),
},
},
});

View File

@@ -1,32 +1,179 @@
import {
Button,
createReactComponentFromLit,
useLitPortalFactory,
} from '@affine/component';
import { TextRenderer } from '@affine/core/blocksuite/presets';
import {
type Backlink,
DocLinksService,
type Link,
} from '@affine/core/modules/doc-link';
import { toURLSearchParams } from '@affine/core/modules/navigation';
import { WorkbenchLink } from '@affine/core/modules/workbench';
import { useI18n } from '@affine/i18n';
import { LiveData, useLiveData, useServices } from '@toeverything/infra';
import { Fragment, useCallback, useState } from 'react';
import type { JobMiddleware } from '@blocksuite/affine/store';
import { ToggleExpandIcon } from '@blocksuite/icons/rc';
import * as Collapsible from '@radix-ui/react-collapsible';
import {
getAFFiNEWorkspaceSchema,
LiveData,
useFramework,
useLiveData,
useServices,
WorkspaceService,
} from '@toeverything/infra';
import React, {
Fragment,
type ReactNode,
useCallback,
useMemo,
useState,
} from 'react';
import { AffinePageReference } from '../../affine/reference-link';
import {
AffinePageReference,
AffineSharedPageReference,
} from '../../affine/reference-link';
import * as styles from './bi-directional-link-panel.css';
import {
patchReferenceRenderer,
type ReferenceReactRenderer,
} from './specs/custom/spec-patchers';
import { createPageModeSpecs } from './specs/page';
const BlocksuiteTextRenderer = createReactComponentFromLit({
react: React,
elementClass: TextRenderer,
});
const CollapsibleSection = ({
title,
children,
length,
}: {
title: ReactNode;
children: ReactNode;
length?: number;
}) => {
const [open, setOpen] = useState(false);
return (
<Collapsible.Root open={open} onOpenChange={setOpen}>
<Collapsible.Trigger className={styles.link}>
{title}
{length ? (
<ToggleExpandIcon
className={styles.collapsedIcon}
data-collapsed={!open}
/>
) : null}
</Collapsible.Trigger>
<Collapsible.Content>{children}</Collapsible.Content>
</Collapsible.Root>
);
};
const usePreviewExtensions = () => {
const [reactToLit, portals] = useLitPortalFactory();
const framework = useFramework();
const { workspaceService } = useServices({
WorkspaceService,
});
const referenceRenderer: ReferenceReactRenderer = useMemo(() => {
return function customReference(reference) {
const data = reference.delta.attributes?.reference;
if (!data) return <span />;
const pageId = data.pageId;
if (!pageId) return <span />;
const params = toURLSearchParams(data.params);
if (workspaceService.workspace.openOptions.isSharedMode) {
return (
<AffineSharedPageReference
docCollection={workspaceService.workspace.docCollection}
pageId={pageId}
params={params}
/>
);
}
return <AffinePageReference pageId={pageId} params={params} />;
};
}, [workspaceService]);
const extensions = useMemo(() => {
const specs = createPageModeSpecs(framework);
return [patchReferenceRenderer(reactToLit, referenceRenderer), ...specs];
}, [reactToLit, referenceRenderer, framework]);
return [extensions, portals] as const;
};
export const BiDirectionalLinkPanel = () => {
const [show, setShow] = useState(false);
const { docLinksService } = useServices({
const { docLinksService, workspaceService } = useServices({
DocLinksService,
WorkspaceService,
});
const [extensions, portals] = usePreviewExtensions();
const t = useI18n();
const links = useLiveData(
show ? docLinksService.links.links$ : new LiveData([] as Link[])
);
const backlinks = useLiveData(
show ? docLinksService.backlinks.backlinks$ : new LiveData([] as Backlink[])
const backlinkGroups = useLiveData(
LiveData.computed(get => {
if (!show) {
return [];
}
const links = get(docLinksService.backlinks.backlinks$);
// group by docId
const groupedLinks = links.reduce(
(acc, link) => {
acc[link.docId] = [...(acc[link.docId] || []), link];
return acc;
},
{} as Record<string, Backlink[]>
);
return Object.entries(groupedLinks).map(([docId, links]) => ({
docId,
title: links[0].title, // title should be the same for all blocks
links,
}));
})
);
const backlinkCount = useMemo(() => {
return backlinkGroups.reduce((acc, link) => acc + link.links.length, 0);
}, [backlinkGroups]);
const handleClickShow = useCallback(() => {
setShow(!show);
}, [show]);
const textRendererOptions = useMemo(() => {
const docLinkBaseURLMiddleware: JobMiddleware = ({ adapterConfigs }) => {
adapterConfigs.set(
'docLinkBaseUrl',
`/workspace/${workspaceService.workspace.id}`
);
};
return {
customHeading: true,
extensions,
additionalMiddlewares: [docLinkBaseURLMiddleware],
};
}, [extensions, workspaceService.workspace.id]);
return (
<div className={styles.container}>
{!show && (
@@ -37,9 +184,11 @@ export const BiDirectionalLinkPanel = () => {
<div className={styles.titleLine}>
<div className={styles.title}>Bi-Directional Links</div>
<div className={styles.showButton} onClick={handleClickShow}>
{show ? 'Hide' : 'Show'}
</div>
<Button className={styles.showButton} onClick={handleClickShow}>
{show
? t['com.affine.editor.bi-directional-link-panel.hide']()
: t['com.affine.editor.bi-directional-link-panel.show']()}
</Button>
</div>
{show && (
@@ -49,17 +198,78 @@ export const BiDirectionalLinkPanel = () => {
</div>
<div className={styles.linksContainer}>
<div className={styles.linksTitles}>
{t['com.affine.page-properties.backlinks']()} · {backlinks.length}
{t['com.affine.page-properties.backlinks']()} · {backlinkCount}
</div>
{backlinks.map(link => (
<Fragment key={link.docId}>
<div className={styles.link}>
<AffinePageReference key={link.docId} pageId={link.docId} />
{backlinkGroups.map(linkGroup => (
<CollapsibleSection
key={linkGroup.docId}
title={<AffinePageReference pageId={linkGroup.docId} />}
length={linkGroup.links.length}
>
<div className={styles.linkPreviewContainer}>
{linkGroup.links.map(link => {
if (!link.markdownPreview) {
return null;
}
const searchParams = new URLSearchParams();
const displayMode = link.displayMode || 'page';
searchParams.set('mode', displayMode);
let blockId = link.blockId;
if (
link.parentFlavour === 'affine:database' &&
link.parentBlockId
) {
// if parentBlockFlavour is 'affine:database',
// we will fallback to the database block instead
blockId = link.parentBlockId;
} else if (displayMode === 'edgeless' && link.noteBlockId) {
// if note has displayMode === 'edgeless' && has noteBlockId,
// set noteBlockId as blockId
blockId = link.noteBlockId;
}
searchParams.set('blockIds', blockId);
const to = {
pathname: '/' + linkGroup.docId,
search: '?' + searchParams.toString(),
hash: '',
};
// if this backlink has no noteBlock && displayMode is edgeless, we will render
// the link as a page link
const edgelessLink =
displayMode === 'edgeless' && !link.noteBlockId;
return (
<WorkbenchLink
to={to}
key={link.blockId}
className={styles.linkPreview}
>
{edgelessLink ? (
<>
[Edgeless]
<AffinePageReference
key={link.blockId}
pageId={linkGroup.docId}
params={searchParams}
/>
</>
) : (
<BlocksuiteTextRenderer
className={styles.linkPreviewRenderer}
answer={link.markdownPreview}
schema={getAFFiNEWorkspaceSchema()}
options={textRendererOptions}
/>
)}
</WorkbenchLink>
);
})}
</div>
<br />
<pre style={{ opacity: 0.5 }}>{link.markdownPreview}</pre>
<br />
</Fragment>
</CollapsibleSection>
))}
</div>
<div className={styles.linksContainer}>
@@ -78,6 +288,13 @@ export const BiDirectionalLinkPanel = () => {
</div>
</>
)}
{
<>
{portals.map(p => (
<Fragment key={p.id}>{p.portal}</Fragment>
))}
</>
}
</div>
);
};

View File

@@ -48,11 +48,23 @@ export const InfoTable = ({
[docId, docsSearchService]
)
);
const backlinks = useLiveData(
useMemo(
() => LiveData.from(docsSearchService.watchRefsTo(docId), null),
[docId, docsSearchService]
)
useMemo(() => {
return LiveData.from(docsSearchService.watchRefsTo(docId), []).map(
links => {
const visitedDoc = new Set<string>();
// for each doc, we only show the first block
return links.filter(link => {
if (visitedDoc.has(link.docId)) {
return false;
}
visitedDoc.add(link.docId);
return true;
});
}
);
}, [docId, docsSearchService])
);
const onBacklinkPropertyChange = useCallback(

View File

@@ -47,10 +47,21 @@ export const DocInfoSheet = ({
)
);
const backlinks = useLiveData(
useMemo(
() => LiveData.from(docsSearchService.watchRefsTo(docId), null),
[docId, docsSearchService]
)
useMemo(() => {
return LiveData.from(docsSearchService.watchRefsTo(docId), []).map(
links => {
const visitedDoc = new Set<string>();
// for each doc, we only show the first block
return links.filter(link => {
if (visitedDoc.has(link.docId)) {
return false;
}
visitedDoc.add(link.docId);
return true;
});
}
);
}, [docId, docsSearchService])
);
const [newPropertyId, setNewPropertyId] = useState<string | null>(null);

View File

@@ -7,6 +7,10 @@ export interface Backlink {
docId: string;
blockId: string;
title: string;
noteBlockId?: string;
displayMode?: string;
parentBlockId?: string;
parentFlavour?: string;
markdownPreview?: string;
}

View File

@@ -146,6 +146,7 @@ export class DocsIndexer extends Entity {
allIndexedDocs,
rootDocBuffer,
reindexAll: isUpgrade,
rootDocId: this.workspaceId,
});
} else {
const rootDocBuffer =
@@ -167,6 +168,7 @@ export class DocsIndexer extends Entity {
docBuffer,
storageDocId,
rootDocBuffer,
rootDocId: this.workspaceId,
});
}

View File

@@ -26,7 +26,7 @@ export const blockIndexSchema = defineSchema({
// parent block id
parentBlockId: 'String',
// additional info
// { "databaseName": "xxx" }
// { "databaseName": "xxx", "displayMode": "page/edgeless", "noteBlockId": "xxx" }
additional: { type: 'String', index: false },
markdownPreview: { type: 'String', index: false },
});

View File

@@ -478,9 +478,16 @@ export class DocsSearchService extends Service {
'docId',
{
hits: {
fields: ['docId', 'blockId', 'markdownPreview'],
fields: [
'docId',
'blockId',
'parentBlockId',
'parentFlavour',
'additional',
'markdownPreview',
],
pagination: {
limit: 1,
limit: 5, // the max number of backlinks to show for each doc
},
},
pagination: {
@@ -495,21 +502,60 @@ export class DocsSearchService extends Service {
buckets.map(bucket => bucket.key)
);
return buckets.map(bucket => {
return buckets.flatMap(bucket => {
const title =
docData.find(doc => doc.id === bucket.key)?.get('title') ?? '';
const blockId = bucket.hits.nodes[0]?.fields.blockId ?? '';
const markdownPreview =
bucket.hits.nodes[0]?.fields.markdownPreview ?? '';
return {
docId: bucket.key,
blockId: typeof blockId === 'string' ? blockId : blockId[0],
title: typeof title === 'string' ? title : title[0],
markdownPreview:
typeof markdownPreview === 'string'
? markdownPreview
: markdownPreview[0],
};
return bucket.hits.nodes.map(node => {
const blockId = node.fields.blockId ?? '';
const markdownPreview = node.fields.markdownPreview ?? '';
const additional =
typeof node.fields.additional === 'string'
? node.fields.additional
: node.fields.additional[0];
const additionalData: {
displayMode?: string;
noteBlockId?: string;
} = JSON.parse(additional || '{}');
const displayMode = additionalData.displayMode ?? '';
const noteBlockId = additionalData.noteBlockId ?? '';
const parentBlockId =
typeof node.fields.parentBlockId === 'string'
? node.fields.parentBlockId
: node.fields.parentBlockId[0];
const parentFlavour =
typeof node.fields.parentFlavour === 'string'
? node.fields.parentFlavour
: node.fields.parentFlavour[0];
return {
docId: bucket.key,
blockId: typeof blockId === 'string' ? blockId : blockId[0],
title: typeof title === 'string' ? title : title[0],
markdownPreview:
typeof markdownPreview === 'string'
? markdownPreview
: markdownPreview[0],
displayMode:
typeof displayMode === 'string'
? displayMode
: displayMode[0],
noteBlockId:
typeof noteBlockId === 'string'
? noteBlockId
: noteBlockId[0],
parentBlockId:
typeof parentBlockId === 'string'
? parentBlockId
: parentBlockId[0],
parentFlavour:
typeof parentFlavour === 'string'
? parentFlavour
: parentFlavour[0],
};
});
});
});
})

View File

@@ -1,12 +1,17 @@
import {
type AffineTextAttributes,
MarkdownAdapter,
import type {
AffineTextAttributes,
AttachmentBlockModel,
BookmarkBlockModel,
EmbedBlockModel,
ImageBlockModel,
} from '@blocksuite/affine/blocks';
import { MarkdownAdapter } from '@blocksuite/affine/blocks';
import {
createYProxy,
DocCollection,
type DraftModel,
Job,
type JobMiddleware,
type YBlock,
} from '@blocksuite/affine/store';
import type { DeltaInsert } from '@blocksuite/inline';
@@ -73,42 +78,6 @@ async function getOrCreateCachedYDoc(data: Uint8Array) {
}
}
function yblockToDraftModal(yblock: YBlock): DraftModel | null {
const flavour = yblock.get('sys:flavour');
const blockSchema = blocksuiteSchema.flavourSchemaMap.get(flavour);
if (!blockSchema) {
return null;
}
const keys = Array.from(yblock.keys())
.filter(key => key.startsWith('prop:'))
.map(key => key.substring(5));
const props = Object.fromEntries(
keys.map(key => [key, createYProxy(yblock.get(`prop:${key}`))])
);
return {
...props,
id: yblock.get('sys:id'),
flavour,
children: [],
role: blockSchema.model.role,
version: (yblock.get('sys:version') as number) ?? blockSchema.version,
keys: Array.from(yblock.keys())
.filter(key => key.startsWith('prop:'))
.map(key => key.substring(5)),
};
}
const markdownAdapter = new MarkdownAdapter(
new Job({
collection: new DocCollection({
id: 'indexer',
schema: blocksuiteSchema,
}),
})
);
interface BlockDocumentInfo {
docId: string;
blockId: string;
@@ -119,50 +88,364 @@ interface BlockDocumentInfo {
ref?: string[];
parentFlavour?: string;
parentBlockId?: string;
additional?: { databaseName?: string };
additional?: {
databaseName?: string;
displayMode?: string;
noteBlockId?: string;
};
yblock: YMap<any>;
markdownPreview?: string;
}
const markdownPreviewCache = new WeakMap<BlockDocumentInfo, string | null>();
const generateMarkdownPreview = async (block: BlockDocumentInfo) => {
if (markdownPreviewCache.has(block)) {
return markdownPreviewCache.get(block);
const bookmarkFlavours = new Set([
'affine:bookmark',
'affine:embed-youtube',
'affine:embed-figma',
'affine:embed-github',
'affine:embed-loom',
]);
function generateMarkdownPreviewBuilder(
yRootDoc: YDoc,
workspaceId: string,
blocks: BlockDocumentInfo[]
) {
function yblockToDraftModal(yblock: YBlock): DraftModel | null {
const flavour = yblock.get('sys:flavour');
const blockSchema = blocksuiteSchema.flavourSchemaMap.get(flavour);
if (!blockSchema) {
return null;
}
const keys = Array.from(yblock.keys())
.filter(key => key.startsWith('prop:'))
.map(key => key.substring(5));
const props = Object.fromEntries(
keys.map(key => [key, createYProxy(yblock.get(`prop:${key}`))])
);
return {
...props,
id: yblock.get('sys:id'),
flavour,
children: [],
role: blockSchema.model.role,
version: (yblock.get('sys:version') as number) ?? blockSchema.version,
keys: Array.from(yblock.keys())
.filter(key => key.startsWith('prop:'))
.map(key => key.substring(5)),
};
}
const flavour = block.flavour;
let markdown: string | null = null;
if (
flavour === 'affine:paragraph' ||
flavour === 'affine:list' ||
flavour === 'affine:code'
) {
const titleMiddleware: JobMiddleware = ({ adapterConfigs }) => {
const pages = yRootDoc.getMap('meta').get('pages');
if (!(pages instanceof YArray)) {
return;
}
for (const meta of pages.toArray()) {
adapterConfigs.set(
'title:' + meta.get('id'),
meta.get('title')?.toString() ?? 'Untitled'
);
}
};
const baseUrl = `/workspace/${workspaceId}`;
function getDocLink(docId: string, blockId: string) {
const searchParams = new URLSearchParams();
searchParams.set('blockIds', blockId);
return `${baseUrl}/${docId}?${searchParams.toString()}`;
}
const docLinkBaseURLMiddleware: JobMiddleware = ({ adapterConfigs }) => {
adapterConfigs.set('docLinkBaseUrl', baseUrl);
};
const markdownAdapter = new MarkdownAdapter(
new Job({
collection: new DocCollection({
id: 'indexer',
schema: blocksuiteSchema,
}),
middlewares: [docLinkBaseURLMiddleware, titleMiddleware],
})
);
const markdownPreviewCache = new WeakMap<BlockDocumentInfo, string | null>();
function trimCodeBlock(markdown: string) {
const lines = markdown.split('\n').filter(line => line.trim() !== '');
if (lines.length > 5) {
return [...lines.slice(0, 4), '...', lines.at(-1), ''].join('\n');
}
return [...lines, ''].join('\n');
}
function trimParagraph(markdown: string) {
const lines = markdown.split('\n').filter(line => line.trim() !== '');
if (lines.length > 3) {
return [...lines.slice(0, 3), '...', lines.at(-1), ''].join('\n');
}
return [...lines, ''].join('\n');
}
function getListDepth(block: BlockDocumentInfo) {
let parentBlockCount = 0;
let currentBlock: BlockDocumentInfo | undefined = block;
do {
currentBlock = blocks.find(
b => b.blockId === currentBlock?.parentBlockId
);
// reach the root block. do not count it.
if (!currentBlock || currentBlock.flavour !== 'affine:list') {
break;
}
parentBlockCount++;
} while (currentBlock);
return parentBlockCount;
}
// only works for list block
function indentMarkdown(markdown: string, depth: number) {
if (depth <= 0) {
return markdown;
}
return (
markdown
.split('\n')
.map(line => ' '.repeat(depth) + line)
.join('\n') + '\n'
);
}
const generateDatabaseMarkdownPreview = (block: BlockDocumentInfo) => {
const isDatabaseBlock = (block: BlockDocumentInfo) => {
return block.flavour === 'affine:database';
};
const model = yblockToDraftModal(block.yblock);
if (!model) {
return null;
}
let dbBlock: BlockDocumentInfo | null = null;
if (isDatabaseBlock(block)) {
dbBlock = block;
} else {
const parentBlock = blocks.find(b => b.blockId === block.parentBlockId);
if (parentBlock && isDatabaseBlock(parentBlock)) {
dbBlock = parentBlock;
}
}
if (!dbBlock) {
return null;
}
const url = getDocLink(block.docId, dbBlock.blockId);
const title = dbBlock.additional?.databaseName;
return `[database · ${title || 'Untitled'}][](${url})\n`;
};
const generateImageMarkdownPreview = (block: BlockDocumentInfo) => {
const isImageModel = (
model: DraftModel | null
): model is DraftModel<ImageBlockModel> => {
return model?.flavour === 'affine:image';
};
const model = yblockToDraftModal(block.yblock);
if (!isImageModel(model)) {
return null;
}
const info = ['an image block'];
if (model.sourceId) {
info.push(`file id ${model.sourceId}`);
}
if (model.caption) {
info.push(`with caption ${model.caption}`);
}
return info.join(', ') + '\n';
};
const generateEmbedMarkdownPreview = (block: BlockDocumentInfo) => {
const isEmbedModel = (
model: DraftModel | null
): model is DraftModel<EmbedBlockModel> => {
return (
model?.flavour === 'affine:embed-linked-doc' ||
model?.flavour === 'affine:embed-synced-doc'
);
};
const draftModel = yblockToDraftModal(block.yblock);
markdown =
block.parentFlavour === 'affine:database'
? `database · ${block.additional?.databaseName}\n`
: ((draftModel ? await markdownAdapter.fromBlock(draftModel) : null)
?.file ?? null);
if (!isEmbedModel(draftModel)) {
return null;
}
const url = getDocLink(block.docId, draftModel.id);
return `[](${url})\n`;
};
const generateLatexMarkdownPreview = (block: BlockDocumentInfo) => {
let content =
typeof block.content === 'string'
? block.content.trim()
: block.content?.join('').trim();
content = content?.split('\n').join(' ') ?? '';
return `LaTeX, with value ${content}\n`;
};
const generateBookmarkMarkdownPreview = (block: BlockDocumentInfo) => {
const isBookmarkModel = (
model: DraftModel | null
): model is DraftModel<BookmarkBlockModel> => {
return bookmarkFlavours.has(model?.flavour ?? '');
};
const draftModel = yblockToDraftModal(block.yblock);
if (!isBookmarkModel(draftModel)) {
return null;
}
const title = draftModel.title;
const url = draftModel.url;
return `[${title}](${url})\n`;
};
const generateAttachmentMarkdownPreview = (block: BlockDocumentInfo) => {
const isAttachmentModel = (
model: DraftModel | null
): model is DraftModel<AttachmentBlockModel> => {
return model?.flavour === 'affine:attachment';
};
const draftModel = yblockToDraftModal(block.yblock);
if (!isAttachmentModel(draftModel)) {
return null;
}
return `[${draftModel.name}](${draftModel.sourceId})\n`;
};
const generateMarkdownPreview = async (block: BlockDocumentInfo) => {
if (markdownPreviewCache.has(block)) {
return markdownPreviewCache.get(block);
}
const flavour = block.flavour;
let markdown: string | null = null;
if (
flavour === 'affine:paragraph' ||
flavour === 'affine:list' ||
flavour === 'affine:code'
) {
const draftModel = yblockToDraftModal(block.yblock);
markdown =
block.parentFlavour === 'affine:database'
? generateDatabaseMarkdownPreview(block)
: ((draftModel ? await markdownAdapter.fromBlock(draftModel) : null)
?.file ?? null);
if (markdown) {
if (flavour === 'affine:code') {
markdown = trimCodeBlock(markdown);
} else if (flavour === 'affine:paragraph') {
markdown = trimParagraph(markdown);
}
}
} else if (flavour === 'affine:database') {
markdown = generateDatabaseMarkdownPreview(block);
} else if (
flavour === 'affine:embed-linked-doc' ||
flavour === 'affine:embed-synced-doc'
) {
markdown = generateEmbedMarkdownPreview(block);
} else if (flavour === 'affine:attachment') {
markdown = generateAttachmentMarkdownPreview(block);
} else if (flavour === 'affine:image') {
markdown = generateImageMarkdownPreview(block);
} else if (flavour === 'affine:surface' || flavour === 'affine:page') {
// skip
} else if (flavour === 'affine:latex') {
markdown = generateLatexMarkdownPreview(block);
} else if (bookmarkFlavours.has(flavour)) {
markdown = generateBookmarkMarkdownPreview(block);
} else {
console.warn(`unknown flavour: ${flavour}`);
}
if (markdown && flavour === 'affine:list') {
const blockDepth = getListDepth(block);
markdown = indentMarkdown(markdown, Math.max(0, blockDepth));
}
markdownPreviewCache.set(block, markdown);
return markdown;
};
return generateMarkdownPreview;
}
// remove the indent of the first line of list
// e.g.,
// ```
// - list item 1
// - list item 2
// ```
// becomes
// ```
// - list item 1
// - list item 2
// ```
function unindentMarkdown(markdown: string) {
const lines = markdown.split('\n');
const res: string[] = [];
let firstListFound = false;
let baseIndent = 0;
for (let current of lines) {
const indent = current.match(/^\s*/)?.[0]?.length ?? 0;
if (indent > 0) {
if (!firstListFound) {
// For the first list item, remove all indentation
firstListFound = true;
baseIndent = indent;
current = current.trimStart();
} else {
// For subsequent list items, maintain relative indentation
current = ' '.repeat(indent - baseIndent) + current.trimStart();
}
}
res.push(current);
}
if (
flavour === 'affine:embed-linked-doc' ||
flavour === 'affine:embed-synced-doc'
) {
markdown = '🔗\n';
}
if (flavour === 'affine:attachment') {
markdown = '📃\n';
}
if (flavour === 'affine:image') {
markdown = '🖼️\n';
}
markdownPreviewCache.set(block, markdown);
return markdown;
};
return res.join('\n');
}
async function crawlingDocData({
docBuffer,
storageDocId,
rootDocBuffer,
rootDocId,
}: WorkerInput & { type: 'doc' }): Promise<WorkerOutput> {
if (isEmptyUpdate(rootDocBuffer)) {
console.warn('[worker]: Empty root doc buffer');
@@ -210,12 +493,49 @@ async function crawlingDocData({
let summary = '';
const blockDocuments: BlockDocumentInfo[] = [];
const generateMarkdownPreview = generateMarkdownPreviewBuilder(
yRootDoc,
rootDocId,
blockDocuments
);
const blocks = ydoc.getMap<any>('blocks');
// build a parent map for quick lookup
// for each block, record its parent id
const parentMap: Record<string, string | null> = {};
for (const [id, block] of blocks.entries()) {
const children = block.get('sys:children') as YArray<string> | undefined;
if (children instanceof YArray && children.length) {
for (const child of children) {
parentMap[child] = id;
}
}
}
if (blocks.size === 0) {
return { deletedDoc: [docId] };
}
// find the nearest block that satisfies the predicate
const nearest = (
blockId: string,
predicate: (block: YMap<any>) => boolean
) => {
let current: string | null = blockId;
while (current) {
const block = blocks.get(current);
if (block && predicate(block)) {
return block;
}
current = parentMap[current] ?? null;
}
return null;
};
const nearestByFlavour = (blockId: string, flavour: string) =>
nearest(blockId, block => block.get('sys:flavour') === flavour);
let rootBlockId: string | null = null;
for (const block of blocks.values()) {
const flavour = block.get('sys:flavour')?.toString();
@@ -261,21 +581,40 @@ async function crawlingDocData({
const flavour = block.get('sys:flavour')?.toString();
const parentFlavour = parentBlock?.get('sys:flavour')?.toString();
const noteBlock = nearestByFlavour(blockId, 'affine:note');
// display mode:
// - both: page and edgeless -> fallback to page
// - page: only page -> page
// - edgeless: only edgeless -> edgeless
// - undefined: edgeless (assuming it is a normal element on the edgeless)
let displayMode = noteBlock?.get('prop:displayMode') ?? 'edgeless';
if (displayMode === 'both') {
displayMode = 'page';
}
const noteBlockId: string | undefined = noteBlock
?.get('sys:id')
?.toString();
pushChildren(blockId, block);
const commonBlockProps = {
docId,
flavour,
blockId,
yblock: block,
additional: { displayMode, noteBlockId },
};
if (flavour === 'affine:page') {
docTitle = block.get('prop:title').toString();
blockDocuments.push({
docId,
flavour,
blockId,
...commonBlockProps,
content: docTitle,
yblock: block,
});
}
if (
} else if (
flavour === 'affine:paragraph' ||
flavour === 'affine:list' ||
flavour === 'affine:code'
@@ -313,9 +652,7 @@ async function crawlingDocData({
: undefined;
blockDocuments.push({
docId,
flavour,
blockId,
...commonBlockProps,
content: text.toString(),
...refs.reduce<{ refDocId: string[]; ref: string[] }>(
(prev, curr) => {
@@ -327,17 +664,14 @@ async function crawlingDocData({
),
parentFlavour,
parentBlockId,
additional: { databaseName },
yblock: block,
additional: { ...commonBlockProps.additional, databaseName },
});
if (summaryLenNeeded > 0) {
summary += text.toString();
summaryLenNeeded -= text.length;
}
}
if (
} else if (
flavour === 'affine:embed-linked-doc' ||
flavour === 'affine:embed-synced-doc'
) {
@@ -346,34 +680,27 @@ async function crawlingDocData({
// reference info
const params = block.get('prop:params') ?? {};
blockDocuments.push({
docId,
flavour,
blockId,
...commonBlockProps,
refDocId: [pageId],
ref: [JSON.stringify({ docId: pageId, ...params })],
parentFlavour,
parentBlockId,
yblock: block,
});
}
}
if (flavour === 'affine:attachment' || flavour === 'affine:image') {
} else if (
flavour === 'affine:attachment' ||
flavour === 'affine:image'
) {
const blobId = block.get('prop:sourceId');
if (typeof blobId === 'string') {
blockDocuments.push({
docId,
flavour,
blockId,
...commonBlockProps,
blob: [blobId],
parentFlavour,
parentBlockId,
yblock: block,
});
}
}
if (flavour === 'affine:surface') {
} else if (flavour === 'affine:surface') {
const texts = [];
const elementsObj = block.get('prop:elements');
@@ -403,17 +730,12 @@ async function crawlingDocData({
}
blockDocuments.push({
docId,
flavour,
blockId,
...commonBlockProps,
content: texts,
parentFlavour,
parentBlockId,
yblock: block,
});
}
if (flavour === 'affine:database') {
} else if (flavour === 'affine:database') {
const texts = [];
const columnsObj = block.get('prop:columns');
const databaseTitle = block.get('prop:title');
@@ -450,11 +772,21 @@ async function crawlingDocData({
}
blockDocuments.push({
docId,
flavour,
blockId,
...commonBlockProps,
content: texts,
yblock: block,
additional: {
...commonBlockProps.additional,
databaseName: databaseTitle?.toString(),
},
});
} else if (flavour === 'affine:latex') {
blockDocuments.push({
...commonBlockProps,
content: block.get('prop:latex')?.toString() ?? '',
});
} else if (bookmarkFlavours.has(flavour)) {
blockDocuments.push({
...commonBlockProps,
});
}
}
@@ -464,15 +796,28 @@ async function crawlingDocData({
const TARGET_PREVIEW_CHARACTER = 500;
const TARGET_PREVIOUS_BLOCK = 1;
const TARGET_FOLLOW_BLOCK = 4;
for (let i = 0; i < blockDocuments.length; i++) {
const block = blockDocuments[i];
if (block.ref) {
for (const block of blockDocuments) {
if (block.ref?.length) {
const target = block;
// should only generate the markdown preview belong to the same affine:note
const noteBlock = nearestByFlavour(block.blockId, 'affine:note');
const sameNoteBlocks = noteBlock
? blockDocuments.filter(
candidate =>
nearestByFlavour(candidate.blockId, 'affine:note') === noteBlock
)
: [];
// only generate markdown preview for reference blocks
let previewText = (await generateMarkdownPreview(block)) ?? '';
let previewText = (await generateMarkdownPreview(target)) ?? '';
let previousBlock = 0;
let followBlock = 0;
let previousIndex = i;
let followIndex = i;
let previousIndex = sameNoteBlocks.findIndex(
block => block.blockId === target.blockId
);
let followIndex = previousIndex;
while (
!(
@@ -480,14 +825,14 @@ async function crawlingDocData({
previewText.length > TARGET_PREVIEW_CHARACTER || // stop if preview text reaches the limit
((previousBlock >= TARGET_PREVIOUS_BLOCK || previousIndex < 0) &&
(followBlock >= TARGET_FOLLOW_BLOCK ||
followIndex >= blockDocuments.length))
followIndex >= sameNoteBlocks.length))
) // stop if no more blocks, or preview block reaches the limit
)
) {
if (previousBlock < TARGET_PREVIOUS_BLOCK) {
previousIndex--;
const block =
previousIndex >= 0 ? blockDocuments.at(previousIndex) : null;
previousIndex >= 0 ? sameNoteBlocks.at(previousIndex) : null;
const markdown = block
? await generateMarkdownPreview(block)
: null;
@@ -497,14 +842,14 @@ async function crawlingDocData({
markdown
) /* A small hack to skip blocks with the same content */
) {
previewText = markdown + previewText;
previewText = markdown + '\n' + previewText;
previousBlock++;
}
}
if (followBlock < TARGET_FOLLOW_BLOCK) {
followIndex++;
const block = blockDocuments.at(followIndex);
const block = sameNoteBlocks.at(followIndex);
const markdown = block
? await generateMarkdownPreview(block)
: null;
@@ -514,13 +859,13 @@ async function crawlingDocData({
markdown
) /* A small hack to skip blocks with the same content */
) {
previewText = previewText + markdown;
previewText = previewText + '\n' + markdown;
followBlock++;
}
}
}
block.markdownPreview = previewText;
block.markdownPreview = unindentMarkdown(previewText);
}
}
// #endregion

View File

@@ -30,12 +30,14 @@ export type WorkerInput =
| {
type: 'rootDoc';
rootDocBuffer: Uint8Array;
rootDocId: string;
allIndexedDocs: string[];
reindexAll?: boolean;
}
| {
type: 'doc';
storageDocId: string;
rootDocId: string;
rootDocBuffer: Uint8Array;
docBuffer: Uint8Array;
};