Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-02 02:00:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): add markdown preview for backlinks (#8883)

Browse Source
This commit is contained in:
EYHN
2024-12-02 03:42:07 +00:00
parent 11b453f4d8
commit cee5d02f71
12 changed files with 292 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch
+13
View File
@@ -0,0 +1,13 @@
diff --git a/package.json b/package.json
index 5fef2811aa86f3f1f8228daef7d867863e71db72..b795fbd2a0e1cba0b6389ff051220f4e3c52fc13 100644
--- a/package.json
+++ b/package.json
@@ -34,7 +34,7 @@
"deno": "./index.js",
"react-native": "./index.js",
"worker": "./index.js",
- "browser": "./index.dom.js",
+ "browser": "./index.js",
"default": "./index.js"
}
},
package.json
+2 -1
View File
@@ -156,6 +156,7 @@
"which-typed-array": "npm:@nolyfill/which-typed-array@latest",
"macos-alias": "npm:@napi-rs/macos-alias@0.0.4",
"fs-xattr": "npm:@napi-rs/xattr@latest",
"vite": "6.0.1"
"vite": "6.0.1",
"decode-named-character-reference@npm:^1.0.0": "patch:decode-named-character-reference@npm%3A1.0.2#~/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch"
}
}
packages/common/infra/src/sync/indexer/document.ts
+1 -1
View File
@@ -40,7 +40,7 @@ export class Document<S extends Schema = any> {
}
} else {
for (const key in map) {
if (map[key] === undefined) {
if (map[key] === undefined || map[key] === null) {
continue;
}
doc.insert(key, map[key]);
packages/frontend/component/src/ui/property/property.tsx
+1 -1
View File
@@ -39,7 +39,7 @@ export const PropertyCollapsibleSection = forwardRef<
collapsed?: boolean;
onCollapseChange?: (collapsed: boolean) => void;
}> &
HTMLProps<HTMLDivElement>
Omit<HTMLProps<HTMLDivElement>, 'title'>
>(
(
{
packages/frontend/core/src/components/blocksuite/block-suite-editor/bi-directional-link-panel.tsx
+9 -4
View File
@@ -5,7 +5,7 @@ import {
} from '@affine/core/modules/doc-link';
import { useI18n } from '@affine/i18n';
import { LiveData, useLiveData, useServices } from '@toeverything/infra';
import { useCallback, useState } from 'react';
import { Fragment, useCallback, useState } from 'react';
import { AffinePageReference } from '../../affine/reference-link';
import * as styles from './bi-directional-link-panel.css';
@@ -52,9 +52,14 @@ export const BiDirectionalLinkPanel = () => {
{t['com.affine.page-properties.backlinks']()} · {backlinks.length}
</div>
{backlinks.map(link => (
<div key={link.docId} className={styles.link}>
<AffinePageReference key={link.docId} pageId={link.docId} />
</div>
<Fragment key={link.docId}>
<div className={styles.link}>
<AffinePageReference key={link.docId} pageId={link.docId} />
</div>
<br />
<pre style={{ opacity: 0.5 }}>{link.markdownPreview}</pre>
<br />
</Fragment>
))}
</div>
<div className={styles.linksContainer}>
packages/frontend/core/src/modules/doc-info/views/database-properties/doc-database-backlink-info.tsx
-1
View File
@@ -122,7 +122,6 @@ const DatabaseBacklinkRow = ({
return (
<PropertyCollapsibleSection
// @ts-expect-error fix type
title={
<span className={styles.databaseNameWrapper}>
<span className={styles.databaseName}>
packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.ts
+1
View File
@@ -7,6 +7,7 @@ export interface Backlink {
docId: string;
blockId: string;
title: string;
markdownPreview?: string;
}
export class DocBacklinks extends Entity {
packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts
+1 -1
View File
@@ -36,7 +36,7 @@ export class DocsIndexer extends Entity {
/**
* increase this number to re-index all docs
*/
static INDEXER_VERSION = 6;
static INDEXER_VERSION = 10;
private readonly jobQueue: JobQueue<IndexerJobPayload> =
new IndexedDBJobQueue<IndexerJobPayload>(
packages/frontend/core/src/modules/docs-search/schema.ts
+1
View File
@@ -28,6 +28,7 @@ export const blockIndexSchema = defineSchema({
// additional info
// { "databaseName": "xxx" }
additional: { type: 'String', index: false },
markdownPreview: { type: 'String', index: false },
});
export type BlockIndexSchema = typeof blockIndexSchema;
packages/frontend/core/src/modules/docs-search/services/docs-search.ts
+7 -1
View File
@@ -478,7 +478,7 @@ export class DocsSearchService extends Service {
'docId',
{
hits: {
fields: ['docId', 'blockId'],
fields: ['docId', 'blockId', 'markdownPreview'],
pagination: {
limit: 1,
},
@@ -499,10 +499,16 @@ export class DocsSearchService extends Service {
const title =
docData.find(doc => doc.id === bucket.key)?.get('title') ?? '';
const blockId = bucket.hits.nodes[0]?.fields.blockId ?? '';
const markdownPreview =
bucket.hits.nodes[0]?.fields.markdownPreview ?? '';
return {
docId: bucket.key,
blockId: typeof blockId === 'string' ? blockId : blockId[0],
title: typeof title === 'string' ? title : title[0],
markdownPreview:
typeof markdownPreview === 'string'
? markdownPreview
: markdownPreview[0],
};
});
});
packages/frontend/core/src/modules/docs-search/worker/in-worker.ts
+246 -72
View File
@@ -1,6 +1,16 @@
import type { AffineTextAttributes } from '@blocksuite/affine/blocks';
import {
type AffineTextAttributes,
MarkdownAdapter,
} from '@blocksuite/affine/blocks';
import {
createYProxy,
DocCollection,
type DraftModel,
Job,
type YBlock,
} from '@blocksuite/affine/store';
import type { DeltaInsert } from '@blocksuite/inline';
import { Document } from '@toeverything/infra';
import { Document, getAFFiNEWorkspaceSchema } from '@toeverything/infra';
import { toHexString } from 'lib0/buffer.js';
import { digest as lib0Digest } from 'lib0/hash/sha256';
import { difference, uniq } from 'lodash-es';
@@ -20,6 +30,8 @@ import type {
WorkerOutput,
} from './types';
const blocksuiteSchema = getAFFiNEWorkspaceSchema();
const LRU_CACHE_SIZE = 5;
// lru cache for ydoc instances, last used at the end of the array
@@ -61,6 +73,92 @@ async function getOrCreateCachedYDoc(data: Uint8Array) {
}
}
function yblockToDraftModal(yblock: YBlock): DraftModel | null {
const flavour = yblock.get('sys:flavour');
const blockSchema = blocksuiteSchema.flavourSchemaMap.get(flavour);
if (!blockSchema) {
return null;
}
const keys = Array.from(yblock.keys())
.filter(key => key.startsWith('prop:'))
.map(key => key.substring(5));
const props = Object.fromEntries(
keys.map(key => [key, createYProxy(yblock.get(`prop:${key}`))])
);
return {
...props,
id: yblock.get('sys:id'),
flavour,
children: [],
role: blockSchema.model.role,
version: (yblock.get('sys:version') as number) ?? blockSchema.version,
keys: Array.from(yblock.keys())
.filter(key => key.startsWith('prop:'))
.map(key => key.substring(5)),
};
}
const markdownAdapter = new MarkdownAdapter(
new Job({
collection: new DocCollection({
id: 'indexer',
schema: blocksuiteSchema,
}),
})
);
interface BlockDocumentInfo {
docId: string;
blockId: string;
content?: string | string[];
flavour: string;
blob?: string[];
refDocId?: string[];
ref?: string[];
parentFlavour?: string;
parentBlockId?: string;
additional?: { databaseName?: string };
yblock: YMap<any>;
markdownPreview?: string;
}
const markdownPreviewCache = new WeakMap<BlockDocumentInfo, string | null>();
const generateMarkdownPreview = async (block: BlockDocumentInfo) => {
if (markdownPreviewCache.has(block)) {
return markdownPreviewCache.get(block);
}
const flavour = block.flavour;
let markdown: string | null = null;
if (
flavour === 'affine:paragraph' ||
flavour === 'affine:list' ||
flavour === 'affine:code'
) {
const draftModel = yblockToDraftModal(block.yblock);
markdown =
block.parentFlavour === 'affine:database'
? `database · ${block.additional?.databaseName}\n`
: ((draftModel ? await markdownAdapter.fromBlock(draftModel) : null)
?.file ?? null);
}
if (
flavour === 'affine:embed-linked-doc' ||
flavour === 'affine:embed-synced-doc'
) {
markdown = '🔗\n';
}
if (flavour === 'affine:attachment') {
markdown = '📃\n';
}
if (flavour === 'affine:image') {
markdown = '🖼️\n';
}
markdownPreviewCache.set(block, markdown);
return markdown;
};
async function crawlingDocData({
docBuffer,
storageDocId,
@@ -110,7 +208,7 @@ async function crawlingDocData({
let docTitle = '';
let summaryLenNeeded = 1000;
let summary = '';
const blockDocuments: Document<BlockIndexSchema>[] = [];
const blockDocuments: BlockDocumentInfo[] = [];
const blocks = ydoc.getMap<any>('blocks');
@@ -147,6 +245,7 @@ async function crawlingDocData({
}
};
// #region first loop - generate block base info
while (queue.length) {
const next = queue.pop();
if (!next) {
@@ -167,14 +266,13 @@ async function crawlingDocData({
if (flavour === 'affine:page') {
docTitle = block.get('prop:title').toString();
blockDocuments.push(
Document.from(`${docId}:${blockId}`, {
docId,
flavour,
blockId,
content: docTitle,
})
);
blockDocuments.push({
docId,
flavour,
blockId,
content: docTitle,
yblock: block,
});
}
if (
@@ -183,6 +281,7 @@ async function crawlingDocData({
flavour === 'affine:code'
) {
const text = block.get('prop:text') as YText;
if (!text) {
continue;
}
@@ -213,27 +312,24 @@ async function crawlingDocData({
? parentBlock?.get('prop:title')?.toString()
: undefined;
blockDocuments.push(
Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {
docId,
flavour,
blockId,
content: text.toString(),
...refs.reduce<{ refDocId: string[]; ref: string[] }>(
(prev, curr) => {
prev.refDocId.push(curr.refDocId);
prev.ref.push(curr.ref);
return prev;
},
{ refDocId: [], ref: [] }
),
parentFlavour,
parentBlockId,
additional: databaseName
? JSON.stringify({ databaseName })
: undefined,
})
);
blockDocuments.push({
docId,
flavour,
blockId,
content: text.toString(),
...refs.reduce<{ refDocId: string[]; ref: string[] }>(
(prev, curr) => {
prev.refDocId.push(curr.refDocId);
prev.ref.push(curr.ref);
return prev;
},
{ refDocId: [], ref: [] }
),
parentFlavour,
parentBlockId,
additional: { databaseName },
yblock: block,
});
if (summaryLenNeeded > 0) {
summary += text.toString();
@@ -249,33 +345,31 @@ async function crawlingDocData({
if (typeof pageId === 'string') {
// reference info
const params = block.get('prop:params') ?? {};
blockDocuments.push(
Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {
docId,
flavour,
blockId,
refDocId: [pageId],
ref: [JSON.stringify({ docId: pageId, ...params })],
parentFlavour,
parentBlockId,
})
);
blockDocuments.push({
docId,
flavour,
blockId,
refDocId: [pageId],
ref: [JSON.stringify({ docId: pageId, ...params })],
parentFlavour,
parentBlockId,
yblock: block,
});
}
}
if (flavour === 'affine:attachment' || flavour === 'affine:image') {
const blobId = block.get('prop:sourceId');
if (typeof blobId === 'string') {
blockDocuments.push(
Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {
docId,
flavour,
blockId,
blob: [blobId],
parentFlavour,
parentBlockId,
})
);
blockDocuments.push({
docId,
flavour,
blockId,
blob: [blobId],
parentFlavour,
parentBlockId,
yblock: block,
});
}
}
@@ -308,16 +402,15 @@ async function crawlingDocData({
texts.push(text.toString());
}
blockDocuments.push(
Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {
docId,
flavour,
blockId,
content: texts,
parentFlavour,
parentBlockId,
})
);
blockDocuments.push({
docId,
flavour,
blockId,
content: texts,
parentFlavour,
parentBlockId,
yblock: block,
});
}
if (flavour === 'affine:database') {
@@ -356,16 +449,81 @@ async function crawlingDocData({
}
}
blockDocuments.push(
Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {
docId,
flavour,
blockId,
content: texts,
})
);
blockDocuments.push({
docId,
flavour,
blockId,
content: texts,
yblock: block,
});
}
}
// #endregion
// #region second loop - generate markdown preview
const TARGET_PREVIEW_CHARACTER = 500;
const TARGET_PREVIOUS_BLOCK = 1;
const TARGET_FOLLOW_BLOCK = 4;
for (let i = 0; i < blockDocuments.length; i++) {
const block = blockDocuments[i];
if (block.ref) {
// only generate markdown preview for reference blocks
let previewText = (await generateMarkdownPreview(block)) ?? '';
let previousBlock = 0;
let followBlock = 0;
let previousIndex = i;
let followIndex = i;
while (
!(
(
previewText.length > TARGET_PREVIEW_CHARACTER || // stop if preview text reaches the limit
((previousBlock >= TARGET_PREVIOUS_BLOCK || previousIndex < 0) &&
(followBlock >= TARGET_FOLLOW_BLOCK ||
followIndex >= blockDocuments.length))
) // stop if no more blocks, or preview block reaches the limit
)
) {
if (previousBlock < TARGET_PREVIOUS_BLOCK) {
previousIndex--;
const block =
previousIndex >= 0 ? blockDocuments.at(previousIndex) : null;
const markdown = block
? await generateMarkdownPreview(block)
: null;
if (
markdown &&
!previewText.startsWith(
markdown
) /* A small hack to skip blocks with the same content */
) {
previewText = markdown + previewText;
previousBlock++;
}
}
if (followBlock < TARGET_FOLLOW_BLOCK) {
followIndex++;
const block = blockDocuments.at(followIndex);
const markdown = block
? await generateMarkdownPreview(block)
: null;
if (
markdown &&
!previewText.endsWith(
markdown
) /* A small hack to skip blocks with the same content */
) {
previewText = previewText + markdown;
followBlock++;
}
}
}
block.markdownPreview = previewText;
}
}
// #endregion
return {
addedDoc: [
@@ -375,7 +533,23 @@ async function crawlingDocData({
title: docTitle,
summary,
}),
blocks: blockDocuments,
blocks: blockDocuments.map(block =>
Document.from<BlockIndexSchema>(`${docId}:${block.blockId}`, {
docId: block.docId,
blockId: block.blockId,
content: block.content,
flavour: block.flavour,
blob: block.blob,
refDocId: block.refDocId,
ref: block.ref,
parentFlavour: block.parentFlavour,
parentBlockId: block.parentBlockId,
additional: block.additional
? JSON.stringify(block.additional)
: undefined,
markdownPreview: block.markdownPreview,
})
),
},
],
};
yarn.lock
+10 -1
View File
@@ -18060,7 +18060,7 @@ __metadata:
languageName: node
linkType: hard
"decode-named-character-reference@npm:^1.0.0":
"decode-named-character-reference@npm:1.0.2":
version: 1.0.2
resolution: "decode-named-character-reference@npm:1.0.2"
dependencies:
@@ -18069,6 +18069,15 @@ __metadata:
languageName: node
linkType: hard
"decode-named-character-reference@patch:decode-named-character-reference@npm%3A1.0.2#~/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch":
version: 1.0.2
resolution: "decode-named-character-reference@patch:decode-named-character-reference@npm%3A1.0.2#~/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch::version=1.0.2&hash=2c2160"
dependencies:
character-entities: "npm:^2.0.0"
checksum: 10/bd6e42b2cc162f55351a34fa5123cee0bfdebd983aa3690e5347c9ec23ce8b7f701fce0f77099b3a51eb1451b6a17e66f41d69e7cfc482ea3a0a1e38fe2442bf
languageName: node
linkType: hard
"decode-uri-component@npm:^0.2.2":
version: 0.2.2
resolution: "decode-uri-component@npm:0.2.2"