mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-25 22:38:56 +08:00
feat(nbstore): add indexer storage (#10953)
This commit is contained in:
EYHN
@@ -420,7 +420,11 @@ class BlobSyncPeerStatus {
|
||||
return () => {
|
||||
dispose.unsubscribe();
|
||||
};
|
||||
});
|
||||
}).pipe(
|
||||
share({
|
||||
connector: () => new ReplaySubject(1),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
private readonly statusUpdatedSubject$ = new Subject<string | true>();
|
||||
|
||||
@@ -4,10 +4,12 @@ import type { SpaceStorage } from '../storage';
|
||||
import { AwarenessSyncImpl } from './awareness';
|
||||
import { BlobSyncImpl } from './blob';
|
||||
import { DocSyncImpl, type DocSyncState } from './doc';
|
||||
import { IndexerSyncImpl } from './indexer';
|
||||
import type { PeerStorageOptions } from './types';
|
||||
|
||||
export type { BlobSyncState } from './blob';
|
||||
export type { DocSyncDocState, DocSyncState } from './doc';
|
||||
export type { IndexerDocSyncState, IndexerSyncState } from './indexer';
|
||||
|
||||
export interface SyncState {
|
||||
doc?: DocSyncState;
|
||||
@@ -17,6 +19,7 @@ export class Sync {
|
||||
readonly doc: DocSyncImpl;
|
||||
readonly blob: BlobSyncImpl;
|
||||
readonly awareness: AwarenessSyncImpl;
|
||||
readonly indexer: IndexerSyncImpl;
|
||||
|
||||
readonly state$: Observable<SyncState>;
|
||||
|
||||
@@ -26,6 +29,8 @@ export class Sync {
|
||||
const docSync = storages.local.get('docSync');
|
||||
const blobSync = storages.local.get('blobSync');
|
||||
const awareness = storages.local.get('awareness');
|
||||
const indexer = storages.local.get('indexer');
|
||||
const indexerSync = storages.local.get('indexerSync');
|
||||
|
||||
this.doc = new DocSyncImpl(
|
||||
{
|
||||
@@ -60,6 +65,7 @@ export class Sync {
|
||||
])
|
||||
),
|
||||
});
|
||||
this.indexer = new IndexerSyncImpl(doc, indexer, indexerSync);
|
||||
|
||||
this.state$ = this.doc.state$.pipe(map(doc => ({ doc })));
|
||||
}
|
||||
@@ -67,10 +73,12 @@ export class Sync {
|
||||
start() {
|
||||
this.doc?.start();
|
||||
this.blob?.start();
|
||||
this.indexer?.start();
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.doc?.stop();
|
||||
this.blob?.stop();
|
||||
this.indexer?.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,862 @@
|
||||
import { defaultBlockMarkdownAdapterMatchers } from '@blocksuite/affine/adapters';
|
||||
import { Container } from '@blocksuite/affine/global/di';
|
||||
import {
|
||||
InlineDeltaToMarkdownAdapterExtensions,
|
||||
MarkdownInlineToDeltaAdapterExtensions,
|
||||
} from '@blocksuite/affine/inlines/preset';
|
||||
import type {
|
||||
AttachmentBlockModel,
|
||||
BookmarkBlockModel,
|
||||
EmbedBlockModel,
|
||||
ImageBlockModel,
|
||||
TableBlockModel,
|
||||
} from '@blocksuite/affine/model';
|
||||
import { AffineSchemas } from '@blocksuite/affine/schemas';
|
||||
import { MarkdownAdapter } from '@blocksuite/affine/shared/adapters';
|
||||
import type { AffineTextAttributes } from '@blocksuite/affine/shared/types';
|
||||
import {
|
||||
createYProxy,
|
||||
type DeltaInsert,
|
||||
type DraftModel,
|
||||
Schema,
|
||||
Transformer,
|
||||
type TransformerMiddleware,
|
||||
type YBlock,
|
||||
} from '@blocksuite/affine/store';
|
||||
import { uniq } from 'lodash-es';
|
||||
import {
|
||||
Array as YArray,
|
||||
type Doc as YDoc,
|
||||
Map as YMap,
|
||||
Text as YText,
|
||||
} from 'yjs';
|
||||
|
||||
import { IndexerDocument } from '../../storage';
|
||||
|
||||
const blocksuiteSchema = new Schema();
|
||||
blocksuiteSchema.register([...AffineSchemas]);
|
||||
|
||||
interface BlockDocumentInfo {
|
||||
docId: string;
|
||||
blockId: string;
|
||||
content?: string | string[];
|
||||
flavour: string;
|
||||
blob?: string[];
|
||||
refDocId?: string[];
|
||||
ref?: string[];
|
||||
parentFlavour?: string;
|
||||
parentBlockId?: string;
|
||||
additional?: {
|
||||
databaseName?: string;
|
||||
displayMode?: string;
|
||||
noteBlockId?: string;
|
||||
};
|
||||
yblock: YMap<any>;
|
||||
markdownPreview?: string;
|
||||
}
|
||||
|
||||
const bookmarkFlavours = new Set([
|
||||
'affine:bookmark',
|
||||
'affine:embed-youtube',
|
||||
'affine:embed-figma',
|
||||
'affine:embed-github',
|
||||
'affine:embed-loom',
|
||||
]);
|
||||
|
||||
function generateMarkdownPreviewBuilder(
|
||||
yRootDoc: YDoc,
|
||||
workspaceId: string,
|
||||
blocks: BlockDocumentInfo[]
|
||||
) {
|
||||
function yblockToDraftModal(yblock: YBlock): DraftModel | null {
|
||||
const flavour = yblock.get('sys:flavour') as string;
|
||||
const blockSchema = blocksuiteSchema.flavourSchemaMap.get(flavour);
|
||||
if (!blockSchema) {
|
||||
return null;
|
||||
}
|
||||
const keys = Array.from(yblock.keys())
|
||||
.filter(key => key.startsWith('prop:'))
|
||||
.map(key => key.substring(5));
|
||||
|
||||
const props = Object.fromEntries(
|
||||
keys.map(key => [key, createYProxy(yblock.get(`prop:${key}`))])
|
||||
);
|
||||
|
||||
return {
|
||||
props,
|
||||
id: yblock.get('sys:id') as string,
|
||||
flavour,
|
||||
children: [],
|
||||
role: blockSchema.model.role,
|
||||
version: (yblock.get('sys:version') as number) ?? blockSchema.version,
|
||||
keys: Array.from(yblock.keys())
|
||||
.filter(key => key.startsWith('prop:'))
|
||||
.map(key => key.substring(5)),
|
||||
} as unknown as DraftModel;
|
||||
}
|
||||
|
||||
const titleMiddleware: TransformerMiddleware = ({ adapterConfigs }) => {
|
||||
const pages = yRootDoc.getMap('meta').get('pages');
|
||||
if (!(pages instanceof YArray)) {
|
||||
return;
|
||||
}
|
||||
for (const meta of pages.toArray()) {
|
||||
adapterConfigs.set(
|
||||
'title:' + meta.get('id'),
|
||||
meta.get('title')?.toString() ?? 'Untitled'
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const baseUrl = `/workspace/${workspaceId}`;
|
||||
|
||||
function getDocLink(docId: string, blockId: string) {
|
||||
const searchParams = new URLSearchParams();
|
||||
searchParams.set('blockIds', blockId);
|
||||
return `${baseUrl}/${docId}?${searchParams.toString()}`;
|
||||
}
|
||||
|
||||
const docLinkBaseURLMiddleware: TransformerMiddleware = ({
|
||||
adapterConfigs,
|
||||
}) => {
|
||||
adapterConfigs.set('docLinkBaseUrl', baseUrl);
|
||||
};
|
||||
|
||||
const container = new Container();
|
||||
[
|
||||
...MarkdownInlineToDeltaAdapterExtensions,
|
||||
...defaultBlockMarkdownAdapterMatchers,
|
||||
...InlineDeltaToMarkdownAdapterExtensions,
|
||||
].forEach(ext => {
|
||||
ext.setup(container);
|
||||
});
|
||||
|
||||
const provider = container.provider();
|
||||
const markdownAdapter = new MarkdownAdapter(
|
||||
new Transformer({
|
||||
schema: blocksuiteSchema,
|
||||
blobCRUD: {
|
||||
delete: () => Promise.resolve(),
|
||||
get: () => Promise.resolve(null),
|
||||
list: () => Promise.resolve([]),
|
||||
set: () => Promise.resolve(''),
|
||||
},
|
||||
docCRUD: {
|
||||
create: () => {
|
||||
throw new Error('Not implemented');
|
||||
},
|
||||
get: () => null,
|
||||
delete: () => {},
|
||||
},
|
||||
middlewares: [docLinkBaseURLMiddleware, titleMiddleware],
|
||||
}),
|
||||
provider
|
||||
);
|
||||
|
||||
const markdownPreviewCache = new WeakMap<BlockDocumentInfo, string | null>();
|
||||
|
||||
function trimCodeBlock(markdown: string) {
|
||||
const lines = markdown.split('\n').filter(line => line.trim() !== '');
|
||||
if (lines.length > 5) {
|
||||
return [...lines.slice(0, 4), '...', lines.at(-1), ''].join('\n');
|
||||
}
|
||||
return [...lines, ''].join('\n');
|
||||
}
|
||||
|
||||
function trimParagraph(markdown: string) {
|
||||
const lines = markdown.split('\n').filter(line => line.trim() !== '');
|
||||
|
||||
if (lines.length > 3) {
|
||||
return [...lines.slice(0, 3), '...', lines.at(-1), ''].join('\n');
|
||||
}
|
||||
|
||||
return [...lines, ''].join('\n');
|
||||
}
|
||||
|
||||
function getListDepth(block: BlockDocumentInfo) {
|
||||
let parentBlockCount = 0;
|
||||
let currentBlock: BlockDocumentInfo | undefined = block;
|
||||
do {
|
||||
currentBlock = blocks.find(
|
||||
b => b.blockId === currentBlock?.parentBlockId
|
||||
);
|
||||
|
||||
// reach the root block. do not count it.
|
||||
if (!currentBlock || currentBlock.flavour !== 'affine:list') {
|
||||
break;
|
||||
}
|
||||
parentBlockCount++;
|
||||
} while (currentBlock);
|
||||
return parentBlockCount;
|
||||
}
|
||||
|
||||
// only works for list block
|
||||
function indentMarkdown(markdown: string, depth: number) {
|
||||
if (depth <= 0) {
|
||||
return markdown;
|
||||
}
|
||||
|
||||
return (
|
||||
markdown
|
||||
.split('\n')
|
||||
.map(line => ' '.repeat(depth) + line)
|
||||
.join('\n') + '\n'
|
||||
);
|
||||
}
|
||||
|
||||
const generateDatabaseMarkdownPreview = (block: BlockDocumentInfo) => {
|
||||
const isDatabaseBlock = (block: BlockDocumentInfo) => {
|
||||
return block.flavour === 'affine:database';
|
||||
};
|
||||
|
||||
const model = yblockToDraftModal(block.yblock);
|
||||
|
||||
if (!model) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let dbBlock: BlockDocumentInfo | null = null;
|
||||
|
||||
if (isDatabaseBlock(block)) {
|
||||
dbBlock = block;
|
||||
} else {
|
||||
const parentBlock = blocks.find(b => b.blockId === block.parentBlockId);
|
||||
|
||||
if (parentBlock && isDatabaseBlock(parentBlock)) {
|
||||
dbBlock = parentBlock;
|
||||
}
|
||||
}
|
||||
|
||||
if (!dbBlock) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const url = getDocLink(block.docId, dbBlock.blockId);
|
||||
const title = dbBlock.additional?.databaseName;
|
||||
|
||||
return `[database · ${title || 'Untitled'}][](${url})\n`;
|
||||
};
|
||||
|
||||
const generateImageMarkdownPreview = (block: BlockDocumentInfo) => {
|
||||
const isImageModel = (
|
||||
model: DraftModel | null
|
||||
): model is DraftModel<ImageBlockModel> => {
|
||||
return model?.flavour === 'affine:image';
|
||||
};
|
||||
|
||||
const model = yblockToDraftModal(block.yblock);
|
||||
|
||||
if (!isImageModel(model)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const info = ['an image block'];
|
||||
|
||||
if (model.props.sourceId) {
|
||||
info.push(`file id ${model.props.sourceId}`);
|
||||
}
|
||||
|
||||
if (model.props.caption) {
|
||||
info.push(`with caption ${model.props.caption}`);
|
||||
}
|
||||
|
||||
return info.join(', ') + '\n';
|
||||
};
|
||||
|
||||
const generateEmbedMarkdownPreview = (block: BlockDocumentInfo) => {
|
||||
const isEmbedModel = (
|
||||
model: DraftModel | null
|
||||
): model is DraftModel<EmbedBlockModel> => {
|
||||
return (
|
||||
model?.flavour === 'affine:embed-linked-doc' ||
|
||||
model?.flavour === 'affine:embed-synced-doc'
|
||||
);
|
||||
};
|
||||
|
||||
const draftModel = yblockToDraftModal(block.yblock);
|
||||
if (!isEmbedModel(draftModel)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const url = getDocLink(block.docId, draftModel.id);
|
||||
|
||||
return `[](${url})\n`;
|
||||
};
|
||||
|
||||
const generateLatexMarkdownPreview = (block: BlockDocumentInfo) => {
|
||||
let content =
|
||||
typeof block.content === 'string'
|
||||
? block.content.trim()
|
||||
: block.content?.join('').trim();
|
||||
|
||||
content = content?.split('\n').join(' ') ?? '';
|
||||
|
||||
return `LaTeX, with value ${content}\n`;
|
||||
};
|
||||
|
||||
const generateBookmarkMarkdownPreview = (block: BlockDocumentInfo) => {
|
||||
const isBookmarkModel = (
|
||||
model: DraftModel | null
|
||||
): model is DraftModel<BookmarkBlockModel> => {
|
||||
return bookmarkFlavours.has(model?.flavour ?? '');
|
||||
};
|
||||
|
||||
const draftModel = yblockToDraftModal(block.yblock);
|
||||
if (!isBookmarkModel(draftModel)) {
|
||||
return null;
|
||||
}
|
||||
const title = draftModel.props.title;
|
||||
const url = draftModel.props.url;
|
||||
return `[${title}](${url})\n`;
|
||||
};
|
||||
|
||||
const generateAttachmentMarkdownPreview = (block: BlockDocumentInfo) => {
|
||||
const isAttachmentModel = (
|
||||
model: DraftModel | null
|
||||
): model is DraftModel<AttachmentBlockModel> => {
|
||||
return model?.flavour === 'affine:attachment';
|
||||
};
|
||||
|
||||
const draftModel = yblockToDraftModal(block.yblock);
|
||||
if (!isAttachmentModel(draftModel)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return `[${draftModel.props.name}](${draftModel.props.sourceId})\n`;
|
||||
};
|
||||
|
||||
const generateTableMarkdownPreview = (block: BlockDocumentInfo) => {
|
||||
const isTableModel = (
|
||||
model: DraftModel | null
|
||||
): model is DraftModel<TableBlockModel> => {
|
||||
return model?.flavour === 'affine:table';
|
||||
};
|
||||
|
||||
const draftModel = yblockToDraftModal(block.yblock);
|
||||
if (!isTableModel(draftModel)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const url = getDocLink(block.docId, draftModel.id);
|
||||
|
||||
return `[table][](${url})\n`;
|
||||
};
|
||||
|
||||
const generateMarkdownPreview = async (block: BlockDocumentInfo) => {
|
||||
if (markdownPreviewCache.has(block)) {
|
||||
return markdownPreviewCache.get(block);
|
||||
}
|
||||
const flavour = block.flavour;
|
||||
let markdown: string | null = null;
|
||||
|
||||
if (
|
||||
flavour === 'affine:paragraph' ||
|
||||
flavour === 'affine:list' ||
|
||||
flavour === 'affine:code'
|
||||
) {
|
||||
const draftModel = yblockToDraftModal(block.yblock);
|
||||
markdown =
|
||||
block.parentFlavour === 'affine:database'
|
||||
? generateDatabaseMarkdownPreview(block)
|
||||
: ((draftModel ? await markdownAdapter.fromBlock(draftModel) : null)
|
||||
?.file ?? null);
|
||||
|
||||
if (markdown) {
|
||||
if (flavour === 'affine:code') {
|
||||
markdown = trimCodeBlock(markdown);
|
||||
} else if (flavour === 'affine:paragraph') {
|
||||
markdown = trimParagraph(markdown);
|
||||
}
|
||||
}
|
||||
} else if (flavour === 'affine:database') {
|
||||
markdown = generateDatabaseMarkdownPreview(block);
|
||||
} else if (
|
||||
flavour === 'affine:embed-linked-doc' ||
|
||||
flavour === 'affine:embed-synced-doc'
|
||||
) {
|
||||
markdown = generateEmbedMarkdownPreview(block);
|
||||
} else if (flavour === 'affine:attachment') {
|
||||
markdown = generateAttachmentMarkdownPreview(block);
|
||||
} else if (flavour === 'affine:image') {
|
||||
markdown = generateImageMarkdownPreview(block);
|
||||
} else if (flavour === 'affine:surface' || flavour === 'affine:page') {
|
||||
// skip
|
||||
} else if (flavour === 'affine:latex') {
|
||||
markdown = generateLatexMarkdownPreview(block);
|
||||
} else if (bookmarkFlavours.has(flavour)) {
|
||||
markdown = generateBookmarkMarkdownPreview(block);
|
||||
} else if (flavour === 'affine:table') {
|
||||
markdown = generateTableMarkdownPreview(block);
|
||||
} else {
|
||||
console.warn(`unknown flavour: ${flavour}`);
|
||||
}
|
||||
|
||||
if (markdown && flavour === 'affine:list') {
|
||||
const blockDepth = getListDepth(block);
|
||||
markdown = indentMarkdown(markdown, Math.max(0, blockDepth));
|
||||
}
|
||||
|
||||
markdownPreviewCache.set(block, markdown);
|
||||
return markdown;
|
||||
};
|
||||
|
||||
return generateMarkdownPreview;
|
||||
}
|
||||
|
||||
// remove the indent of the first line of list
|
||||
// e.g.,
|
||||
// ```
|
||||
// - list item 1
|
||||
// - list item 2
|
||||
// ```
|
||||
// becomes
|
||||
// ```
|
||||
// - list item 1
|
||||
// - list item 2
|
||||
// ```
|
||||
function unindentMarkdown(markdown: string) {
|
||||
const lines = markdown.split('\n');
|
||||
const res: string[] = [];
|
||||
let firstListFound = false;
|
||||
let baseIndent = 0;
|
||||
|
||||
for (let current of lines) {
|
||||
const indent = current.match(/^\s*/)?.[0]?.length ?? 0;
|
||||
|
||||
if (indent > 0) {
|
||||
if (!firstListFound) {
|
||||
// For the first list item, remove all indentation
|
||||
firstListFound = true;
|
||||
baseIndent = indent;
|
||||
current = current.trimStart();
|
||||
} else {
|
||||
// For subsequent list items, maintain relative indentation
|
||||
current =
|
||||
' '.repeat(Math.max(0, indent - baseIndent)) + current.trimStart();
|
||||
}
|
||||
}
|
||||
|
||||
res.push(current);
|
||||
}
|
||||
|
||||
return res.join('\n');
|
||||
}
|
||||
|
||||
export async function crawlingDocData({
|
||||
ydoc,
|
||||
rootYDoc,
|
||||
spaceId,
|
||||
docId,
|
||||
}: {
|
||||
ydoc: YDoc;
|
||||
rootYDoc: YDoc;
|
||||
spaceId: string;
|
||||
docId: string;
|
||||
}): Promise<{
|
||||
blocks: IndexerDocument<'block'>[];
|
||||
preview?: string;
|
||||
}> {
|
||||
let docTitle = '';
|
||||
let summaryLenNeeded = 1000;
|
||||
let summary = '';
|
||||
const blockDocuments: BlockDocumentInfo[] = [];
|
||||
|
||||
const generateMarkdownPreview = generateMarkdownPreviewBuilder(
|
||||
rootYDoc,
|
||||
spaceId,
|
||||
blockDocuments
|
||||
);
|
||||
|
||||
const blocks = ydoc.getMap<any>('blocks');
|
||||
|
||||
// build a parent map for quick lookup
|
||||
// for each block, record its parent id
|
||||
const parentMap: Record<string, string | null> = {};
|
||||
for (const [id, block] of blocks.entries()) {
|
||||
const children = block.get('sys:children') as YArray<string> | undefined;
|
||||
if (children instanceof YArray && children.length) {
|
||||
for (const child of children) {
|
||||
parentMap[child] = id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (blocks.size === 0) {
|
||||
return { blocks: [] };
|
||||
}
|
||||
|
||||
// find the nearest block that satisfies the predicate
|
||||
const nearest = (
|
||||
blockId: string,
|
||||
predicate: (block: YMap<any>) => boolean
|
||||
) => {
|
||||
let current: string | null = blockId;
|
||||
while (current) {
|
||||
const block = blocks.get(current);
|
||||
if (block && predicate(block)) {
|
||||
return block;
|
||||
}
|
||||
current = parentMap[current] ?? null;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const nearestByFlavour = (blockId: string, flavour: string) =>
|
||||
nearest(blockId, block => block.get('sys:flavour') === flavour);
|
||||
|
||||
let rootBlockId: string | null = null;
|
||||
for (const block of blocks.values()) {
|
||||
const flavour = block.get('sys:flavour')?.toString();
|
||||
const blockId = block.get('sys:id')?.toString();
|
||||
if (flavour === 'affine:page' && blockId) {
|
||||
rootBlockId = blockId;
|
||||
}
|
||||
}
|
||||
|
||||
if (!rootBlockId) {
|
||||
return { blocks: [] };
|
||||
}
|
||||
|
||||
const queue: { parent?: string; id: string }[] = [{ id: rootBlockId }];
|
||||
const visited = new Set<string>(); // avoid loop
|
||||
|
||||
const pushChildren = (id: string, block: YMap<any>) => {
|
||||
const children = block.get('sys:children');
|
||||
if (children instanceof YArray && children.length) {
|
||||
for (let i = children.length - 1; i >= 0; i--) {
|
||||
const childId = children.get(i);
|
||||
if (childId && !visited.has(childId)) {
|
||||
queue.push({ parent: id, id: childId });
|
||||
visited.add(childId);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// #region first loop - generate block base info
|
||||
while (queue.length) {
|
||||
const next = queue.pop();
|
||||
if (!next) {
|
||||
break;
|
||||
}
|
||||
|
||||
const { parent: parentBlockId, id: blockId } = next;
|
||||
const block = blockId ? blocks.get(blockId) : null;
|
||||
const parentBlock = parentBlockId ? blocks.get(parentBlockId) : null;
|
||||
if (!block) {
|
||||
break;
|
||||
}
|
||||
|
||||
const flavour = block.get('sys:flavour')?.toString();
|
||||
const parentFlavour = parentBlock?.get('sys:flavour')?.toString();
|
||||
const noteBlock = nearestByFlavour(blockId, 'affine:note');
|
||||
|
||||
// display mode:
|
||||
// - both: page and edgeless -> fallback to page
|
||||
// - page: only page -> page
|
||||
// - edgeless: only edgeless -> edgeless
|
||||
// - undefined: edgeless (assuming it is a normal element on the edgeless)
|
||||
let displayMode = noteBlock?.get('prop:displayMode') ?? 'edgeless';
|
||||
|
||||
if (displayMode === 'both') {
|
||||
displayMode = 'page';
|
||||
}
|
||||
|
||||
const noteBlockId: string | undefined = noteBlock
|
||||
?.get('sys:id')
|
||||
?.toString();
|
||||
|
||||
pushChildren(blockId, block);
|
||||
|
||||
const commonBlockProps = {
|
||||
docId: ydoc.guid,
|
||||
flavour,
|
||||
blockId,
|
||||
yblock: block,
|
||||
additional: { displayMode, noteBlockId },
|
||||
};
|
||||
|
||||
if (flavour === 'affine:page') {
|
||||
docTitle = block.get('prop:title').toString();
|
||||
blockDocuments.push({ ...commonBlockProps, content: docTitle });
|
||||
} else if (
|
||||
flavour === 'affine:paragraph' ||
|
||||
flavour === 'affine:list' ||
|
||||
flavour === 'affine:code'
|
||||
) {
|
||||
const text = block.get('prop:text') as YText;
|
||||
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const deltas: DeltaInsert<AffineTextAttributes>[] = text.toDelta();
|
||||
const refs = uniq(
|
||||
deltas
|
||||
.flatMap(delta => {
|
||||
if (
|
||||
delta.attributes &&
|
||||
delta.attributes.reference &&
|
||||
delta.attributes.reference.pageId
|
||||
) {
|
||||
const { pageId: refDocId, params = {} } =
|
||||
delta.attributes.reference;
|
||||
return {
|
||||
refDocId,
|
||||
ref: JSON.stringify({ docId: refDocId, ...params }),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
})
|
||||
.filter(ref => !!ref)
|
||||
);
|
||||
|
||||
const databaseName =
|
||||
flavour === 'affine:paragraph' && parentFlavour === 'affine:database' // if block is a database row
|
||||
? parentBlock?.get('prop:title')?.toString()
|
||||
: undefined;
|
||||
|
||||
blockDocuments.push({
|
||||
...commonBlockProps,
|
||||
content: text.toString(),
|
||||
...refs.reduce<{ refDocId: string[]; ref: string[] }>(
|
||||
(prev, curr) => {
|
||||
prev.refDocId.push(curr.refDocId);
|
||||
prev.ref.push(curr.ref);
|
||||
return prev;
|
||||
},
|
||||
{ refDocId: [], ref: [] }
|
||||
),
|
||||
parentFlavour,
|
||||
parentBlockId,
|
||||
additional: { ...commonBlockProps.additional, databaseName },
|
||||
});
|
||||
|
||||
if (summaryLenNeeded > 0) {
|
||||
summary += text.toString();
|
||||
summaryLenNeeded -= text.length;
|
||||
}
|
||||
} else if (
|
||||
flavour === 'affine:embed-linked-doc' ||
|
||||
flavour === 'affine:embed-synced-doc'
|
||||
) {
|
||||
const pageId = block.get('prop:pageId');
|
||||
if (typeof pageId === 'string') {
|
||||
// reference info
|
||||
const params = block.get('prop:params') ?? {};
|
||||
blockDocuments.push({
|
||||
...commonBlockProps,
|
||||
refDocId: [pageId],
|
||||
ref: [JSON.stringify({ docId: pageId, ...params })],
|
||||
parentFlavour,
|
||||
parentBlockId,
|
||||
});
|
||||
}
|
||||
} else if (flavour === 'affine:attachment' || flavour === 'affine:image') {
|
||||
const blobId = block.get('prop:sourceId');
|
||||
if (typeof blobId === 'string') {
|
||||
blockDocuments.push({
|
||||
...commonBlockProps,
|
||||
blob: [blobId],
|
||||
parentFlavour,
|
||||
parentBlockId,
|
||||
});
|
||||
}
|
||||
} else if (flavour === 'affine:surface') {
|
||||
const texts = [];
|
||||
|
||||
const elementsObj = block.get('prop:elements');
|
||||
if (
|
||||
!(
|
||||
elementsObj instanceof YMap &&
|
||||
elementsObj.get('type') === '$blocksuite:internal:native$'
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const elements = elementsObj.get('value') as YMap<any>;
|
||||
if (!(elements instanceof YMap)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const element of elements.values()) {
|
||||
if (!(element instanceof YMap)) {
|
||||
continue;
|
||||
}
|
||||
const text = element.get('text') as YText;
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
|
||||
texts.push(text.toString());
|
||||
}
|
||||
|
||||
blockDocuments.push({
|
||||
...commonBlockProps,
|
||||
content: texts,
|
||||
parentFlavour,
|
||||
parentBlockId,
|
||||
});
|
||||
} else if (flavour === 'affine:database') {
|
||||
const texts = [];
|
||||
const columnsObj = block.get('prop:columns');
|
||||
const databaseTitle = block.get('prop:title');
|
||||
if (databaseTitle instanceof YText) {
|
||||
texts.push(databaseTitle.toString());
|
||||
}
|
||||
if (columnsObj instanceof YArray) {
|
||||
for (const column of columnsObj) {
|
||||
if (!(column instanceof YMap)) {
|
||||
continue;
|
||||
}
|
||||
if (typeof column.get('name') === 'string') {
|
||||
texts.push(column.get('name'));
|
||||
}
|
||||
|
||||
const data = column.get('data');
|
||||
if (!(data instanceof YMap)) {
|
||||
continue;
|
||||
}
|
||||
const options = data.get('options');
|
||||
if (!(options instanceof YArray)) {
|
||||
continue;
|
||||
}
|
||||
for (const option of options) {
|
||||
if (!(option instanceof YMap)) {
|
||||
continue;
|
||||
}
|
||||
const value = option.get('value');
|
||||
if (typeof value === 'string') {
|
||||
texts.push(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
blockDocuments.push({
|
||||
...commonBlockProps,
|
||||
content: texts,
|
||||
additional: {
|
||||
...commonBlockProps.additional,
|
||||
databaseName: databaseTitle?.toString(),
|
||||
},
|
||||
});
|
||||
} else if (flavour === 'affine:latex') {
|
||||
blockDocuments.push({
|
||||
...commonBlockProps,
|
||||
content: block.get('prop:latex')?.toString() ?? '',
|
||||
});
|
||||
} else if (flavour === 'affine:table') {
|
||||
const contents = Array.from<string>(block.keys())
|
||||
.map(key => {
|
||||
if (key.startsWith('prop:cells.') && key.endsWith('.text')) {
|
||||
return block.get(key)?.toString() ?? '';
|
||||
}
|
||||
return '';
|
||||
})
|
||||
.filter(Boolean);
|
||||
blockDocuments.push({
|
||||
...commonBlockProps,
|
||||
content: contents,
|
||||
});
|
||||
} else if (bookmarkFlavours.has(flavour)) {
|
||||
blockDocuments.push({ ...commonBlockProps });
|
||||
}
|
||||
}
|
||||
// #endregion
|
||||
|
||||
// #region second loop - generate markdown preview
|
||||
const TARGET_PREVIEW_CHARACTER = 500;
|
||||
const TARGET_PREVIOUS_BLOCK = 1;
|
||||
const TARGET_FOLLOW_BLOCK = 4;
|
||||
for (const block of blockDocuments) {
|
||||
if (block.ref?.length) {
|
||||
const target = block;
|
||||
|
||||
// should only generate the markdown preview belong to the same affine:note
|
||||
const noteBlock = nearestByFlavour(block.blockId, 'affine:note');
|
||||
|
||||
const sameNoteBlocks = noteBlock
|
||||
? blockDocuments.filter(
|
||||
candidate =>
|
||||
nearestByFlavour(candidate.blockId, 'affine:note') === noteBlock
|
||||
)
|
||||
: [];
|
||||
|
||||
// only generate markdown preview for reference blocks
|
||||
let previewText = (await generateMarkdownPreview(target)) ?? '';
|
||||
let previousBlock = 0;
|
||||
let followBlock = 0;
|
||||
let previousIndex = sameNoteBlocks.findIndex(
|
||||
block => block.blockId === target.blockId
|
||||
);
|
||||
let followIndex = previousIndex;
|
||||
|
||||
while (
|
||||
!(
|
||||
(
|
||||
previewText.length > TARGET_PREVIEW_CHARACTER || // stop if preview text reaches the limit
|
||||
((previousBlock >= TARGET_PREVIOUS_BLOCK || previousIndex < 0) &&
|
||||
(followBlock >= TARGET_FOLLOW_BLOCK ||
|
||||
followIndex >= sameNoteBlocks.length))
|
||||
) // stop if no more blocks, or preview block reaches the limit
|
||||
)
|
||||
) {
|
||||
if (previousBlock < TARGET_PREVIOUS_BLOCK) {
|
||||
previousIndex--;
|
||||
const block =
|
||||
previousIndex >= 0 ? sameNoteBlocks.at(previousIndex) : null;
|
||||
const markdown = block ? await generateMarkdownPreview(block) : null;
|
||||
if (
|
||||
markdown &&
|
||||
!previewText.startsWith(
|
||||
markdown
|
||||
) /* A small hack to skip blocks with the same content */
|
||||
) {
|
||||
previewText = markdown + '\n' + previewText;
|
||||
previousBlock++;
|
||||
}
|
||||
}
|
||||
|
||||
if (followBlock < TARGET_FOLLOW_BLOCK) {
|
||||
followIndex++;
|
||||
const block = sameNoteBlocks.at(followIndex);
|
||||
const markdown = block ? await generateMarkdownPreview(block) : null;
|
||||
if (
|
||||
markdown &&
|
||||
!previewText.endsWith(
|
||||
markdown
|
||||
) /* A small hack to skip blocks with the same content */
|
||||
) {
|
||||
previewText = previewText + '\n' + markdown;
|
||||
followBlock++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
block.markdownPreview = unindentMarkdown(previewText);
|
||||
}
|
||||
}
|
||||
// #endregion
|
||||
|
||||
return {
|
||||
blocks: blockDocuments.map(block =>
|
||||
IndexerDocument.from<'block'>(`${docId}:${block.blockId}`, {
|
||||
docId: block.docId,
|
||||
blockId: block.blockId,
|
||||
content: block.content,
|
||||
flavour: block.flavour,
|
||||
blob: block.blob,
|
||||
refDocId: block.refDocId,
|
||||
ref: block.ref,
|
||||
parentFlavour: block.parentFlavour,
|
||||
parentBlockId: block.parentBlockId,
|
||||
additional: block.additional
|
||||
? JSON.stringify(block.additional)
|
||||
: undefined,
|
||||
markdownPreview: block.markdownPreview,
|
||||
})
|
||||
),
|
||||
preview: summary,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,579 @@
|
||||
import {
|
||||
filter,
|
||||
first,
|
||||
Observable,
|
||||
ReplaySubject,
|
||||
share,
|
||||
Subject,
|
||||
throttleTime,
|
||||
} from 'rxjs';
|
||||
import {
|
||||
applyUpdate,
|
||||
type Array as YArray,
|
||||
Doc as YDoc,
|
||||
type Map as YMap,
|
||||
} from 'yjs';
|
||||
|
||||
import {
|
||||
type DocStorage,
|
||||
IndexerDocument,
|
||||
type IndexerStorage,
|
||||
} from '../../storage';
|
||||
import type { IndexerSyncStorage } from '../../storage/indexer-sync';
|
||||
import { AsyncPriorityQueue } from '../../utils/async-priority-queue';
|
||||
import { takeUntilAbort } from '../../utils/take-until-abort';
|
||||
import { MANUALLY_STOP, throwIfAborted } from '../../utils/throw-if-aborted';
|
||||
import { crawlingDocData } from './crawler';
|
||||
|
||||
export interface IndexerSyncState {
|
||||
/**
|
||||
* Number of documents currently in the indexing queue
|
||||
*/
|
||||
indexing: number;
|
||||
/**
|
||||
* Indicates whether all documents have been successfully indexed
|
||||
*
|
||||
* This is only for UI display purposes. For logical operations, please use `waitForCompleted()`
|
||||
*/
|
||||
completed: boolean;
|
||||
/**
|
||||
* Total number of documents in the workspace
|
||||
*/
|
||||
total: number;
|
||||
errorMessage: string | null;
|
||||
}
|
||||
|
||||
export interface IndexerDocSyncState {
|
||||
/**
|
||||
* Indicates whether this document is currently in the indexing queue
|
||||
*/
|
||||
indexing: boolean;
|
||||
/**
|
||||
* Indicates whether this document has been successfully indexed
|
||||
*
|
||||
* This is only for UI display purposes. For logical operations, please use `waitForDocCompleted()`
|
||||
*/
|
||||
completed: boolean;
|
||||
}
|
||||
|
||||
export interface IndexerSync {
|
||||
state$: Observable<IndexerSyncState>;
|
||||
docState$(docId: string): Observable<IndexerDocSyncState>;
|
||||
addPriority(docId: string, priority: number): () => void;
|
||||
waitForCompleted(signal?: AbortSignal): Promise<void>;
|
||||
waitForDocCompleted(docId: string, signal?: AbortSignal): Promise<void>;
|
||||
}
|
||||
|
||||
export class IndexerSyncImpl implements IndexerSync {
|
||||
private abort: AbortController | null = null;
|
||||
private readonly rootDocId = this.doc.spaceId;
|
||||
private readonly status = new IndexerSyncStatus(this.rootDocId);
|
||||
|
||||
state$ = this.status.state$.pipe(
|
||||
// throttle the state to 1 second to avoid spamming the UI
|
||||
throttleTime(1000)
|
||||
);
|
||||
docState$(docId: string) {
|
||||
return this.status.docState$(docId).pipe(
|
||||
// throttle the state to 1 second to avoid spamming the UI
|
||||
throttleTime(1000)
|
||||
);
|
||||
}
|
||||
|
||||
waitForCompleted(signal?: AbortSignal) {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
this.status.state$
|
||||
.pipe(
|
||||
filter(state => state.completed),
|
||||
takeUntilAbort(signal),
|
||||
first()
|
||||
)
|
||||
.subscribe({
|
||||
next: () => {
|
||||
resolve();
|
||||
},
|
||||
error: err => {
|
||||
reject(err);
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
waitForDocCompleted(docId: string, signal?: AbortSignal) {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
this.status
|
||||
.docState$(docId)
|
||||
.pipe(
|
||||
filter(state => state.completed),
|
||||
takeUntilAbort(signal),
|
||||
first()
|
||||
)
|
||||
.subscribe({
|
||||
next: () => {
|
||||
resolve();
|
||||
},
|
||||
error: err => {
|
||||
reject(err);
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
readonly interval = () =>
|
||||
new Promise<void>(resolve =>
|
||||
requestIdleCallback(() => resolve(), {
|
||||
timeout: 200,
|
||||
})
|
||||
);
|
||||
|
||||
constructor(
|
||||
readonly doc: DocStorage,
|
||||
readonly indexer: IndexerStorage,
|
||||
readonly indexerSync: IndexerSyncStorage
|
||||
) {}
|
||||
|
||||
start() {
|
||||
if (this.abort) {
|
||||
this.abort.abort(MANUALLY_STOP);
|
||||
}
|
||||
|
||||
const abort = new AbortController();
|
||||
this.abort = abort;
|
||||
|
||||
this.mainLoop(abort.signal).catch(error => {
|
||||
if (error === MANUALLY_STOP) {
|
||||
return;
|
||||
}
|
||||
console.error('index error', error);
|
||||
});
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.abort?.abort(MANUALLY_STOP);
|
||||
this.abort = null;
|
||||
}
|
||||
|
||||
addPriority(id: string, priority: number) {
|
||||
return this.status.addPriority(id, priority);
|
||||
}
|
||||
|
||||
private async mainLoop(signal?: AbortSignal) {
|
||||
if (this.indexer.isReadonly) {
|
||||
return;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
await this.retryLoop(signal);
|
||||
} catch (error) {
|
||||
if (signal?.aborted) {
|
||||
return;
|
||||
}
|
||||
console.error('index error, retry in 5s', error);
|
||||
this.status.errorMessage =
|
||||
error instanceof Error ? error.message : `${error}`;
|
||||
this.status.statusUpdatedSubject$.next(true);
|
||||
} finally {
|
||||
// reset all status
|
||||
this.status.reset();
|
||||
// wait for 5s before next retry
|
||||
await Promise.race([
|
||||
new Promise<void>(resolve => {
|
||||
setTimeout(resolve, 5000);
|
||||
}),
|
||||
new Promise((_, reject) => {
|
||||
// exit if manually stopped
|
||||
if (signal?.aborted) {
|
||||
reject(signal.reason);
|
||||
}
|
||||
signal?.addEventListener('abort', () => {
|
||||
reject(signal.reason);
|
||||
});
|
||||
}),
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async retryLoop(signal?: AbortSignal) {
|
||||
await Promise.race([
|
||||
Promise.all([
|
||||
this.doc.connection.waitForConnected(signal),
|
||||
this.indexer.connection.waitForConnected(signal),
|
||||
this.indexerSync.connection.waitForConnected(signal),
|
||||
]),
|
||||
new Promise<void>((_, reject) => {
|
||||
setTimeout(() => {
|
||||
reject(new Error('Connect to remote timeout'));
|
||||
}, 1000 * 30);
|
||||
}),
|
||||
new Promise((_, reject) => {
|
||||
signal?.addEventListener('abort', reason => {
|
||||
reject(reason);
|
||||
});
|
||||
}),
|
||||
]);
|
||||
|
||||
this.status.errorMessage = null;
|
||||
this.status.statusUpdatedSubject$.next(true);
|
||||
|
||||
console.log('indexer sync start');
|
||||
|
||||
const unsubscribe = this.doc.subscribeDocUpdate(update => {
|
||||
if (!this.status.rootDocReady) {
|
||||
return;
|
||||
}
|
||||
if (update.docId === this.rootDocId) {
|
||||
applyUpdate(this.status.rootDoc, update.bin);
|
||||
|
||||
const allDocs = this.getAllDocsFromRootDoc();
|
||||
|
||||
for (const [docId, { title }] of allDocs) {
|
||||
const existingDoc = this.status.docsInRootDoc.get(docId);
|
||||
if (!existingDoc) {
|
||||
this.status.scheduleJob(docId);
|
||||
this.status.docsInRootDoc.set(docId, { title });
|
||||
this.status.statusUpdatedSubject$.next(docId);
|
||||
} else {
|
||||
if (existingDoc.title !== title) {
|
||||
this.status.docsInRootDoc.set(docId, { title });
|
||||
this.status.statusUpdatedSubject$.next(docId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const docId of this.status.docsInRootDoc.keys()) {
|
||||
if (!allDocs.has(docId)) {
|
||||
this.status.docsInRootDoc.delete(docId);
|
||||
this.status.statusUpdatedSubject$.next(docId);
|
||||
}
|
||||
}
|
||||
this.status.scheduleJob(this.rootDocId);
|
||||
} else {
|
||||
const docId = update.docId;
|
||||
const existingDoc = this.status.docsInRootDoc.get(docId);
|
||||
if (existingDoc) {
|
||||
this.status.scheduleJob(docId);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
const rootDocBin = (await this.doc.getDoc(this.rootDocId))?.bin;
|
||||
if (rootDocBin) {
|
||||
applyUpdate(this.status.rootDoc, rootDocBin);
|
||||
}
|
||||
|
||||
this.status.scheduleJob(this.rootDocId);
|
||||
|
||||
const allDocs = this.getAllDocsFromRootDoc();
|
||||
this.status.docsInRootDoc = allDocs;
|
||||
this.status.statusUpdatedSubject$.next(true);
|
||||
|
||||
for (const docId of allDocs.keys()) {
|
||||
this.status.scheduleJob(docId);
|
||||
}
|
||||
|
||||
this.status.rootDocReady = true;
|
||||
this.status.statusUpdatedSubject$.next(true);
|
||||
|
||||
const allIndexedDocs = await this.getAllDocsFromIndexer();
|
||||
this.status.docsInIndexer = allIndexedDocs;
|
||||
this.status.statusUpdatedSubject$.next(true);
|
||||
|
||||
while (true) {
|
||||
throwIfAborted(signal);
|
||||
|
||||
const docId = await this.status.acceptJob(signal);
|
||||
|
||||
if (docId === this.rootDocId) {
|
||||
// #region crawl root doc
|
||||
for (const [docId, { title }] of this.status.docsInRootDoc) {
|
||||
const existingDoc = this.status.docsInIndexer.get(docId);
|
||||
if (existingDoc) {
|
||||
if (existingDoc.title !== title) {
|
||||
// need update
|
||||
await this.indexer.update(
|
||||
'doc',
|
||||
IndexerDocument.from(docId, {
|
||||
docId,
|
||||
title,
|
||||
})
|
||||
);
|
||||
this.status.docsInIndexer.set(docId, { title });
|
||||
this.status.statusUpdatedSubject$.next(docId);
|
||||
}
|
||||
} else {
|
||||
// need add
|
||||
await this.indexer.insert(
|
||||
'doc',
|
||||
IndexerDocument.from(docId, {
|
||||
docId,
|
||||
title,
|
||||
})
|
||||
);
|
||||
this.status.docsInIndexer.set(docId, { title });
|
||||
this.status.statusUpdatedSubject$.next(docId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const docId of this.status.docsInIndexer.keys()) {
|
||||
if (!this.status.docsInRootDoc.has(docId)) {
|
||||
await this.indexer.delete('doc', docId);
|
||||
await this.indexer.deleteByQuery('block', {
|
||||
type: 'match',
|
||||
field: 'docId',
|
||||
match: docId,
|
||||
});
|
||||
await this.indexerSync.clearDocIndexedClock(docId);
|
||||
this.status.docsInIndexer.delete(docId);
|
||||
this.status.statusUpdatedSubject$.next(docId);
|
||||
}
|
||||
}
|
||||
await this.indexer.refresh('block');
|
||||
await this.indexer.refresh('doc');
|
||||
// #endregion
|
||||
} else {
|
||||
// #region crawl doc
|
||||
const existingDoc = this.status.docsInIndexer.get(docId);
|
||||
if (!existingDoc) {
|
||||
// doc is deleted, just skip
|
||||
continue;
|
||||
}
|
||||
|
||||
const docClock = await this.doc.getDocTimestamp(docId);
|
||||
if (!docClock) {
|
||||
// doc is deleted, just skip
|
||||
continue;
|
||||
}
|
||||
|
||||
const docIndexedClock =
|
||||
await this.indexerSync.getDocIndexedClock(docId);
|
||||
if (
|
||||
docIndexedClock &&
|
||||
docIndexedClock.timestamp.getTime() === docClock.timestamp.getTime()
|
||||
) {
|
||||
// doc is already indexed, just skip
|
||||
continue;
|
||||
}
|
||||
|
||||
const docBin = await this.doc.getDoc(docId);
|
||||
if (!docBin) {
|
||||
// doc is deleted, just skip
|
||||
continue;
|
||||
}
|
||||
const docYDoc = new YDoc({ guid: docId });
|
||||
applyUpdate(docYDoc, docBin.bin);
|
||||
|
||||
let blocks: IndexerDocument<'block'>[] = [];
|
||||
let preview: string | undefined;
|
||||
|
||||
try {
|
||||
const result = await crawlingDocData({
|
||||
ydoc: docYDoc,
|
||||
rootYDoc: this.status.rootDoc,
|
||||
spaceId: this.status.rootDocId,
|
||||
docId,
|
||||
});
|
||||
blocks = result.blocks;
|
||||
preview = result.preview;
|
||||
} catch (error) {
|
||||
console.error('error crawling doc', error);
|
||||
}
|
||||
|
||||
await this.indexer.deleteByQuery('block', {
|
||||
type: 'match',
|
||||
field: 'docId',
|
||||
match: docId,
|
||||
});
|
||||
|
||||
for (const block of blocks) {
|
||||
await this.indexer.insert('block', block);
|
||||
}
|
||||
|
||||
await this.indexer.refresh('block');
|
||||
|
||||
if (preview) {
|
||||
await this.indexer.update(
|
||||
'doc',
|
||||
IndexerDocument.from(docId, {
|
||||
summary: preview,
|
||||
})
|
||||
);
|
||||
await this.indexer.refresh('doc');
|
||||
}
|
||||
|
||||
await this.indexerSync.setDocIndexedClock({
|
||||
docId,
|
||||
timestamp: docClock.timestamp,
|
||||
});
|
||||
// #endregion
|
||||
}
|
||||
|
||||
this.status.completeJob();
|
||||
}
|
||||
} finally {
|
||||
unsubscribe();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all docs from the root doc, without deleted docs
|
||||
*/
|
||||
private getAllDocsFromRootDoc() {
|
||||
const docs = this.status.rootDoc.getMap('meta').get('pages') as
|
||||
| YArray<YMap<any>>
|
||||
| undefined;
|
||||
const availableDocs = new Map<string, { title: string | undefined }>();
|
||||
|
||||
if (docs) {
|
||||
for (const page of docs) {
|
||||
const docId = page.get('id');
|
||||
|
||||
if (typeof docId !== 'string') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const inTrash = page.get('trash') ?? false;
|
||||
const title = page.get('title');
|
||||
|
||||
if (!inTrash) {
|
||||
availableDocs.set(docId, { title });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return availableDocs;
|
||||
}
|
||||
|
||||
private async getAllDocsFromIndexer() {
|
||||
const docs = await this.indexer.search(
|
||||
'doc',
|
||||
{
|
||||
type: 'all',
|
||||
},
|
||||
{
|
||||
pagination: {
|
||||
limit: Infinity,
|
||||
},
|
||||
fields: ['docId', 'title'],
|
||||
}
|
||||
);
|
||||
|
||||
return new Map(
|
||||
docs.nodes.map(node => {
|
||||
const title = node.fields.title;
|
||||
return [
|
||||
node.id,
|
||||
{
|
||||
title: typeof title === 'string' ? title : title.at(0),
|
||||
},
|
||||
];
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class IndexerSyncStatus {
|
||||
prioritySettings = new Map<string, number>();
|
||||
jobs = new AsyncPriorityQueue();
|
||||
rootDoc = new YDoc({ guid: this.rootDocId });
|
||||
rootDocReady = false;
|
||||
docsInIndexer = new Map<string, { title: string | undefined }>();
|
||||
docsInRootDoc = new Map<string, { title: string | undefined }>();
|
||||
currentJob: string | null = null;
|
||||
errorMessage: string | null = null;
|
||||
statusUpdatedSubject$ = new Subject<string | true>();
|
||||
|
||||
state$ = new Observable<IndexerSyncState>(subscribe => {
|
||||
const next = () => {
|
||||
subscribe.next({
|
||||
indexing: this.jobs.length() + (this.currentJob ? 1 : 0),
|
||||
total: this.docsInRootDoc.size + 1,
|
||||
errorMessage: this.errorMessage,
|
||||
completed: this.rootDocReady && this.jobs.length() === 0,
|
||||
});
|
||||
};
|
||||
next();
|
||||
const dispose = this.statusUpdatedSubject$.subscribe(() => {
|
||||
next();
|
||||
});
|
||||
return () => {
|
||||
dispose.unsubscribe();
|
||||
};
|
||||
}).pipe(
|
||||
share({
|
||||
connector: () => new ReplaySubject(1),
|
||||
})
|
||||
);
|
||||
|
||||
docState$(docId: string) {
|
||||
return new Observable<IndexerDocSyncState>(subscribe => {
|
||||
const next = () => {
|
||||
subscribe.next({
|
||||
indexing: this.jobs.has(docId),
|
||||
completed: this.docsInIndexer.has(docId) && !this.jobs.has(docId),
|
||||
});
|
||||
};
|
||||
next();
|
||||
const dispose = this.statusUpdatedSubject$.subscribe(updatedDocId => {
|
||||
if (updatedDocId === docId || updatedDocId === true) {
|
||||
next();
|
||||
}
|
||||
});
|
||||
return () => {
|
||||
dispose.unsubscribe();
|
||||
};
|
||||
}).pipe(
|
||||
share({
|
||||
connector: () => new ReplaySubject(1),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
constructor(readonly rootDocId: string) {
|
||||
this.prioritySettings.set(this.rootDocId, Infinity);
|
||||
}
|
||||
|
||||
scheduleJob(docId: string) {
|
||||
const priority = this.prioritySettings.get(docId) ?? 0;
|
||||
this.jobs.push(docId, priority);
|
||||
this.statusUpdatedSubject$.next(docId);
|
||||
}
|
||||
|
||||
async acceptJob(abort?: AbortSignal) {
|
||||
const job = await this.jobs.asyncPop(abort);
|
||||
this.currentJob = job;
|
||||
this.statusUpdatedSubject$.next(job);
|
||||
return job;
|
||||
}
|
||||
|
||||
completeJob() {
|
||||
const job = this.currentJob;
|
||||
this.currentJob = null;
|
||||
this.statusUpdatedSubject$.next(job ?? true);
|
||||
}
|
||||
|
||||
addPriority(id: string, priority: number) {
|
||||
const oldPriority = this.prioritySettings.get(id) ?? 0;
|
||||
this.prioritySettings.set(id, priority);
|
||||
this.jobs.setPriority(id, oldPriority + priority);
|
||||
|
||||
return () => {
|
||||
const currentPriority = this.prioritySettings.get(id) ?? 0;
|
||||
this.prioritySettings.set(id, currentPriority - priority);
|
||||
this.jobs.setPriority(id, currentPriority - priority);
|
||||
};
|
||||
}
|
||||
|
||||
reset() {
|
||||
// reset all state, except prioritySettings
|
||||
this.jobs.clear();
|
||||
this.docsInRootDoc.clear();
|
||||
this.docsInIndexer.clear();
|
||||
this.rootDoc = new YDoc();
|
||||
this.rootDocReady = false;
|
||||
this.currentJob = null;
|
||||
this.statusUpdatedSubject$.next(true);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user