refactor(core): use new backlink indexer (#7296)

packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.ts

@@ -0,0 +1,24 @@
+import type { DocService } from '@toeverything/infra';
+import { Entity, LiveData } from '@toeverything/infra';
+
+import type { DocsSearchService } from '../../docs-search';
+
+interface Backlink {
+  docId: string;
+  blockId: string;
+  title: string;
+}
+
+export class DocBacklinks extends Entity {
+  constructor(
+    private readonly docsSearchService: DocsSearchService,
+    private readonly docService: DocService
+  ) {
+    super();
+  }
+
+  backlinks$ = LiveData.from<Backlink[]>(
+    this.docsSearchService.watchRefsTo(this.docService.doc.id),
+    []
+  );
+}

packages/frontend/core/src/modules/doc-link/entities/doc-links.ts

@@ -0,0 +1,23 @@
+import type { DocService } from '@toeverything/infra';
+import { Entity, LiveData } from '@toeverything/infra';
+
+import type { DocsSearchService } from '../../docs-search';
+
+interface Link {
+  docId: string;
+  title: string;
+}
+
+export class DocLinks extends Entity {
+  constructor(
+    private readonly docsSearchService: DocsSearchService,
+    private readonly docService: DocService
+  ) {
+    super();
+  }
+
+  links$ = LiveData.from<Link[]>(
+    this.docsSearchService.watchRefsFrom(this.docService.doc.id),
+    []
+  );
+}

packages/frontend/core/src/modules/doc-link/index.ts

@@ -0,0 +1,22 @@
+import {
+  DocScope,
+  DocService,
+  type Framework,
+  WorkspaceScope,
+} from '@toeverything/infra';
+
+import { DocsSearchService } from '../docs-search';
+import { DocBacklinks } from './entities/doc-backlinks';
+import { DocLinks } from './entities/doc-links';
+import { DocLinksService } from './services/doc-links';
+
+export { DocLinksService } from './services/doc-links';
+
+export function configureDocLinksModule(framework: Framework) {
+  framework
+    .scope(WorkspaceScope)
+    .scope(DocScope)
+    .service(DocLinksService)
+    .entity(DocBacklinks, [DocsSearchService, DocService])
+    .entity(DocLinks, [DocsSearchService, DocService]);
+}

packages/frontend/core/src/modules/doc-link/services/doc-links.ts

@@ -0,0 +1,9 @@
+import { Service } from '@toeverything/infra';
+
+import { DocBacklinks } from '../entities/doc-backlinks';
+import { DocLinks } from '../entities/doc-links';
+
+export class DocLinksService extends Service {
+  backlinks = this.framework.createEntity(DocBacklinks);
+  links = this.framework.createEntity(DocLinks);
+}

packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts

@@ -91,6 +91,7 @@ export class DocsIndexer extends Entity {
     // jobs should have the same docId, so we just pick the first one
     const docId = jobs[0].payload.docId;
 
+    const startTime = performance.now();
     logger.debug('Start crawling job for docId:', docId);
 
     if (docId) {
@@ -100,6 +101,11 @@ export class DocsIndexer extends Entity {
         await this.crawlingDocData(docId);
       }
     }
+
+    const duration = performance.now() - startTime;
+    logger.debug(
+      'Finish crawling job for docId:' + docId + ' in ' + duration + 'ms '
+    );
   }
 
   startCrawling() {

packages/frontend/core/src/modules/docs-search/services/docs-search.ts

@@ -45,7 +45,7 @@ export class DocsSearchService extends Service {
               },
               {
                 type: 'boost',
-                boost: 100,
+                boost: 1.5,
                 query: {
                   type: 'match',
                   field: 'flavour',
@@ -225,6 +225,195 @@ export class DocsSearchService extends Service {
       );
   }
 
+  async searchRefsFrom(docId: string): Promise<
+    {
+      docId: string;
+      title: string;
+    }[]
+  > {
+    const { nodes } = await this.indexer.blockIndex.search(
+      {
+        type: 'boolean',
+        occur: 'must',
+        queries: [
+          {
+            type: 'match',
+            field: 'docId',
+            match: docId,
+          },
+          {
+            type: 'exists',
+            field: 'ref',
+          },
+        ],
+      },
+      {
+        fields: ['ref'],
+        pagination: {
+          limit: 100,
+        },
+      }
+    );
+
+    const docIds = new Set(
+      nodes.flatMap(node => {
+        const refs = node.fields.ref;
+        return typeof refs === 'string' ? [refs] : refs;
+      })
+    );
+
+    const docData = await this.indexer.docIndex.getAll(Array.from(docIds));
+
+    return docData.map(doc => {
+      const title = doc.get('title');
+      return {
+        docId: doc.id,
+        title: title ? (typeof title === 'string' ? title : title[0]) : '',
+      };
+    });
+  }
+
+  watchRefsFrom(docId: string) {
+    return this.indexer.blockIndex
+      .search$(
+        {
+          type: 'boolean',
+          occur: 'must',
+          queries: [
+            {
+              type: 'match',
+              field: 'docId',
+              match: docId,
+            },
+            {
+              type: 'exists',
+              field: 'ref',
+            },
+          ],
+        },
+        {
+          fields: ['ref'],
+          pagination: {
+            limit: 100,
+          },
+        }
+      )
+      .pipe(
+        switchMap(({ nodes }) => {
+          return fromPromise(async () => {
+            const docIds = new Set(
+              nodes.flatMap(node => {
+                const refs = node.fields.ref;
+                return typeof refs === 'string' ? [refs] : refs;
+              })
+            );
+
+            const docData = await this.indexer.docIndex.getAll(
+              Array.from(docIds)
+            );
+
+            return docData.map(doc => {
+              const title = doc.get('title');
+              return {
+                docId: doc.id,
+                title: title
+                  ? typeof title === 'string'
+                    ? title
+                    : title[0]
+                  : '',
+              };
+            });
+          });
+        })
+      );
+  }
+
+  async searchRefsTo(docId: string): Promise<
+    {
+      docId: string;
+      blockId: string;
+      title: string;
+    }[]
+  > {
+    const { buckets } = await this.indexer.blockIndex.aggregate(
+      {
+        type: 'match',
+        field: 'ref',
+        match: docId,
+      },
+      'docId',
+      {
+        hits: {
+          fields: ['docId', 'blockId'],
+          pagination: {
+            limit: 1,
+          },
+        },
+        pagination: {
+          limit: 100,
+        },
+      }
+    );
+
+    const docData = await this.indexer.docIndex.getAll(
+      buckets.map(bucket => bucket.key)
+    );
+
+    return buckets.map(bucket => {
+      const title =
+        docData.find(doc => doc.id === bucket.key)?.get('title') ?? '';
+      const blockId = bucket.hits.nodes[0]?.fields.blockId ?? '';
+      return {
+        docId: bucket.key,
+        blockId: typeof blockId === 'string' ? blockId : blockId[0],
+        title: typeof title === 'string' ? title : title[0],
+      };
+    });
+  }
+
+  watchRefsTo(docId: string) {
+    return this.indexer.blockIndex
+      .aggregate$(
+        {
+          type: 'match',
+          field: 'ref',
+          match: docId,
+        },
+        'docId',
+        {
+          hits: {
+            fields: ['docId', 'blockId'],
+            pagination: {
+              limit: 1,
+            },
+          },
+          pagination: {
+            limit: 100,
+          },
+        }
+      )
+      .pipe(
+        switchMap(({ buckets }) => {
+          return fromPromise(async () => {
+            const docData = await this.indexer.docIndex.getAll(
+              buckets.map(bucket => bucket.key)
+            );
+
+            return buckets.map(bucket => {
+              const title =
+                docData.find(doc => doc.id === bucket.key)?.get('title') ?? '';
+              const blockId = bucket.hits.nodes[0]?.fields.blockId ?? '';
+              return {
+                docId: bucket.key,
+                blockId: typeof blockId === 'string' ? blockId : blockId[0],
+                title: typeof title === 'string' ? title : title[0],
+              };
+            });
+          });
+        })
+      );
+  }
+
   async getDocTitle(docId: string) {
     const doc = await this.indexer.docIndex.get(docId);
     const title = doc?.get('title');

packages/frontend/core/src/modules/index.ts

@@ -3,6 +3,7 @@ import { configureInfraModules, type Framework } from '@toeverything/infra';
 
 import { configureCloudModule } from './cloud';
 import { configureCollectionModule } from './collection';
+import { configureDocLinksModule } from './doc-link';
 import { configureDocsSearchModule } from './docs-search';
 import { configureFindInPageModule } from './find-in-page';
 import { configureNavigationModule } from './navigation';
@@ -34,6 +35,7 @@ export function configureCommonModules(framework: Framework) {
   configurePeekViewModule(framework);
   configureQuickSearchModule(framework);
   configureDocsSearchModule(framework);
+  configureDocLinksModule(framework);
 }
 
 export function configureImpls(framework: Framework) {

packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts

@@ -210,6 +210,8 @@ export class CloudWorkspaceFlavourProviderService
     const bs = new DocCollection({
       id,
       schema: globalBlockSuiteSchema,
+      disableBacklinkIndex: true,
+      disableSearchIndex: true,
     });
 
     if (localData) applyUpdate(bs.doc, localData);

packages/frontend/core/src/modules/workspace-engine/impls/local.ts

@@ -142,6 +142,8 @@ export class LocalWorkspaceFlavourProvider
     const bs = new DocCollection({
       id,
       schema: globalBlockSuiteSchema,
+      disableBacklinkIndex: true,
+      disableSearchIndex: true,
     });
 
     if (localData) applyUpdate(bs.doc, localData);