feat(server): handle workspace doc updates (#11937)

packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.md

@@ -454,3 +454,43 @@ Generated by [AVA](https://avajs.dev).
         ],
       },
     ]
+
+## should index doc work
+
+> Snapshot 1
+
+    {
+      summary: [
+        'AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. You own your data, with no compromisesLocal-first & Real-time collaborativeWe love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.Blocks that assemble your next docs, tasks kanban or whiteboardThere is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ',
+      ],
+      title: [
+        'Write, Draw, Plan all at Once.',
+      ],
+    }
+
+> Snapshot 2
+
+    [
+      {
+        blockId: [
+          'VMx9lHw3TR',
+        ],
+        content: [
+          'For developers or installations guides, please go to AFFiNE Doc',
+        ],
+        flavour: [
+          'affine:paragraph',
+        ],
+      },
+      {
+        blockId: [
+          '9-K49otbCv',
+        ],
+        content: [
+          'For developer or installation guides, please go to AFFiNE Development',
+        ],
+        flavour: [
+          'affine:paragraph',
+        ],
+      },
+    ]

packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts

@@ -0,0 +1,108 @@
+import { randomUUID } from 'node:crypto';
+import { mock } from 'node:test';
+
+import test from 'ava';
+import Sinon from 'sinon';
+
+import { createModule } from '../../../__tests__/create-module';
+import { Mockers } from '../../../__tests__/mocks';
+import { ServerConfigModule } from '../../../core/config';
+import { IndexerModule, IndexerService } from '..';
+import { SearchProviderFactory } from '../factory';
+import { IndexerJob } from '../job';
+import { ManticoresearchProvider } from '../providers';
+
+const module = await createModule({
+  imports: [IndexerModule, ServerConfigModule],
+  providers: [IndexerService],
+});
+const indexerService = module.get(IndexerService);
+const indexerJob = module.get(IndexerJob);
+const searchProviderFactory = module.get(SearchProviderFactory);
+const manticoresearch = module.get(ManticoresearchProvider);
+
+const user = await module.create(Mockers.User);
+const workspace = await module.create(Mockers.Workspace, {
+  snapshot: true,
+  owner: user,
+});
+
+test.after.always(async () => {
+  await module.close();
+});
+
+test.afterEach.always(() => {
+  Sinon.restore();
+  mock.reset();
+});
+
+test.beforeEach(() => {
+  mock.method(searchProviderFactory, 'get', () => {
+    return manticoresearch;
+  });
+});
+
+test('should handle indexer.indexDoc job', async t => {
+  const spy = Sinon.spy(indexerService, 'indexDoc');
+  await indexerJob.indexDoc({
+    workspaceId: workspace.id,
+    docId: randomUUID(),
+  });
+  t.is(spy.callCount, 1);
+});
+
+test('should handle indexer.deleteDoc job', async t => {
+  const spy = Sinon.spy(indexerService, 'deleteDoc');
+  await indexerJob.deleteDoc({
+    workspaceId: workspace.id,
+    docId: randomUUID(),
+  });
+  t.is(spy.callCount, 1);
+});
+
+test('should handle indexer.indexWorkspace job', async t => {
+  const count = module.queue.count('indexer.deleteDoc');
+  const spy = Sinon.spy(indexerService, 'listDocIds');
+  await indexerJob.indexWorkspace({
+    workspaceId: workspace.id,
+  });
+  t.is(spy.callCount, 1);
+  const { payload } = await module.queue.waitFor('indexer.indexDoc');
+  t.is(payload.workspaceId, workspace.id);
+  t.is(payload.docId, '5nS9BSp3Px');
+  // no delete job
+  t.is(module.queue.count('indexer.deleteDoc'), count);
+});
+
+test('should not sync existing doc', async t => {
+  const count = module.queue.count('indexer.indexDoc');
+  mock.method(indexerService, 'listDocIds', async () => {
+    return ['5nS9BSp3Px'];
+  });
+  await indexerJob.indexWorkspace({
+    workspaceId: workspace.id,
+  });
+  t.is(module.queue.count('indexer.indexDoc'), count);
+});
+
+test('should delete doc from indexer when docId is not in workspace', async t => {
+  const count = module.queue.count('indexer.deleteDoc');
+  mock.method(indexerService, 'listDocIds', async () => {
+    return ['mock-doc-id1', 'mock-doc-id2'];
+  });
+  await indexerJob.indexWorkspace({
+    workspaceId: workspace.id,
+  });
+  const { payload } = await module.queue.waitFor('indexer.indexDoc');
+  t.is(payload.workspaceId, workspace.id);
+  t.is(payload.docId, '5nS9BSp3Px');
+  t.is(module.queue.count('indexer.deleteDoc'), count + 2);
+});
+
+test('should handle indexer.deleteWorkspace job', async t => {
+  const spy = Sinon.spy(indexerService, 'deleteWorkspace');
+  await indexerJob.deleteWorkspace({
+    workspaceId: workspace.id,
+  });
+  t.is(spy.callCount, 1);
+});

packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts

@@ -27,7 +27,10 @@ const indexerService = module.get(IndexerService);
 const searchProviderFactory = module.get(SearchProviderFactory);
 const manticoresearch = module.get(ManticoresearchProvider);
 const user = await module.create(Mockers.User);
-const workspace = await module.create(Mockers.Workspace);
+const workspace = await module.create(Mockers.Workspace, {
+  snapshot: true,
+  owner: user,
+});
 
 mock.method(searchProviderFactory, 'get', () => {
   return manticoresearch;
@@ -1580,3 +1583,524 @@ test('should throw error when field is not allowed in aggregate input', async t
 });
 
 // #endregion
+
+// #region deleteWorkspace()
+
+test('should delete workspace work', async t => {
+  const workspaceId = randomUUID();
+  const docId1 = randomUUID();
+  const docId2 = randomUUID();
+  await indexerService.write(
+    SearchTable.doc,
+    [
+      {
+        workspaceId,
+        docId: docId1,
+        title: 'hello world',
+        summary: 'this is a test',
+        createdByUserId: user.id,
+        updatedByUserId: user.id,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      },
+      {
+        workspaceId,
+        docId: docId2,
+        title: 'hello world',
+        summary: 'this is a test',
+        createdByUserId: user.id,
+        updatedByUserId: user.id,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      },
+    ],
+    {
+      refresh: true,
+    }
+  );
+  await indexerService.write(
+    SearchTable.block,
+    [
+      {
+        workspaceId,
+        docId: docId1,
+        blockId: randomUUID(),
+        content: 'hello world',
+        flavour: 'affine:text',
+        createdByUserId: user.id,
+        updatedByUserId: user.id,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      },
+    ],
+    {
+      refresh: true,
+    }
+  );
+
+  let result = await indexerService.search({
+    table: SearchTable.doc,
+    query: {
+      type: SearchQueryType.match,
+      field: 'workspaceId',
+      match: workspaceId,
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'title', 'summary'],
+    },
+  });
+
+  t.is(result.total, 2);
+  t.is(result.nodes.length, 2);
+
+  let result2 = await indexerService.search({
+    table: SearchTable.block,
+    query: {
+      type: SearchQueryType.match,
+      field: 'workspaceId',
+      match: workspaceId,
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'],
+    },
+  });
+
+  t.is(result2.total, 1);
+  t.is(result2.nodes.length, 1);
+
+  await indexerService.deleteWorkspace(workspaceId, {
+    refresh: true,
+  });
+
+  result = await indexerService.search({
+    table: SearchTable.doc,
+    query: {
+      type: SearchQueryType.match,
+      field: 'workspaceId',
+      match: workspaceId,
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'title', 'summary'],
+    },
+  });
+  t.is(result.total, 0);
+  t.is(result.nodes.length, 0);
+
+  result2 = await indexerService.search({
+    table: SearchTable.block,
+    query: {
+      type: SearchQueryType.match,
+      field: 'workspaceId',
+      match: workspaceId,
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'],
+    },
+  });
+
+  t.is(result2.total, 0);
+  t.is(result2.nodes.length, 0);
+});
+
+// #endregion
+
+// #region deleteDoc()
+
+test('should delete doc work', async t => {
+  const workspaceId = randomUUID();
+  const docId1 = randomUUID();
+  const docId2 = randomUUID();
+  await indexerService.write(
+    SearchTable.doc,
+    [
+      {
+        workspaceId,
+        docId: docId1,
+        title: 'hello world',
+        summary: 'this is a test',
+        createdByUserId: user.id,
+        updatedByUserId: user.id,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      },
+      {
+        workspaceId,
+        docId: docId2,
+        title: 'hello world',
+        summary: 'this is a test',
+        createdByUserId: user.id,
+        updatedByUserId: user.id,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      },
+    ],
+    {
+      refresh: true,
+    }
+  );
+  await indexerService.write(
+    SearchTable.block,
+    [
+      {
+        workspaceId,
+        docId: docId1,
+        blockId: randomUUID(),
+        content: 'hello world',
+        flavour: 'affine:text',
+        createdByUserId: user.id,
+        updatedByUserId: user.id,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      },
+      {
+        workspaceId,
+        docId: docId2,
+        blockId: randomUUID(),
+        content: 'hello world',
+        flavour: 'affine:text',
+        createdByUserId: user.id,
+        updatedByUserId: user.id,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      },
+    ],
+    {
+      refresh: true,
+    }
+  );
+
+  let result1 = await indexerService.search({
+    table: SearchTable.doc,
+    query: {
+      type: SearchQueryType.boolean,
+      occur: SearchQueryOccur.must,
+      queries: [
+        {
+          type: SearchQueryType.match,
+          field: 'workspaceId',
+          match: workspaceId,
+        },
+        {
+          type: SearchQueryType.match,
+          field: 'docId',
+          match: docId1,
+        },
+      ],
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'title', 'summary'],
+    },
+  });
+
+  t.is(result1.total, 1);
+  t.is(result1.nodes.length, 1);
+  t.deepEqual(result1.nodes[0].fields.docId, [docId1]);
+
+  let result2 = await indexerService.search({
+    table: SearchTable.doc,
+    query: {
+      type: SearchQueryType.boolean,
+      occur: SearchQueryOccur.must,
+      queries: [
+        {
+          type: SearchQueryType.match,
+          field: 'workspaceId',
+          match: workspaceId,
+        },
+        {
+          type: SearchQueryType.match,
+          field: 'docId',
+          match: docId2,
+        },
+      ],
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'title', 'summary'],
+    },
+  });
+
+  t.is(result2.total, 1);
+  t.is(result2.nodes.length, 1);
+  t.deepEqual(result2.nodes[0].fields.docId, [docId2]);
+
+  let result3 = await indexerService.search({
+    table: SearchTable.block,
+    query: {
+      type: SearchQueryType.boolean,
+      occur: SearchQueryOccur.must,
+      queries: [
+        {
+          type: SearchQueryType.match,
+          field: 'workspaceId',
+          match: workspaceId,
+        },
+        {
+          type: SearchQueryType.match,
+          field: 'docId',
+          match: docId1,
+        },
+      ],
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'],
+    },
+  });
+  t.is(result3.total, 1);
+  t.is(result3.nodes.length, 1);
+  t.deepEqual(result3.nodes[0].fields.docId, [docId1]);
+
+  let result4 = await indexerService.search({
+    table: SearchTable.block,
+    query: {
+      type: SearchQueryType.boolean,
+      occur: SearchQueryOccur.must,
+      queries: [
+        {
+          type: SearchQueryType.match,
+          field: 'workspaceId',
+          match: workspaceId,
+        },
+        {
+          type: SearchQueryType.match,
+          field: 'docId',
+          match: docId2,
+        },
+      ],
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'],
+    },
+  });
+  t.is(result4.total, 1);
+  t.is(result4.nodes.length, 1);
+  t.deepEqual(result4.nodes[0].fields.docId, [docId2]);
+
+  const count = module.event.count('doc.indexer.deleted');
+
+  await indexerService.deleteDoc(workspaceId, docId1, {
+    refresh: true,
+  });
+  t.is(module.event.count('doc.indexer.deleted'), count + 1);
+
+  // make sure the docId1 is deleted
+  result1 = await indexerService.search({
+    table: SearchTable.doc,
+    query: {
+      type: SearchQueryType.boolean,
+      occur: SearchQueryOccur.must,
+      queries: [
+        {
+          type: SearchQueryType.match,
+          field: 'workspaceId',
+          match: workspaceId,
+        },
+        {
+          type: SearchQueryType.match,
+          field: 'docId',
+          match: docId1,
+        },
+      ],
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'title', 'summary'],
+    },
+  });
+
+  t.is(result1.total, 0);
+  t.is(result1.nodes.length, 0);
+
+  // make sure the docId2 is not deleted
+  result2 = await indexerService.search({
+    table: SearchTable.doc,
+    query: {
+      type: SearchQueryType.boolean,
+      occur: SearchQueryOccur.must,
+      queries: [
+        {
+          type: SearchQueryType.match,
+          field: 'workspaceId',
+          match: workspaceId,
+        },
+        {
+          type: SearchQueryType.match,
+          field: 'docId',
+          match: docId2,
+        },
+      ],
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'title', 'summary'],
+    },
+  });
+
+  t.is(result2.total, 1);
+  t.is(result2.nodes.length, 1);
+  t.deepEqual(result2.nodes[0].fields.docId, [docId2]);
+
+  // make sure the docId1 block is deleted
+  result3 = await indexerService.search({
+    table: SearchTable.block,
+    query: {
+      type: SearchQueryType.boolean,
+      occur: SearchQueryOccur.must,
+      queries: [
+        {
+          type: SearchQueryType.match,
+          field: 'workspaceId',
+          match: workspaceId,
+        },
+        {
+          type: SearchQueryType.match,
+          field: 'docId',
+          match: docId1,
+        },
+      ],
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'],
+    },
+  });
+
+  t.is(result3.total, 0);
+  t.is(result3.nodes.length, 0);
+
+  // docId2 block should not be deleted
+  result4 = await indexerService.search({
+    table: SearchTable.block,
+    query: {
+      type: SearchQueryType.boolean,
+      occur: SearchQueryOccur.must,
+      queries: [
+        {
+          type: SearchQueryType.match,
+          field: 'workspaceId',
+          match: workspaceId,
+        },
+        {
+          type: SearchQueryType.match,
+          field: 'docId',
+          match: docId2,
+        },
+      ],
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'],
+    },
+  });
+
+  t.is(result4.total, 1);
+  t.is(result4.nodes.length, 1);
+  t.deepEqual(result4.nodes[0].fields.docId, [docId2]);
+});
+
+// #endregion
+
+// #region listDocIds()
+
+test('should list doc ids work', async t => {
+  const workspaceId = randomUUID();
+  const docs = [];
+  const docCount = 20011;
+  for (let i = 0; i < docCount; i++) {
+    docs.push({
+      workspaceId,
+      docId: randomUUID(),
+      title: `hello world ${i} ${randomUUID()}`,
+      summary: `this is a test ${i} ${randomUUID()}`,
+      createdByUserId: user.id,
+      updatedByUserId: user.id,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    });
+  }
+  await indexerService.write(SearchTable.doc, docs, {
+    refresh: true,
+  });
+
+  const docIds = await indexerService.listDocIds(workspaceId);
+
+  t.is(docIds.length, docCount);
+  t.deepEqual(docIds.sort(), docs.map(doc => doc.docId).sort());
+
+  await indexerService.deleteWorkspace(workspaceId, {
+    refresh: true,
+  });
+  const docIds2 = await indexerService.listDocIds(workspaceId);
+
+  t.is(docIds2.length, 0);
+});
+
+// #endregion
+
+// #region indexDoc()
+
+test('should index doc work', async t => {
+  const count = module.event.count('doc.indexer.updated');
+  const docSnapshot = await module.create(Mockers.DocSnapshot, {
+    workspaceId: workspace.id,
+    user,
+  });
+
+  await indexerService.indexDoc(workspace.id, docSnapshot.id, {
+    refresh: true,
+  });
+
+  const result = await indexerService.search({
+    table: SearchTable.doc,
+    query: {
+      type: SearchQueryType.match,
+      field: 'docId',
+      match: docSnapshot.id,
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'title', 'summary'],
+    },
+  });
+
+  t.is(result.total, 1);
+  t.deepEqual(result.nodes[0].fields.workspaceId, [workspace.id]);
+  t.deepEqual(result.nodes[0].fields.docId, [docSnapshot.id]);
+  t.snapshot(omit(result.nodes[0].fields, ['workspaceId', 'docId']));
+
+  // search blocks
+  const result2 = await indexerService.search({
+    table: SearchTable.block,
+    query: {
+      type: SearchQueryType.boolean,
+      occur: SearchQueryOccur.must,
+      queries: [
+        {
+          type: SearchQueryType.match,
+          field: 'workspaceId',
+          match: workspace.id,
+        },
+        {
+          type: SearchQueryType.match,
+          field: 'content',
+          match:
+            'For developers or installations guides, please go to AFFiNE Doc',
+        },
+      ],
+    },
+    options: {
+      fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'],
+      highlights: [
+        {
+          field: 'content',
+          before: '<b>',
+          end: '</b>',
+        },
+      ],
+      pagination: {
+        limit: 2,
+      },
+    },
+  });
+
+  t.is(result2.nodes.length, 2);
+  t.snapshot(
+    result2.nodes.map(node => omit(node.fields, ['workspaceId', 'docId']))
+  );
+  t.is(module.event.count('doc.indexer.updated'), count + 1);
+});
+// #endregion

packages/backend/server/src/plugins/indexer/index.ts

@@ -5,6 +5,7 @@ import { Module } from '@nestjs/common';
 import { ServerConfigModule } from '../../core/config';
 import { PermissionModule } from '../../core/permission';
 import { SearchProviderFactory } from './factory';
+import { IndexerJob } from './job';
 import { SearchProviders } from './providers';
 import { IndexerResolver } from './resolver';
 import { IndexerService } from './service';
@@ -14,6 +15,7 @@ import { IndexerService } from './service';
   providers: [
     IndexerResolver,
     IndexerService,
+    IndexerJob,
     SearchProviderFactory,
     ...SearchProviders,
   ],
@@ -22,3 +24,16 @@ import { IndexerService } from './service';
 export class IndexerModule {}
 
 export { IndexerService };
+
+declare global {
+  interface Events {
+    'doc.indexer.updated': {
+      workspaceId: string;
+      docId: string;
+    };
+    'doc.indexer.deleted': {
+      workspaceId: string;
+      docId: string;
+    };
+  }
+}

packages/backend/server/src/plugins/indexer/job.ts

@@ -0,0 +1,110 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { JobQueue, OnJob } from '../../base';
+import { readAllDocIdsFromWorkspaceSnapshot } from '../../core/utils/blocksuite';
+import { Models } from '../../models';
+import { IndexerService } from './service';
+
+declare global {
+  interface Jobs {
+    'indexer.indexDoc': {
+      workspaceId: string;
+      docId: string;
+    };
+    'indexer.deleteDoc': {
+      workspaceId: string;
+      docId: string;
+    };
+    'indexer.indexWorkspace': {
+      workspaceId: string;
+    };
+    'indexer.deleteWorkspace': {
+      workspaceId: string;
+    };
+  }
+}
+
+@Injectable()
+export class IndexerJob {
+  private readonly logger = new Logger(IndexerJob.name);
+
+  constructor(
+    private readonly models: Models,
+    private readonly service: IndexerService,
+    private readonly queue: JobQueue
+  ) {}
+
+  @OnJob('indexer.indexDoc')
+  async indexDoc({ workspaceId, docId }: Jobs['indexer.indexDoc']) {
+    // delete the 'indexer.deleteDoc' job from the queue
+    await this.queue.remove(`${workspaceId}/${docId}`, 'indexer.deleteDoc');
+    await this.service.indexDoc(workspaceId, docId);
+  }
+
+  @OnJob('indexer.deleteDoc')
+  async deleteDoc({ workspaceId, docId }: Jobs['indexer.deleteDoc']) {
+    // delete the 'indexer.updateDoc' job from the queue
+    await this.queue.remove(`${workspaceId}/${docId}`, 'indexer.indexDoc');
+    await this.service.deleteDoc(workspaceId, docId);
+  }
+
+  @OnJob('indexer.indexWorkspace')
+  async indexWorkspace({ workspaceId }: Jobs['indexer.indexWorkspace']) {
+    await this.queue.remove(workspaceId, 'indexer.deleteWorkspace');
+    const snapshot = await this.models.doc.getSnapshot(
+      workspaceId,
+      workspaceId
+    );
+    if (!snapshot) {
+      this.logger.warn(`workspace ${workspaceId} not found`);
+      return;
+    }
+    const docIdsInWorkspace = readAllDocIdsFromWorkspaceSnapshot(snapshot.blob);
+    const docIdsInIndexer = await this.service.listDocIds(workspaceId);
+    const docIdsInWorkspaceSet = new Set(docIdsInWorkspace);
+    const docIdsInIndexerSet = new Set(docIdsInIndexer);
+    // diff the docIdsInWorkspace and docIdsInIndexer
+    const missingDocIds = docIdsInWorkspace.filter(
+      docId => !docIdsInIndexerSet.has(docId)
+    );
+    const deletedDocIds = docIdsInIndexer.filter(
+      docId => !docIdsInWorkspaceSet.has(docId)
+    );
+    for (const docId of deletedDocIds) {
+      await this.queue.add(
+        'indexer.deleteDoc',
+        {
+          workspaceId,
+          docId,
+        },
+        {
+          jobId: `${workspaceId}/${docId}`,
+          // the delete job should be higher priority than the update job
+          priority: 0,
+        }
+      );
+    }
+    for (const docId of missingDocIds) {
+      await this.queue.add(
+        'indexer.indexDoc',
+        {
+          workspaceId,
+          docId,
+        },
+        {
+          jobId: `${workspaceId}/${docId}`,
+          priority: 100,
+        }
+      );
+    }
+    this.logger.debug(
+      `indexed workspace ${workspaceId} with ${missingDocIds.length} missing docs and ${deletedDocIds.length} deleted docs`
+    );
+  }
+
+  @OnJob('indexer.deleteWorkspace')
+  async deleteWorkspace({ workspaceId }: Jobs['indexer.deleteWorkspace']) {
+    await this.queue.remove(workspaceId, 'indexer.indexWorkspace');
+    await this.service.deleteWorkspace(workspaceId);
+  }
+}

packages/backend/server/src/plugins/indexer/service.ts

@@ -1,7 +1,13 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { camelCase, chunk, mapKeys, snakeCase } from 'lodash-es';
 
-import { InvalidIndexerInput, SearchProviderNotFound } from '../../base';
+import {
+  EventBus,
+  InvalidIndexerInput,
+  SearchProviderNotFound,
+} from '../../base';
+import { readAllBlocksFromDocSnapshot } from '../../core/utils/blocksuite';
+import { Models } from '../../models';
 import { SearchProviderType } from './config';
 import { SearchProviderFactory } from './factory';
 import {
@@ -30,6 +36,7 @@ import {
   SearchHighlight,
   SearchInput,
   SearchQuery,
+  SearchQueryOccur,
   SearchQueryType,
 } from './types';
 
@@ -99,7 +106,11 @@ export interface SearchNodeWithMeta extends SearchNode {
 export class IndexerService {
   private readonly logger = new Logger(IndexerService.name);
 
-  constructor(private readonly factory: SearchProviderFactory) {}
+  constructor(
+    private readonly models: Models,
+    private readonly factory: SearchProviderFactory,
+    private readonly event: EventBus
+  ) {}
 
   async createTables() {
     let searchProvider: SearchProvider | undefined;
@@ -161,6 +172,204 @@ export class IndexerService {
     return result;
   }
 
+  async listDocIds(workspaceId: string) {
+    const docIds: string[] = [];
+    let cursor: string | undefined;
+    do {
+      const result = await this.search({
+        table: SearchTable.doc,
+        query: {
+          type: SearchQueryType.match,
+          field: 'workspaceId',
+          match: workspaceId,
+        },
+        options: {
+          fields: ['docId'],
+          pagination: {
+            limit: 10000,
+            cursor,
+          },
+        },
+      });
+      docIds.push(...result.nodes.map(node => node.fields.docId[0] as string));
+      cursor = result.nextCursor;
+      this.logger.debug(
+        `get ${result.nodes.length} new / ${docIds.length} total doc ids for workspace ${workspaceId}, nextCursor: ${cursor}`
+      );
+    } while (cursor);
+    return docIds;
+  }
+
+  async indexDoc(
+    workspaceId: string,
+    docId: string,
+    options?: OperationOptions
+  ) {
+    const workspaceSnapshot = await this.models.doc.getSnapshot(
+      workspaceId,
+      workspaceId
+    );
+    if (!workspaceSnapshot) {
+      this.logger.debug(`workspace ${workspaceId} not found`);
+      return;
+    }
+    const docSnapshot = await this.models.doc.getSnapshot(workspaceId, docId);
+    if (!docSnapshot) {
+      this.logger.debug(`doc ${workspaceId}/${docId} not found`);
+      return;
+    }
+    if (docSnapshot.blob.length <= 2) {
+      this.logger.debug(`doc ${workspaceId}/${docId} is empty, skip indexing`);
+      return;
+    }
+    const result = await readAllBlocksFromDocSnapshot(
+      workspaceId,
+      workspaceSnapshot.blob,
+      docId,
+      docSnapshot.blob
+    );
+    if (!result) {
+      this.logger.warn(
+        `parse doc ${workspaceId}/${docId} failed, workspaceSnapshot size: ${workspaceSnapshot.blob.length}, docSnapshot size: ${docSnapshot.blob.length}`
+      );
+      return;
+    }
+    await this.write(
+      SearchTable.doc,
+      [
+        {
+          workspaceId,
+          docId,
+          title: result.title,
+          summary: result.summary,
+          // NOTE(@fengmk): journal is not supported yet
+          // journal: result.journal,
+          createdByUserId: docSnapshot.createdBy ?? '',
+          updatedByUserId: docSnapshot.updatedBy ?? '',
+          createdAt: docSnapshot.createdAt,
+          updatedAt: docSnapshot.updatedAt,
+        },
+      ],
+      options
+    );
+    await this.deleteBlocksByDocId(workspaceId, docId, options);
+    await this.write(
+      SearchTable.block,
+      result.blocks.map(block => ({
+        workspaceId,
+        docId,
+        blockId: block.blockId,
+        content: block.content ?? '',
+        flavour: block.flavour,
+        blob: block.blob,
+        refDocId: block.refDocId,
+        ref: block.ref,
+        parentFlavour: block.parentFlavour,
+        parentBlockId: block.parentBlockId,
+        additional: block.additional
+          ? JSON.stringify(block.additional)
+          : undefined,
+        markdownPreview: block.markdownPreview,
+        createdByUserId: docSnapshot.createdBy ?? '',
+        updatedByUserId: docSnapshot.updatedBy ?? '',
+        createdAt: docSnapshot.createdAt,
+        updatedAt: docSnapshot.updatedAt,
+      })),
+      options
+    );
+    this.event.emit('doc.indexer.updated', {
+      workspaceId,
+      docId,
+    });
+    this.logger.debug(
+      `synced doc ${workspaceId}/${docId} with ${result.blocks.length} blocks`
+    );
+  }
+
+  async deleteDoc(
+    workspaceId: string,
+    docId: string,
+    options?: OperationOptions
+  ) {
+    await this.deleteByQuery(
+      SearchTable.doc,
+      {
+        type: SearchQueryType.boolean,
+        occur: SearchQueryOccur.must,
+        queries: [
+          {
+            type: SearchQueryType.match,
+            field: 'workspaceId',
+            match: workspaceId,
+          },
+          {
+            type: SearchQueryType.match,
+            field: 'docId',
+            match: docId,
+          },
+        ],
+      },
+      options
+    );
+    this.logger.debug(`deleted doc ${workspaceId}/${docId}`);
+    await this.deleteBlocksByDocId(workspaceId, docId, options);
+    this.event.emit('doc.indexer.deleted', {
+      workspaceId,
+      docId,
+    });
+  }
+
+  async deleteBlocksByDocId(
+    workspaceId: string,
+    docId: string,
+    options?: OperationOptions
+  ) {
+    await this.deleteByQuery(
+      SearchTable.block,
+      {
+        type: SearchQueryType.boolean,
+        occur: SearchQueryOccur.must,
+        queries: [
+          {
+            type: SearchQueryType.match,
+            field: 'workspaceId',
+            match: workspaceId,
+          },
+          {
+            type: SearchQueryType.match,
+            field: 'docId',
+            match: docId,
+          },
+        ],
+      },
+      options
+    );
+    this.logger.debug(`deleted all blocks in doc ${workspaceId}/${docId}`);
+  }
+
+  async deleteWorkspace(workspaceId: string, options?: OperationOptions) {
+    await this.deleteByQuery(
+      SearchTable.doc,
+      {
+        type: SearchQueryType.match,
+        field: 'workspaceId',
+        match: workspaceId,
+      },
+      options
+    );
+    this.logger.debug(`deleted all docs in workspace ${workspaceId}`);
+    await this.deleteByQuery(
+      SearchTable.block,
+      {
+        type: SearchQueryType.match,
+        field: 'workspaceId',
+        match: workspaceId,
+      },
+      options
+    );
+    this.logger.debug(`deleted all blocks in workspace ${workspaceId}`);
+  }
+
   async deleteByQuery<T extends SearchTable>(
     table: T,
     query: SearchQuery,