feat(server): impl doc gc (#15282)

#### PR Dependency Tree


* **PR #15282** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added automated document cleanup to reconcile missing workspace docs,
delete related stored data, and recover if the doc returns.
* Added effect-based follow-up reconciliation for search indexing,
Copilot embeddings, and comment attachment cleanup with explicit
acknowledgements.
* **Bug Fixes**
* Deleted-document references now persist as dangling references rather
than disappearing.
* Improved document deletion flow to enforce permissions and ensure
authorized deletions succeed.
* **Tests**
* Expanded coverage for cleanup recovery, indexing/embedding
reconciliation, permissions, and reference semantics.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts

@@ -6,14 +6,17 @@ import Sinon from 'sinon';
 
 import { createModule } from '../../../__tests__/create-module';
 import { Mockers } from '../../../__tests__/mocks';
-import { JOB_SIGNAL } from '../../../base';
+import { Config, JOB_SIGNAL } from '../../../base';
 import { ConfigModule } from '../../../base/config';
 import { ServerConfigModule } from '../../../core/config';
+import { DocReader } from '../../../core/doc';
 import { Models } from '../../../models';
+import { addDocToRootDoc } from '../../../native';
 import { SearchProviderFactory } from '../factory';
 import { IndexerModule, IndexerService } from '../index';
 import { IndexerJob } from '../job';
 import { ManticoresearchProvider } from '../providers';
+import { blockSQL, docSQL, SearchTable } from '../tables';
 
 const module = await createModule({
   imports: [
@@ -32,6 +35,8 @@ const indexerJob = module.get(IndexerJob);
 const searchProviderFactory = module.get(SearchProviderFactory);
 const manticoresearch = module.get(ManticoresearchProvider);
 const models = module.get(Models);
+const docReader = module.get(DocReader);
+const config = module.get(Config);
 
 const user = await module.create(Mockers.User);
 const workspace = await module.create(Mockers.Workspace, {
@@ -39,6 +44,11 @@ const workspace = await module.create(Mockers.Workspace, {
   owner: user,
 });
 
+test.before(async () => {
+  await manticoresearch.recreateTable(SearchTable.block, blockSQL);
+  await manticoresearch.recreateTable(SearchTable.doc, docSQL);
+});
+
 test.after.always(async () => {
   await module.close();
 });
@@ -105,7 +115,7 @@ test('should not sync existing doc', async t => {
   t.is(module.queue.count('indexer.indexDoc'), count);
 });
 
-test('should delete doc from indexer when docId is not in workspace', async t => {
+test('should delete dangling indexed docs absent from the root live set', async t => {
   const count = module.queue.count('indexer.deleteDoc');
   mock.method(indexerService, 'listDocIds', async () => {
     return ['mock-doc-id1', 'mock-doc-id2'];
@@ -121,6 +131,104 @@ test('should delete doc from indexer when docId is not in workspace', async t =>
   t.is(module.queue.count('indexer.deleteDoc'), count + 2);
 });
 
+test('document cleanup reconcile deletes missing search state before ack', async t => {
+  const deleteSpy = Sinon.spy(indexerService, 'deleteDoc');
+  const indexSpy = Sinon.spy(indexerService, 'indexDoc');
+  const cleanupWorkspace = await module.create(Mockers.Workspace, {
+    owner: user,
+  });
+  await module.create(Mockers.DocSnapshot, {
+    workspaceId: cleanupWorkspace.id,
+    docId: cleanupWorkspace.id,
+    user,
+    blob: addDocToRootDoc(Buffer.from([0, 0]), 'live-doc', 'Live'),
+  });
+
+  await indexerJob.reconcileDocumentCleanup({
+    workspaceId: cleanupWorkspace.id,
+    docId: 'missing-doc',
+    cleanupVersion: 'version-1',
+  });
+
+  t.true(deleteSpy.calledOnceWith(cleanupWorkspace.id, 'missing-doc'));
+  t.false(indexSpy.called);
+  const { payload } = await module.queue.waitFor(
+    'backendRuntime.ackDocumentCleanupEffect'
+  );
+  t.deepEqual(payload, {
+    workspaceId: cleanupWorkspace.id,
+    docId: 'missing-doc',
+    cleanupVersion: 'version-1',
+    effect: 'search',
+  });
+});
+
+test('document cleanup reconcile reindexes restored doc before ack', async t => {
+  const deleteSpy = Sinon.spy(indexerService, 'deleteDoc');
+  const indexSpy = Sinon.spy(indexerService, 'indexDoc');
+  const cleanupWorkspace = await module.create(Mockers.Workspace, {
+    owner: user,
+  });
+  await module.create(Mockers.DocSnapshot, {
+    workspaceId: cleanupWorkspace.id,
+    docId: cleanupWorkspace.id,
+    user,
+    blob: addDocToRootDoc(Buffer.from([0, 0]), 'restored-doc', 'Restored'),
+  });
+  await module.create(Mockers.DocSnapshot, {
+    workspaceId: cleanupWorkspace.id,
+    docId: 'restored-doc',
+    user,
+  });
+  const getDocSpy = Sinon.spy(docReader, 'getDoc');
+
+  await indexerJob.reconcileDocumentCleanup({
+    workspaceId: cleanupWorkspace.id,
+    docId: 'restored-doc',
+    cleanupVersion: 'version-2',
+  });
+
+  t.true(indexSpy.calledOnceWith(cleanupWorkspace.id, 'restored-doc'));
+  t.false(deleteSpy.called);
+  t.true(getDocSpy.calledWith(cleanupWorkspace.id, cleanupWorkspace.id));
+  t.true(getDocSpy.calledWith(cleanupWorkspace.id, 'restored-doc'));
+  const { payload } = await module.queue.waitFor(
+    'backendRuntime.ackDocumentCleanupEffect'
+  );
+  t.deepEqual(payload, {
+    workspaceId: cleanupWorkspace.id,
+    docId: 'restored-doc',
+    cleanupVersion: 'version-2',
+    effect: 'search',
+  });
+});
+
+test('document cleanup reconcile only acknowledges when indexer is disabled', async t => {
+  Sinon.stub(config.indexer, 'enabled').value(false);
+  const deleteSpy = Sinon.spy(indexerService, 'deleteDoc');
+  const indexSpy = Sinon.spy(indexerService, 'indexDoc');
+  const getDocSpy = Sinon.spy(docReader, 'getDoc');
+
+  await indexerJob.reconcileDocumentCleanup({
+    workspaceId: workspace.id,
+    docId: 'disabled-doc',
+    cleanupVersion: 'version-disabled',
+  });
+
+  t.false(deleteSpy.called);
+  t.false(indexSpy.called);
+  t.false(getDocSpy.called);
+  const { payload } = await module.queue.waitFor(
+    'backendRuntime.ackDocumentCleanupEffect'
+  );
+  t.deepEqual(payload, {
+    workspaceId: workspace.id,
+    docId: 'disabled-doc',
+    cleanupVersion: 'version-disabled',
+    effect: 'search',
+  });
+});
+
 test('should handle indexer.deleteWorkspace job', async t => {
   const spy = Sinon.spy(indexerService, 'deleteWorkspace');
 

packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts

@@ -1887,12 +1887,9 @@ test('should delete doc work', async t => {
   t.is(result4.nodes.length, 1);
   t.deepEqual(result4.nodes[0].fields.docId, [docId2]);
 
-  const count = module.queue.count('copilot.embedding.deleteDoc');
-
   await indexerService.deleteDoc(workspaceId, docId1, {
     refresh: true,
   });
-  t.is(module.queue.count('copilot.embedding.deleteDoc'), count + 1);
 
   // make sure the docId1 is deleted
   result1 = await indexerService.search({

packages/backend/server/src/plugins/indexer/index.ts

@@ -3,6 +3,7 @@ import './config';
 import { Module } from '@nestjs/common';
 
 import { ServerConfigModule } from '../../core/config';
+import { DocStorageModule } from '../../core/doc';
 import { PermissionModule } from '../../core/permission';
 import { QuotaServiceModule } from '../../core/quota';
 import { IndexerEvent } from './event';
@@ -13,7 +14,12 @@ import { IndexerResolver } from './resolver';
 import { IndexerService } from './service';
 
 @Module({
-  imports: [ServerConfigModule, PermissionModule, QuotaServiceModule],
+  imports: [
+    ServerConfigModule,
+    DocStorageModule,
+    PermissionModule,
+    QuotaServiceModule,
+  ],
   providers: [
     IndexerResolver,
     IndexerService,

packages/backend/server/src/plugins/indexer/job.ts

@@ -1,6 +1,7 @@
 import { Injectable, Logger } from '@nestjs/common';
 
 import { Config, JOB_SIGNAL, JobQueue, OnJob } from '../../base';
+import { DocReader } from '../../core/doc';
 import { readAllDocIdsFromWorkspaceSnapshot } from '../../core/utils/blocksuite';
 import { Models } from '../../models';
 import { IndexerService } from './service';
@@ -24,6 +25,11 @@ declare global {
     'indexer.autoIndexWorkspaces': {
       lastIndexedWorkspaceSid?: number;
     };
+    'indexer.reconcileDocumentCleanup': {
+      workspaceId: string;
+      docId: string;
+      cleanupVersion: string;
+    };
   }
 }
 
@@ -35,7 +41,8 @@ export class IndexerJob {
     private readonly models: Models,
     private readonly service: IndexerService,
     private readonly queue: JobQueue,
-    private readonly config: Config
+    private readonly config: Config,
+    private readonly doc: DocReader
   ) {}
 
   @OnJob('indexer.indexDoc')
@@ -66,6 +73,39 @@ export class IndexerJob {
     await this.service.deleteDoc(workspaceId, docId);
   }
 
+  @OnJob('indexer.reconcileDocumentCleanup')
+  async reconcileDocumentCleanup({
+    workspaceId,
+    docId,
+    cleanupVersion,
+  }: Jobs['indexer.reconcileDocumentCleanup']) {
+    if (this.config.indexer.enabled) {
+      const root = await this.doc.getDoc(workspaceId, workspaceId);
+      if (!root) {
+        throw new Error(`workspace root ${workspaceId} not found`);
+      }
+      const live = readAllDocIdsFromWorkspaceSnapshot(root.bin, true).includes(
+        docId
+      );
+      if (live) {
+        if (!(await this.doc.getDoc(workspaceId, docId))) {
+          throw new Error(
+            `restored document ${workspaceId}/${docId} not found`
+          );
+        }
+        await this.service.indexDoc(workspaceId, docId);
+      } else {
+        await this.service.deleteDoc(workspaceId, docId);
+      }
+    }
+    await this.queue.add('backendRuntime.ackDocumentCleanupEffect', {
+      workspaceId,
+      docId,
+      cleanupVersion,
+      effect: 'search',
+    });
+  }
+
   @OnJob('indexer.indexWorkspace')
   async indexWorkspace({ workspaceId }: Jobs['indexer.indexWorkspace']) {
     if (!this.config.indexer.enabled) {
@@ -79,16 +119,13 @@ export class IndexerJob {
       return;
     }
 
-    const snapshot = await this.models.doc.getSnapshot(
-      workspaceId,
-      workspaceId
-    );
-    if (!snapshot) {
+    const root = await this.doc.getDoc(workspaceId, workspaceId);
+    if (!root) {
       this.logger.warn(`workspace snapshot ${workspaceId} not found`);
       return;
     }
 
-    const docIdsInWorkspace = readAllDocIdsFromWorkspaceSnapshot(snapshot.blob);
+    const docIdsInWorkspace = readAllDocIdsFromWorkspaceSnapshot(root.bin);
     const docIdsInIndexer = await this.service.listDocIds(workspaceId);
 
     const docIdsInWorkspaceSet = new Set(docIdsInWorkspace);

packages/backend/server/src/plugins/indexer/service.ts

@@ -374,14 +374,6 @@ export class IndexerService {
     );
 
     await this.deleteBlocksByDocId(workspaceId, docId, options);
-    await this.queue.add('copilot.session.deleteDoc', {
-      workspaceId,
-      docId,
-    });
-    await this.queue.add('copilot.embedding.deleteDoc', {
-      workspaceId,
-      docId,
-    });
     this.logger.log(`deleted doc ${workspaceId}/${docId}`);
   }