feat(server): impl doc gc (#15282)

#### PR Dependency Tree


* **PR #15282** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added automated document cleanup to reconcile missing workspace docs,
delete related stored data, and recover if the doc returns.
* Added effect-based follow-up reconciliation for search indexing,
Copilot embeddings, and comment attachment cleanup with explicit
acknowledgements.
* **Bug Fixes**
* Deleted-document references now persist as dangling references rather
than disappearing.
* Improved document deletion flow to enforce permissions and ensure
authorized deletions succeed.
* **Tests**
* Expanded coverage for cleanup recovery, indexing/embedding
reconciliation, permissions, and reference semantics.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/core/doc/adapters/workspace.ts

@@ -137,8 +137,8 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
     }));
   }
 
-  async deleteDoc(workspaceId: string, docId: string) {
-    await this.models.doc.delete(workspaceId, docId);
+  async deleteDoc(_workspaceId: string, _docId: string) {
+    return;
   }
 
   async deleteSpace(workspaceId: string) {

packages/backend/server/src/core/storage-runtime/provider.ts

@@ -258,6 +258,33 @@ export class StorageRuntimeProvider
     );
   }
 
+  async reconcileWorkspaceDocuments(workspaceId: string) {
+    return await this.measured('reconcileWorkspaceDocuments', rt =>
+      rt.reconcileWorkspaceDocuments(workspaceId)
+    );
+  }
+
+  async executeDocumentCleanupCandidates(
+    workspaceId: string | null | undefined,
+    gracePeriodDays: number,
+    limit: number
+  ) {
+    return await this.measured('executeDocumentCleanupCandidates', rt =>
+      rt.executeDocumentCleanupCandidates(workspaceId, gracePeriodDays, limit)
+    );
+  }
+
+  async ackDocumentCleanupEffect(
+    workspaceId: string,
+    docId: string,
+    cleanupVersion: string,
+    effect: 'search' | 'copilot'
+  ) {
+    return await this.measured('ackDocumentCleanupEffect', rt =>
+      rt.ackDocumentCleanupEffect(workspaceId, docId, cleanupVersion, effect)
+    );
+  }
+
   async planUnreferencedWorkspaceBlobs(
     workspaceId: string,
     gracePeriodDays: number,

packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts

@@ -6,10 +6,13 @@ import { StorageBlobJob } from '../blob-job';
 interface Context {
   runtime: {
     health: Sinon.SinonStub;
+    reconcileWorkspaceDocuments: Sinon.SinonStub;
     backfillMissingBlobMetadata: Sinon.SinonStub;
     rebuildWorkspaceDocBlobRefs: Sinon.SinonStub;
     planUnreferencedWorkspaceBlobs: Sinon.SinonStub;
     executeBlobCleanupCandidates: Sinon.SinonStub;
+    executeDocumentCleanupCandidates: Sinon.SinonStub;
+    ackDocumentCleanupEffect: Sinon.SinonStub;
   };
   event: {
     emitAsync: Sinon.SinonStub;
@@ -35,10 +38,18 @@ test.beforeEach(t => {
       providerConfigured: true,
       provider: 'fs',
     }),
+    reconcileWorkspaceDocuments: Sinon.stub().resolves({
+      scannedDocs: 1,
+      marked: 0,
+      reset: 0,
+      recovered: 0,
+    }),
     backfillMissingBlobMetadata: Sinon.stub(),
     rebuildWorkspaceDocBlobRefs: Sinon.stub(),
     planUnreferencedWorkspaceBlobs: Sinon.stub(),
     executeBlobCleanupCandidates: Sinon.stub(),
+    executeDocumentCleanupCandidates: Sinon.stub(),
+    ackDocumentCleanupEffect: Sinon.stub(),
   };
   t.context.event = {
     emitAsync: Sinon.stub().resolves(undefined),
@@ -47,7 +58,18 @@ test.beforeEach(t => {
     add: Sinon.stub().resolves(undefined),
   };
   t.context.db = {
-    $queryRaw: Sinon.stub(),
+    $queryRaw: Sinon.stub().resolves([
+      {
+        marked: 0n,
+        failed: 0n,
+        effectsPending: 0n,
+        failedWorkspaceCheckpoints: 0n,
+        rootFailureCheckpoints: 0n,
+        staleProjectionBlocks: 0n,
+        oldestFailedSeconds: null,
+        oldestEffectsPendingSeconds: null,
+      },
+    ]),
     workspace: {
       findMany: Sinon.stub(),
     },
@@ -129,46 +151,6 @@ for (const scenario of objectStorageRequiredCases) {
   });
 }
 
-test('doc blob refs sweep continues after one workspace fails', async t => {
-  t.context.db.workspace.findMany.resolves([
-    { id: 'workspace-1', sid: 1 },
-    { id: 'workspace-2', sid: 2 },
-  ]);
-  t.context.runtime.rebuildWorkspaceDocBlobRefs
-    .onFirstCall()
-    .rejects(new Error('bad root doc'))
-    .onSecondCall()
-    .resolves({
-      scannedDocs: 1,
-      parsedDocs: 1,
-      refsWritten: 0,
-      refsDeleted: 0,
-      failedDocs: 0,
-      nextCursor: null,
-    });
-
-  await t.context.job.rebuildWorkspaceDocBlobRefsBySid({
-    workspaceLimit: 2,
-    docLimit: 100,
-  });
-
-  t.is(t.context.runtime.rebuildWorkspaceDocBlobRefs.callCount, 2);
-  t.deepEqual(t.context.runtime.rebuildWorkspaceDocBlobRefs.firstCall.args, [
-    'workspace-1',
-    100,
-  ]);
-  t.deepEqual(t.context.runtime.rebuildWorkspaceDocBlobRefs.secondCall.args, [
-    'workspace-2',
-    100,
-  ]);
-  t.true(
-    t.context.queue.add.calledWith(
-      'backendRuntime.rebuildWorkspaceDocBlobRefsBySid',
-      { lastSid: 2, workspaceLimit: 2, docLimit: 100 }
-    )
-  );
-});
-
 test('blob cleanup planning drains each workspace cursor before continuing', async t => {
   t.context.db.workspace.findMany.resolves([
     { id: 'workspace-1', sid: 1 },
@@ -229,6 +211,13 @@ test('blob cleanup planning drains each workspace cursor before continuing', asy
 test('daily blob cleanup execution uses a fixed job id', async t => {
   await t.context.job.dailyBlobCleanupExecution();
 
+  t.true(
+    t.context.queue.add.calledWith(
+      'backendRuntime.executeDocumentCleanupCandidates',
+      {},
+      { jobId: 'daily-backend-runtime-document-cleanup-execution' }
+    )
+  );
   t.true(
     t.context.queue.add.calledWith(
       'backendRuntime.executeBlobCleanupCandidatesByMarkedRuns',
@@ -238,6 +227,132 @@ test('daily blob cleanup execution uses a fixed job id', async t => {
   );
 });
 
+test('daily storage reconciliation uses a fixed job id', async t => {
+  await t.context.job.dailyStorageReconciliation();
+
+  t.true(
+    t.context.queue.add.calledWith(
+      'backendRuntime.reconcileWorkspaceStorageBySid',
+      {},
+      { jobId: 'daily-backend-runtime-storage-reconciliation' }
+    )
+  );
+});
+
+test('storage reconciliation orders document retention before blob cleanup', async t => {
+  t.context.db.workspace.findMany.resolves([{ id: 'workspace-1', sid: 1 }]);
+  t.context.runtime.rebuildWorkspaceDocBlobRefs.resolves({
+    scannedDocs: 1,
+    parsedDocs: 1,
+    refsWritten: 1,
+    refsDeleted: 0,
+    failedDocs: 0,
+    nextCursor: null,
+  });
+  t.context.runtime.planUnreferencedWorkspaceBlobs.resolves({
+    runId: 'run-1',
+    scannedBlobs: 1,
+    candidatesMarked: 0,
+    nextCursor: null,
+  });
+
+  await t.context.job.reconcileWorkspaceStorageBySid({ workspaceLimit: 10 });
+
+  Sinon.assert.callOrder(
+    t.context.runtime.reconcileWorkspaceDocuments,
+    t.context.runtime.rebuildWorkspaceDocBlobRefs,
+    t.context.runtime.planUnreferencedWorkspaceBlobs
+  );
+  t.pass();
+});
+
+test('storage reconciliation still refreshes document retention without object storage', async t => {
+  t.context.runtime.health.resolves({
+    databaseConnected: true,
+    providerConfigured: true,
+    provider: undefined,
+  });
+  t.context.db.workspace.findMany.resolves([{ id: 'workspace-1', sid: 1 }]);
+  t.context.runtime.rebuildWorkspaceDocBlobRefs.resolves({
+    scannedDocs: 1,
+    parsedDocs: 1,
+    refsWritten: 0,
+    refsDeleted: 0,
+    failedDocs: 0,
+    nextCursor: null,
+  });
+
+  await t.context.job.reconcileWorkspaceStorageBySid({});
+
+  t.true(t.context.runtime.reconcileWorkspaceDocuments.calledOnce);
+  t.true(t.context.runtime.rebuildWorkspaceDocBlobRefs.calledOnce);
+  t.false(t.context.runtime.planUnreferencedWorkspaceBlobs.called);
+});
+
+test('document cleanup dispatches independent stable search and copilot effects', async t => {
+  t.context.runtime.executeDocumentCleanupCandidates.resolves({
+    scannedCandidates: 1,
+    serializationRetries: 0,
+    executed: 1,
+    recovered: 0,
+    reset: 0,
+    failed: 0,
+    deletedRows: 3,
+    effects: [
+      {
+        workspaceId: 'workspace-1',
+        docId: 'doc-1',
+        cleanupVersion: 'version-1',
+        commentObjectsDone: true,
+        searchDone: false,
+        copilotDone: false,
+      },
+    ],
+  });
+
+  await t.context.job.executeDocumentCleanupCandidates({});
+
+  t.true(
+    t.context.queue.add.calledWith(
+      'indexer.reconcileDocumentCleanup',
+      Sinon.match({ docId: 'doc-1' }),
+      {
+        jobId: 'document-cleanup:search:workspace-1:doc-1:version-1',
+      }
+    )
+  );
+  t.true(
+    t.context.queue.add.calledWith(
+      'copilot.embedding.reconcileDocumentCleanup',
+      Sinon.match({ docId: 'doc-1' }),
+      {
+        jobId: 'document-cleanup:copilot:workspace-1:doc-1:version-1',
+      }
+    )
+  );
+  t.true(
+    t.context.event.emitAsync.calledWith('workspace.blobs.updated', {
+      workspaceId: 'workspace-1',
+    })
+  );
+});
+
+test('document cleanup effect ack delegates to storage runtime', async t => {
+  await t.context.job.ackDocumentCleanupEffect({
+    workspaceId: 'workspace-1',
+    docId: 'doc-1',
+    cleanupVersion: 'version-1',
+    effect: 'search',
+  });
+
+  t.deepEqual(t.context.runtime.ackDocumentCleanupEffect.firstCall.args, [
+    'workspace-1',
+    'doc-1',
+    'version-1',
+    'search',
+  ]);
+});
+
 test('blob cleanup execution sweep drains marked runs and continues by page', async t => {
   t.context.db.$queryRaw
     .onFirstCall()

packages/backend/server/src/core/storage/blob-job.ts

@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
 import { PrismaClient } from '@prisma/client';
 
-import { EventBus, JobQueue, OnJob } from '../../base';
+import { EventBus, JobQueue, metrics, OnJob } from '../../base';
 import { StorageRuntimeProvider } from '../storage-runtime';
 
 // Queue keys are persisted API; keep the legacy backendRuntime.* names while
@@ -18,15 +18,22 @@ declare global {
       workspaceLimit?: number;
       objectLimit?: number;
     };
-    'backendRuntime.rebuildWorkspaceDocBlobRefs': {
-      workspaceId: string;
-      limit?: number;
-    };
-    'backendRuntime.rebuildWorkspaceDocBlobRefsBySid': {
+    'backendRuntime.reconcileWorkspaceStorageBySid': {
       lastSid?: number;
       workspaceLimit?: number;
       docLimit?: number;
     };
+    'backendRuntime.executeDocumentCleanupCandidates': {
+      workspaceId?: string;
+      gracePeriodDays?: number;
+      limit?: number;
+    };
+    'backendRuntime.ackDocumentCleanupEffect': {
+      workspaceId: string;
+      docId: string;
+      cleanupVersion: string;
+      effect: 'search' | 'copilot';
+    };
     'backendRuntime.planUnreferencedWorkspaceBlobs': {
       workspaceId: string;
       gracePeriodDays?: number;
@@ -89,25 +96,6 @@ export class StorageBlobJob {
     });
   }
 
-  async enqueueRebuildWorkspaceDocBlobRefs(workspaceId: string, limit = 1000) {
-    await this.queue.add('backendRuntime.rebuildWorkspaceDocBlobRefs', {
-      workspaceId,
-      limit,
-    });
-  }
-
-  async enqueueRebuildWorkspaceDocBlobRefsBySid(
-    lastSid = 0,
-    workspaceLimit = 100,
-    docLimit = 1000
-  ) {
-    await this.queue.add('backendRuntime.rebuildWorkspaceDocBlobRefsBySid', {
-      lastSid,
-      workspaceLimit,
-      docLimit,
-    });
-  }
-
   @OnJob('backendRuntime.backfillMissingBlobMetadataBySid')
   async backfillMissingBlobMetadataBySid({
     lastSid = 0,
@@ -211,25 +199,21 @@ export class StorageBlobJob {
   }
 
   @Cron(CronExpression.EVERY_DAY_AT_2AM)
-  async dailyDocBlobRefsRebuild() {
+  async dailyStorageReconciliation() {
     await this.queue.add(
-      'backendRuntime.rebuildWorkspaceDocBlobRefsBySid',
+      'backendRuntime.reconcileWorkspaceStorageBySid',
       {},
-      { jobId: 'daily-backend-runtime-doc-blob-refs-rebuild' }
-    );
-  }
-
-  @Cron(CronExpression.EVERY_DAY_AT_3AM)
-  async dailyBlobCleanupPlanning() {
-    await this.queue.add(
-      'backendRuntime.planUnreferencedWorkspaceBlobsBySid',
-      {},
-      { jobId: 'daily-backend-runtime-blob-cleanup-planning' }
+      { jobId: 'daily-backend-runtime-storage-reconciliation' }
     );
   }
 
   @Cron(CronExpression.EVERY_DAY_AT_4AM)
   async dailyBlobCleanupExecution() {
+    await this.queue.add(
+      'backendRuntime.executeDocumentCleanupCandidates',
+      {},
+      { jobId: 'daily-backend-runtime-document-cleanup-execution' }
+    );
     await this.queue.add(
       'backendRuntime.executeBlobCleanupCandidatesByMarkedRuns',
       {},
@@ -249,44 +233,39 @@ export class StorageBlobJob {
     await this.drainBlobMetadataBackfill(workspaceId, limit);
   }
 
-  @OnJob('backendRuntime.rebuildWorkspaceDocBlobRefs')
-  async rebuildWorkspaceDocBlobRefs({
-    workspaceId,
-    limit = 1000,
-  }: Jobs['backendRuntime.rebuildWorkspaceDocBlobRefs']) {
-    await this.drainWorkspaceDocBlobRefs(workspaceId, limit);
-  }
-
-  @OnJob('backendRuntime.rebuildWorkspaceDocBlobRefsBySid')
-  async rebuildWorkspaceDocBlobRefsBySid({
+  @OnJob('backendRuntime.reconcileWorkspaceStorageBySid')
+  async reconcileWorkspaceStorageBySid({
     lastSid = 0,
     workspaceLimit = 100,
     docLimit = 1000,
-  }: Jobs['backendRuntime.rebuildWorkspaceDocBlobRefsBySid']) {
+  }: Jobs['backendRuntime.reconcileWorkspaceStorageBySid']) {
     const workspaces = await this.db.workspace.findMany({
-      where: {
-        sid: {
-          gt: lastSid,
-        },
-      },
-      orderBy: {
-        sid: 'asc',
-      },
-      select: {
-        id: true,
-        sid: true,
-      },
+      where: { sid: { gt: lastSid } },
+      orderBy: { sid: 'asc' },
+      select: { id: true, sid: true },
       take: workspaceLimit,
     });
+    const objectStorageConfigured = await this.hasObjectStorage(
+      'storage reconciliation blob cleanup planning'
+    );
 
     for (const workspace of workspaces) {
       try {
+        await this.rt.reconcileWorkspaceDocuments(workspace.id);
         await this.drainWorkspaceDocBlobRefs(workspace.id, docLimit, {
           sid: workspace.sid,
         });
+        if (objectStorageConfigured) {
+          await this.drainBlobCleanupPlanning(workspace.id, 30, 1000, {
+            sid: workspace.sid,
+          });
+        }
       } catch (err) {
+        metrics.storage
+          .counter('document_cleanup_workspace_failure_total')
+          .add(1);
         this.logger.error(
-          `doc blob refs rebuild failed workspace=${workspace.id} sid=${workspace.sid}`,
+          `storage reconciliation failed workspace=${workspace.id} sid=${workspace.sid}`,
           err
         );
       }
@@ -294,14 +273,71 @@ export class StorageBlobJob {
 
     const nextSid = workspaces.at(-1)?.sid;
     if (nextSid !== undefined && workspaces.length === workspaceLimit) {
-      await this.enqueueRebuildWorkspaceDocBlobRefsBySid(
-        nextSid,
+      await this.queue.add('backendRuntime.reconcileWorkspaceStorageBySid', {
+        lastSid: nextSid,
         workspaceLimit,
-        docLimit
-      );
+        docLimit,
+      });
     }
   }
 
+  @OnJob('backendRuntime.executeDocumentCleanupCandidates')
+  async executeDocumentCleanupCandidates({
+    workspaceId,
+    gracePeriodDays = 30,
+    limit = 100,
+  }: Jobs['backendRuntime.executeDocumentCleanupCandidates']) {
+    const result = await this.rt.executeDocumentCleanupCandidates(
+      workspaceId,
+      gracePeriodDays,
+      limit
+    );
+    metrics.storage
+      .counter('document_cleanup_serialization_retry_total')
+      .add(result.serializationRetries);
+    metrics.storage
+      .counter('document_cleanup_execute_failure_total')
+      .add(result.failed);
+    for (const effect of result.effects) {
+      if (!effect.searchDone) {
+        await this.queue.add('indexer.reconcileDocumentCleanup', effect, {
+          jobId: `document-cleanup:search:${effect.workspaceId}:${effect.docId}:${effect.cleanupVersion}`,
+        });
+      }
+      if (!effect.copilotDone) {
+        await this.queue.add(
+          'copilot.embedding.reconcileDocumentCleanup',
+          effect,
+          {
+            jobId: `document-cleanup:copilot:${effect.workspaceId}:${effect.docId}:${effect.cleanupVersion}`,
+          }
+        );
+      }
+      if (effect.commentObjectsDone) {
+        await this.event.emitAsync('workspace.blobs.updated', {
+          workspaceId: effect.workspaceId,
+        });
+      }
+    }
+    await this.recordDocumentCleanupHealth();
+    return result;
+  }
+
+  @OnJob('backendRuntime.ackDocumentCleanupEffect')
+  async ackDocumentCleanupEffect({
+    workspaceId,
+    docId,
+    cleanupVersion,
+    effect,
+  }: Jobs['backendRuntime.ackDocumentCleanupEffect']) {
+    await this.rt.ackDocumentCleanupEffect(
+      workspaceId,
+      docId,
+      cleanupVersion,
+      effect
+    );
+  }
+
   @OnJob('backendRuntime.planUnreferencedWorkspaceBlobs')
   async planUnreferencedWorkspaceBlobs({
     workspaceId,
@@ -545,6 +581,82 @@ export class StorageBlobJob {
     return rows.map(row => row.runId);
   }
 
+  private async recordDocumentCleanupHealth() {
+    const [health] = await this.db.$queryRaw<
+      {
+        marked: bigint;
+        failed: bigint;
+        effectsPending: bigint;
+        failedWorkspaceCheckpoints: bigint;
+        rootFailureCheckpoints: bigint;
+        staleProjectionBlocks: bigint;
+        oldestFailedSeconds: number | null;
+        oldestEffectsPendingSeconds: number | null;
+      }[]
+    >`
+      SELECT
+        COUNT(*) FILTER (WHERE status = 'marked') AS marked,
+        COUNT(*) FILTER (WHERE status = 'failed') AS failed,
+        COUNT(*) FILTER (WHERE status = 'effects_pending') AS "effectsPending",
+        EXTRACT(EPOCH FROM CURRENT_TIMESTAMP -
+          MIN(updated_at) FILTER (WHERE status = 'failed'))::double precision AS "oldestFailedSeconds",
+        EXTRACT(EPOCH FROM CURRENT_TIMESTAMP -
+          MIN(updated_at) FILTER (WHERE status = 'effects_pending'))::double precision AS "oldestEffectsPendingSeconds",
+        (SELECT COUNT(*) FROM storage_reconciliation_checkpoints
+          WHERE kind = 'document_cleanup' AND status = 'failed') AS "failedWorkspaceCheckpoints",
+        (SELECT COUNT(*) FROM storage_reconciliation_checkpoints
+          WHERE kind = 'document_cleanup' AND status = 'failed'
+            AND metadata->>'failureKind' = 'root') AS "rootFailureCheckpoints",
+        (SELECT COUNT(*) FROM storage_reconciliation_runs
+          WHERE kind = 'blob_cleanup_plan'
+            AND started_at >= CURRENT_TIMESTAMP - INTERVAL '1 day'
+            AND jsonb_array_length(COALESCE(metadata->'staleOrFailedProjectionWorkspaces', '[]')) > 0
+        ) AS "staleProjectionBlocks"
+      FROM document_cleanup_candidates
+    `;
+    if (!health) {
+      return;
+    }
+    for (const [name, value] of [
+      ['document_cleanup_marked', health.marked],
+      ['document_cleanup_failed', health.failed],
+      ['document_cleanup_effects_pending', health.effectsPending],
+      [
+        'document_cleanup_failed_workspace_checkpoints',
+        health.failedWorkspaceCheckpoints,
+      ],
+      [
+        'document_cleanup_root_failure_checkpoints',
+        health.rootFailureCheckpoints,
+      ],
+      [
+        'document_cleanup_stale_projection_blocks',
+        health.staleProjectionBlocks,
+      ],
+    ] as const) {
+      metrics.storage.gauge(name).record(Number(value));
+    }
+    metrics.storage
+      .gauge('document_cleanup_oldest_failed_seconds')
+      .record(health.oldestFailedSeconds ?? 0);
+    metrics.storage
+      .gauge('document_cleanup_oldest_effects_pending_seconds')
+      .record(health.oldestEffectsPendingSeconds ?? 0);
+
+    if (
+      health.failedWorkspaceCheckpoints > 0n ||
+      health.rootFailureCheckpoints > 0n ||
+      health.staleProjectionBlocks > 0n ||
+      (health.oldestFailedSeconds ?? 0) >= 86_400 ||
+      (health.oldestEffectsPendingSeconds ?? 0) >= 86_400 ||
+      health.marked >= 10_000n
+    ) {
+      this.logger.warn(
+        `document cleanup health marked=${health.marked} failed=${health.failed} effectsPending=${health.effectsPending} failedWorkspaceCheckpoints=${health.failedWorkspaceCheckpoints} rootFailureCheckpoints=${health.rootFailureCheckpoints} staleProjectionBlocks=${health.staleProjectionBlocks} oldestFailedSeconds=${health.oldestFailedSeconds ?? 0} oldestEffectsPendingSeconds=${health.oldestEffectsPendingSeconds ?? 0}`
+      );
+    }
+  }
+
   private async hasMarkedBlobCleanupCandidates() {
     const rows = await this.db.$queryRaw<{ exists: boolean }[]>`
       SELECT EXISTS(

packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts

@@ -1,5 +1,6 @@
 import test from 'ava';
 import { omit } from 'lodash-es';
+import * as Y from 'yjs';
 
 import { createModule } from '../../../__tests__/create-module';
 import { Mockers } from '../../../__tests__/mocks';
@@ -38,6 +39,86 @@ test('can read all doc ids from workspace snapshot', async t => {
   t.snapshot(docIds);
 });
 
+test('merged root updates retain trash and exclude permanently removed docs', async t => {
+  const rootDoc = await models.doc.get(workspace.id, workspace.id);
+  t.truthy(rootDoc);
+
+  const root = new Y.Doc();
+  Y.applyUpdate(root, rootDoc!.blob);
+  const pending = new Y.Doc();
+  Y.applyUpdate(pending, Y.encodeStateAsUpdate(root));
+  const trashMeta = new Y.Map<unknown>();
+  trashMeta.set('id', 'trash-doc');
+  trashMeta.set('title', 'Trash');
+  trashMeta.set('trash', true);
+  (pending.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>).push([
+    trashMeta,
+  ]);
+  const pendingUpdate = Y.encodeStateAsUpdate(
+    pending,
+    Y.encodeStateVector(root)
+  );
+  const merged = Y.mergeUpdates([rootDoc!.blob, pendingUpdate]);
+
+  t.false(readAllDocIdsFromWorkspaceSnapshot(merged).includes('trash-doc'));
+  t.true(
+    readAllDocIdsFromWorkspaceSnapshot(merged, true).includes('trash-doc')
+  );
+
+  const removed = new Y.Doc();
+  Y.applyUpdate(removed, merged);
+  const pages = removed.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>;
+  const target = pages
+    .toArray()
+    .findIndex(meta => meta instanceof Y.Map && meta.get('id') === 'trash-doc');
+  pages.delete(target, 1);
+  t.false(
+    readAllDocIdsFromWorkspaceSnapshot(
+      Y.encodeStateAsUpdate(removed),
+      true
+    ).includes('trash-doc')
+  );
+});
+
+test('nested concurrent meta edits do not restore a deleted entry', t => {
+  const initial = new Y.Doc();
+  const pages = new Y.Array<Y.Map<unknown>>();
+  const meta = new Y.Map<unknown>();
+  meta.set('id', 'doc-1');
+  meta.set('title', 'Initial');
+  pages.push([meta]);
+  initial.getMap('meta').set('pages', pages);
+  const state = Y.encodeStateAsUpdate(initial);
+
+  const deletingClient = new Y.Doc();
+  const editingClient = new Y.Doc();
+  Y.applyUpdate(deletingClient, state);
+  Y.applyUpdate(editingClient, state);
+  (
+    deletingClient.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>
+  ).delete(0, 1);
+  (editingClient.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>)
+    .get(0)
+    .set('title', 'Offline edit');
+
+  const merged = new Y.Doc();
+  Y.applyUpdate(merged, Y.encodeStateAsUpdate(deletingClient));
+  Y.applyUpdate(merged, Y.encodeStateAsUpdate(editingClient));
+  t.is(
+    (merged.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>).length,
+    0
+  );
+
+  (editingClient.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>).push([
+    meta.clone(),
+  ]);
+  Y.applyUpdate(merged, Y.encodeStateAsUpdate(editingClient));
+  t.is(
+    (merged.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>).length,
+    1
+  );
+});
+
 test('can read all blocks from doc snapshot', async t => {
   const rootDoc = await models.doc.get(workspace.id, workspace.id);
   t.truthy(rootDoc);

packages/backend/server/src/core/utils/blocksuite.ts

@@ -45,8 +45,11 @@ export function parsePageDoc(
   );
 }
 
-export function readAllDocIdsFromWorkspaceSnapshot(snapshot: Uint8Array) {
-  return readAllDocIdsFromRootDoc(Buffer.from(snapshot), false);
+export function readAllDocIdsFromWorkspaceSnapshot(
+  snapshot: Uint8Array,
+  includeTrash = false
+) {
+  return readAllDocIdsFromRootDoc(Buffer.from(snapshot), includeTrash);
 }
 
 function safeParseJson<T>(str: string): T | undefined {