feat(server): blob reconciliation (#15165)

#### PR Dependency Tree


* **PR #15165** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added automated backend maintenance for missing blob metadata
backfill, document-to-blob reference rebuilding, and unreferenced blob
cleanup planning/execution.
* Introduced scheduled batch processing (workspace-paged) and paginated
object-storage listing.
* **Bug Fixes**
* Improved reliability of object-storage reads by treating expected “not
found” results as non-errors.
* Strengthened blob/expired cleanup flows with runtime-driven batching
and reduced coupling to metadata synchronization.
* **Tests**
* Expanded unit and e2e coverage for partial blob metadata and updated
runtime/job cleanup test assertions.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/__tests__/auth/job.spec.ts

@@ -2,17 +2,25 @@ import { ScheduleModule } from '@nestjs/schedule';
 import { TestingModule } from '@nestjs/testing';
 import { PrismaClient } from '@prisma/client';
 import test from 'ava';
+import Sinon from 'sinon';
 
 import { AuthModule, AuthService } from '../../core/auth';
 import { AuthCronJob } from '../../core/auth/job';
+import { BackendRuntimeProvider } from '../../core/backend-runtime';
 import { createTestingModule } from '../utils';
 
 let m: TestingModule;
 let db: PrismaClient;
+const runtime = {
+  cleanupExpiredUserSessions: Sinon.stub(),
+};
 
 test.before(async () => {
   m = await createTestingModule({
     imports: [ScheduleModule.forRoot(), AuthModule],
+    tapModule: builder => {
+      builder.overrideProvider(BackendRuntimeProvider).useValue(runtime);
+    },
   });
 
   db = m.get(PrismaClient);
@@ -32,16 +40,17 @@ test('should clean expired user sessions', async t => {
   let userSessions = await db.userSession.findMany();
   t.is(userSessions.length, 2);
 
-  // no expired sessions
+  runtime.cleanupExpiredUserSessions.reset();
+  runtime.cleanupExpiredUserSessions.resolves(0);
   await job.cleanExpiredUserSessions();
-  userSessions = await db.userSession.findMany();
-  t.is(userSessions.length, 2);
+  t.true(runtime.cleanupExpiredUserSessions.calledOnce);
+  t.deepEqual(runtime.cleanupExpiredUserSessions.firstCall.args, [1000]);
 
-  // clean all expired sessions
-  await db.userSession.updateMany({
-    data: { expiresAt: new Date(Date.now() - 1000) },
-  });
+  runtime.cleanupExpiredUserSessions.reset();
+  runtime.cleanupExpiredUserSessions.onCall(0).resolves(1000);
+  runtime.cleanupExpiredUserSessions.onCall(1).resolves(2);
   await job.cleanExpiredUserSessions();
-  userSessions = await db.userSession.findMany();
-  t.is(userSessions.length, 0);
+  t.is(runtime.cleanupExpiredUserSessions.callCount, 2);
+  t.deepEqual(runtime.cleanupExpiredUserSessions.firstCall.args, [1000]);
+  t.deepEqual(runtime.cleanupExpiredUserSessions.secondCall.args, [1000]);
 });

packages/backend/server/src/__tests__/doc/cron.spec.ts

@@ -1,7 +1,9 @@
 import { ScheduleModule } from '@nestjs/schedule';
 import { PrismaClient } from '@prisma/client';
 import ava, { TestFn } from 'ava';
+import Sinon from 'sinon';
 
+import { BackendRuntimeProvider } from '../../core/backend-runtime';
 import { DocStorageModule } from '../../core/doc';
 import { DocStorageCronJob } from '../../core/doc/job';
 import { createTestingModule, type TestingModule } from '../utils';
@@ -10,14 +12,23 @@ interface Context {
   module: TestingModule;
   db: PrismaClient;
   cronJob: DocStorageCronJob;
+  runtime: { cleanupExpiredSnapshotHistories: Sinon.SinonStub };
 }
 
 const test = ava as TestFn<Context>;
 
 // cleanup database before each test
 test.before(async t => {
+  t.context.runtime = {
+    cleanupExpiredSnapshotHistories: Sinon.stub(),
+  };
   t.context.module = await createTestingModule({
     imports: [ScheduleModule.forRoot(), DocStorageModule],
+    tapModule: builder => {
+      builder
+        .overrideProvider(BackendRuntimeProvider)
+        .useValue(t.context.runtime);
+    },
   });
 
   t.context.db = t.context.module.get(PrismaClient);
@@ -26,6 +37,7 @@ test.before(async t => {
 
 test.beforeEach(async t => {
   await t.context.module.initTestingDB();
+  t.context.runtime.cleanupExpiredSnapshotHistories.reset();
 });
 
 test.after.always(async t => {
@@ -33,7 +45,7 @@ test.after.always(async t => {
 });
 
 test('should be able to cleanup expired history', async t => {
-  const { db } = t.context;
+  const { db, runtime } = t.context;
   const timestamp = Date.now();
 
   // insert expired data
@@ -65,12 +77,10 @@ test('should be able to cleanup expired history', async t => {
   let count = await db.snapshotHistory.count();
   t.is(count, 20);
 
+  runtime.cleanupExpiredSnapshotHistories.onCall(0).resolves(1000);
+  runtime.cleanupExpiredSnapshotHistories.onCall(1).resolves(10);
+
   await t.context.cronJob.cleanExpiredHistories();
 
-  count = await db.snapshotHistory.count();
-  t.is(count, 10);
-
-  const example = await db.snapshotHistory.findFirst();
-  t.truthy(example);
-  t.true(example!.expiredAt > new Date());
+  t.is(runtime.cleanupExpiredSnapshotHistories.callCount, 2);
 });

packages/backend/server/src/__tests__/models/session.spec.ts

@@ -303,28 +303,3 @@ test('should delete userSession fail when sessionId not match', async t => {
   );
   t.is(count, 0);
 });
-
-test('should cleanup expired userSessions', async t => {
-  const user = await t.context.user.create({
-    email: 'test@affine.pro',
-  });
-  const session = await t.context.db.session.create({
-    data: {},
-  });
-  const userSession = await t.context.session.createOrRefreshUserSession(
-    user.id,
-    session.id
-  );
-  await t.context.session.cleanExpiredUserSessions();
-  let count = await t.context.db.userSession.count();
-  t.is(count, 1);
-
-  // Set expiresAt to past time
-  await t.context.db.userSession.update({
-    where: { id: userSession.id },
-    data: { expiresAt: new Date('2022-01-01') },
-  });
-  await t.context.session.cleanExpiredUserSessions();
-  count = await t.context.db.userSession.count();
-  t.is(count, 0);
-});

packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts

@@ -4,6 +4,7 @@ import ava, { TestFn } from 'ava';
 import Sinon from 'sinon';
 
 import { OneDay } from '../../base';
+import { BackendRuntimeProvider } from '../../core/backend-runtime';
 import { StorageModule, WorkspaceBlobStorage } from '../../core/storage';
 import { BlobUploadCleanupJob } from '../../core/storage/job';
 import { MockUser, MockWorkspace } from '../mocks';
@@ -14,13 +15,22 @@ interface Context {
   db: PrismaClient;
   job: BlobUploadCleanupJob;
   storage: WorkspaceBlobStorage;
+  runtime: { cleanupExpiredPendingBlobs: Sinon.SinonStub };
 }
 
 const test = ava as TestFn<Context>;
 
 test.before(async t => {
+  t.context.runtime = {
+    cleanupExpiredPendingBlobs: Sinon.stub(),
+  };
   t.context.module = await createTestingModule({
     imports: [ScheduleModule.forRoot(), StorageModule],
+    tapModule: builder => {
+      builder
+        .overrideProvider(BackendRuntimeProvider)
+        .useValue(t.context.runtime);
+    },
   });
 
   t.context.db = t.context.module.get(PrismaClient);
@@ -30,6 +40,7 @@ test.before(async t => {
 
 test.beforeEach(async t => {
   await t.context.module.initTestingDB();
+  t.context.runtime.cleanupExpiredPendingBlobs.reset();
 });
 
 test.after.always(async t => {
@@ -86,24 +97,14 @@ test('should cleanup expired pending blobs', async t => {
     ],
   });
 
-  const abortSpy = Sinon.stub(
-    t.context.storage,
-    'abortMultipartUpload'
-  ).resolves();
-  const deleteSpy = Sinon.spy(t.context.storage, 'delete');
-  t.teardown(() => {
-    abortSpy.restore();
-    deleteSpy.restore();
+  t.context.runtime.cleanupExpiredPendingBlobs.resolves({
+    scanned: 2,
+    deleted: 2,
+    abortedMultipart: 1,
+    workspaceIds: [workspace.id],
   });
 
   await t.context.job.cleanExpiredPendingBlobs();
 
-  t.is(abortSpy.callCount, 1);
-  t.is(deleteSpy.callCount, 2);
-
-  const remaining = await t.context.db.blob.findMany({
-    where: { workspaceId: workspace.id },
-  });
-  const remainingKeys = remaining.map(record => record.key).sort();
-  t.deepEqual(remainingKeys, ['completed-keep', 'pending-active']);
+  t.true(t.context.runtime.cleanupExpiredPendingBlobs.calledOnce);
 });

packages/backend/server/src/__tests__/workspace/blobs.e2e.ts

@@ -119,6 +119,50 @@ test('should list blobs', async t => {
   t.deepEqual(ret.map(x => x.key).sort(), [hash1, hash2].sort());
 });
 
+test('should keep partial blob metadata listing on DB path without storage scan', async t => {
+  await app.signupV1('u1@affine.pro');
+
+  const workspace = await createWorkspace(app);
+  const storage = app.get(WorkspaceBlobStorage);
+  const rawProvider = (storage as any).provider;
+  const listSpy = Sinon.spy(rawProvider, 'list');
+  t.teardown(() => listSpy.restore());
+
+  const buffer1 = Buffer.from('with metadata');
+  const buffer2 = Buffer.from('without metadata');
+  const key1 = sha256Base64urlWithPadding(buffer1);
+  const key2 = sha256Base64urlWithPadding(buffer2);
+  const config = app.get(Config);
+  const factory = app.get(StorageProviderFactory);
+  const provider = factory.create(config.storages.blob.storage);
+  await provider.put(`${workspace.id}/${key1}`, buffer1, {
+    contentType: 'text/plain',
+    contentLength: buffer1.length,
+  });
+  await provider.put(`${workspace.id}/${key2}`, buffer2, {
+    contentType: 'text/plain',
+    contentLength: buffer2.length,
+  });
+
+  const blobModel = app.get(BlobModel);
+  await blobModel.upsert({
+    workspaceId: workspace.id,
+    key: key1,
+    mime: 'text/plain',
+    size: buffer1.length,
+    status: 'completed',
+    uploadId: null,
+  });
+
+  const listed = await storage.list(workspace.id);
+
+  t.deepEqual(
+    listed.map(blob => blob.key),
+    [key1]
+  );
+  t.true(listSpy.notCalled);
+});
+
 test('should create pending blob upload with graphql fallback', async t => {
   await app.signupV1('u1@affine.pro');
 
@@ -221,10 +265,13 @@ test('should auto delete blobs when workspace is deleted', async t => {
   const blobs = await listBlobs(app, workspace.id);
   t.is(blobs.length, 2);
 
-  const workspaceBlobStorage = Sinon.spy(app.get(WorkspaceBlobStorage));
+  const storage = app.get(WorkspaceBlobStorage);
+  const rawProvider = (storage as any).provider;
+  const listSpy = Sinon.spy(rawProvider, 'list');
+  t.teardown(() => listSpy.restore());
+
   await deleteWorkspace(app, workspace.id);
-  // should not emit workspace.blob.sync event
-  t.is(workspaceBlobStorage.syncBlobMeta.callCount, 0);
+  t.is(listSpy.callCount, 0);
 });
 
 test('should calc blobs size', async t => {