feat(server): batch blob gc (#15183)

packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts

@@ -18,6 +18,7 @@ interface Context {
     add: Sinon.SinonStub;
   };
   db: {
+    $queryRaw: Sinon.SinonStub;
     workspace: {
       findMany: Sinon.SinonStub;
     };
@@ -46,6 +47,7 @@ test.beforeEach(t => {
     add: Sinon.stub().resolves(undefined),
   };
   t.context.db = {
+    $queryRaw: Sinon.stub(),
     workspace: {
       findMany: Sinon.stub(),
     },
@@ -81,6 +83,16 @@ const objectStorageRequiredCases: {
       context.event.emitAsync,
     ],
   },
+  {
+    name: 'blob cleanup execution sweep',
+    run: context => context.job.executeBlobCleanupCandidatesByMarkedRuns({}),
+    untouched: context => [
+      context.db.$queryRaw,
+      context.runtime.executeBlobCleanupCandidates,
+      context.event.emitAsync,
+      context.queue.add,
+    ],
+  },
   {
     name: 'blob cleanup planning sweep',
     run: context => context.job.planUnreferencedWorkspaceBlobsBySid({}),
@@ -213,3 +225,123 @@ test('blob cleanup planning drains each workspace cursor before continuing', asy
     )
   );
 });
+
+test('daily blob cleanup execution uses a fixed job id', async t => {
+  await t.context.job.dailyBlobCleanupExecution();
+
+  t.true(
+    t.context.queue.add.calledWith(
+      'backendRuntime.executeBlobCleanupCandidatesByMarkedRuns',
+      {},
+      { jobId: 'daily-backend-runtime-blob-cleanup-execution' }
+    )
+  );
+});
+
+test('blob cleanup execution sweep drains marked runs and continues by page', async t => {
+  t.context.db.$queryRaw
+    .onFirstCall()
+    .resolves([{ runId: 'run-1' }, { runId: 'run-2' }])
+    .onSecondCall()
+    .resolves([{ exists: true }]);
+  t.context.runtime.executeBlobCleanupCandidates
+    .onFirstCall()
+    .resolves({
+      scannedCandidates: 100,
+      deletedObjects: 100,
+      deletedMetadata: 100,
+      skippedStillReferenced: 0,
+      failed: 0,
+      workspaceIds: ['workspace-1'],
+    })
+    .onSecondCall()
+    .resolves({
+      scannedCandidates: 5,
+      deletedObjects: 5,
+      deletedMetadata: 5,
+      skippedStillReferenced: 0,
+      failed: 0,
+      workspaceIds: ['workspace-1'],
+    })
+    .onThirdCall()
+    .resolves({
+      scannedCandidates: 1,
+      deletedObjects: 0,
+      deletedMetadata: 0,
+      skippedStillReferenced: 0,
+      failed: 1,
+      workspaceIds: [],
+    });
+
+  await t.context.job.executeBlobCleanupCandidatesByMarkedRuns({
+    runLimit: 2,
+    gracePeriodDays: 14,
+    candidateLimit: 100,
+  });
+
+  t.is(t.context.runtime.executeBlobCleanupCandidates.callCount, 3);
+  t.deepEqual(t.context.runtime.executeBlobCleanupCandidates.firstCall.args, [
+    'run-1',
+    14,
+    100,
+  ]);
+  t.deepEqual(t.context.runtime.executeBlobCleanupCandidates.secondCall.args, [
+    'run-1',
+    14,
+    100,
+  ]);
+  t.deepEqual(t.context.runtime.executeBlobCleanupCandidates.thirdCall.args, [
+    'run-2',
+    14,
+    100,
+  ]);
+  t.is(t.context.event.emitAsync.callCount, 2);
+  t.true(
+    t.context.queue.add.calledWith(
+      'backendRuntime.executeBlobCleanupCandidatesByMarkedRuns',
+      { runLimit: 2, gracePeriodDays: 14, candidateLimit: 100 }
+    )
+  );
+});
+
+test('blob cleanup execution sweep does not continue failed-only backlog', async t => {
+  t.context.db.$queryRaw
+    .onFirstCall()
+    .resolves([{ runId: 'run-1' }])
+    .onSecondCall()
+    .resolves([{ exists: false }]);
+  t.context.runtime.executeBlobCleanupCandidates.resolves({
+    scannedCandidates: 100,
+    deletedObjects: 0,
+    deletedMetadata: 0,
+    skippedStillReferenced: 0,
+    failed: 100,
+    workspaceIds: [],
+  });
+
+  await t.context.job.executeBlobCleanupCandidatesByMarkedRuns({
+    runLimit: 1,
+    candidateLimit: 100,
+  });
+
+  t.is(t.context.runtime.executeBlobCleanupCandidates.callCount, 1);
+  t.false(t.context.queue.add.called);
+});
+
+test('blob cleanup execution sweep does not continue after drain errors', async t => {
+  t.context.db.$queryRaw
+    .onFirstCall()
+    .resolves([{ runId: 'run-1' }])
+    .onSecondCall()
+    .resolves([{ exists: true }]);
+  t.context.runtime.executeBlobCleanupCandidates.rejects(
+    new Error('storage outage')
+  );
+
+  await t.context.job.executeBlobCleanupCandidatesByMarkedRuns({
+    runLimit: 1,
+  });
+
+  t.is(t.context.runtime.executeBlobCleanupCandidates.callCount, 1);
+  t.false(t.context.queue.add.called);
+});

packages/backend/server/src/core/storage/blob-job.ts

@@ -43,6 +43,11 @@ declare global {
       gracePeriodDays?: number;
       limit?: number;
     };
+    'backendRuntime.executeBlobCleanupCandidatesByMarkedRuns': {
+      runLimit?: number;
+      gracePeriodDays?: number;
+      candidateLimit?: number;
+    };
   }
 }
 
@@ -173,6 +178,21 @@ export class StorageBlobJob {
     });
   }
 
+  async enqueueExecuteBlobCleanupCandidatesByMarkedRuns(
+    runLimit = 10,
+    gracePeriodDays = 30,
+    candidateLimit = 1000
+  ) {
+    await this.queue.add(
+      'backendRuntime.executeBlobCleanupCandidatesByMarkedRuns',
+      {
+        runLimit,
+        gracePeriodDays,
+        candidateLimit,
+      }
+    );
+  }
+
   @Cron(CronExpression.EVERY_DAY_AT_1AM)
   async dailyBlobMetadataBackfill() {
     await this.queue.add(
@@ -200,6 +220,15 @@ export class StorageBlobJob {
     );
   }
 
+  @Cron(CronExpression.EVERY_DAY_AT_4AM)
+  async dailyBlobCleanupExecution() {
+    await this.queue.add(
+      'backendRuntime.executeBlobCleanupCandidatesByMarkedRuns',
+      {},
+      { jobId: 'daily-backend-runtime-blob-cleanup-execution' }
+    );
+  }
+
   @OnJob('backendRuntime.backfillMissingBlobMetadata')
   async backfillMissingBlobMetadata({
     workspaceId,
@@ -357,6 +386,46 @@ export class StorageBlobJob {
     );
   }
 
+  @OnJob('backendRuntime.executeBlobCleanupCandidatesByMarkedRuns')
+  async executeBlobCleanupCandidatesByMarkedRuns({
+    runLimit = 10,
+    gracePeriodDays = 30,
+    candidateLimit = 1000,
+  }: Jobs['backendRuntime.executeBlobCleanupCandidatesByMarkedRuns']) {
+    if (!(await this.hasObjectStorage('blob cleanup execution sweep'))) {
+      return;
+    }
+
+    const normalizedRunLimit = Math.max(1, runLimit);
+    const normalizedCandidateLimit = Math.max(1, candidateLimit);
+    const runIds = await this.loadPendingBlobCleanupRunIds(normalizedRunLimit);
+    let hadDrainError = false;
+    for (const runId of runIds) {
+      try {
+        await this.drainBlobCleanupExecution(
+          runId,
+          gracePeriodDays,
+          normalizedCandidateLimit
+        );
+      } catch (err) {
+        hadDrainError = true;
+        this.logger.error(`blob cleanup execution failed run=${runId}`, err);
+      }
+    }
+
+    if (
+      !hadDrainError &&
+      runIds.length === normalizedRunLimit &&
+      (await this.hasMarkedBlobCleanupCandidates())
+    ) {
+      await this.enqueueExecuteBlobCleanupCandidatesByMarkedRuns(
+        normalizedRunLimit,
+        gracePeriodDays,
+        normalizedCandidateLimit
+      );
+    }
+  }
+
   private async drainBlobMetadataBackfill(
     workspaceId: string,
     limit: number,
@@ -421,6 +490,59 @@ export class StorageBlobJob {
     }
   }
 
+  private async drainBlobCleanupExecution(
+    runId: string,
+    gracePeriodDays: number,
+    limit: number
+  ) {
+    for (;;) {
+      const result = await this.rt.executeBlobCleanupCandidates(
+        runId,
+        gracePeriodDays,
+        limit
+      );
+      await Promise.all(
+        result.workspaceIds.map((workspaceId: string) =>
+          this.event.emitAsync('workspace.blobs.updated', { workspaceId })
+        )
+      );
+      this.logger.log(
+        `executed blob cleanup run=${runId} deleted=${result.deletedObjects} skipped=${result.skippedStillReferenced} failed=${result.failed}`
+      );
+
+      const progressed =
+        result.deletedMetadata > 0 || result.skippedStillReferenced > 0;
+      if (result.scannedCandidates < limit || !progressed) {
+        break;
+      }
+    }
+  }
+
+  private async loadPendingBlobCleanupRunIds(limit: number) {
+    const rows = await this.db.$queryRaw<{ runId: string }[]>`
+      SELECT run_id::text AS "runId"
+      FROM blob_cleanup_candidates
+      WHERE status IN ('marked', 'failed')
+      GROUP BY run_id
+      ORDER BY
+        CASE WHEN BOOL_OR(status = 'marked') THEN 0 ELSE 1 END ASC,
+        MIN(planned_at) ASC
+      LIMIT ${limit}
+    `;
+    return rows.map(row => row.runId);
+  }
+
+  private async hasMarkedBlobCleanupCandidates() {
+    const rows = await this.db.$queryRaw<{ exists: boolean }[]>`
+      SELECT EXISTS(
+        SELECT 1
+        FROM blob_cleanup_candidates
+        WHERE status = 'marked'
+      ) AS "exists"
+    `;
+    return rows[0]?.exists ?? false;
+  }
+
   private async hasObjectStorage(operation: string) {
     const health = await this.rt.health();
     if (health.provider) {