feat(core): unused blob management in settings (#9795)

fix AF-2144, PD-2064, PD-2065, PD-2066

packages/frontend/core/src/modules/blob-management/entity/unused-blobs.ts

@@ -0,0 +1,164 @@
+import type { ListedBlobRecord } from '@affine/nbstore';
+import {
+  effect,
+  Entity,
+  fromPromise,
+  LiveData,
+  onComplete,
+  onStart,
+} from '@toeverything/infra';
+import { fileTypeFromBuffer } from 'file-type';
+import { EMPTY, mergeMap, switchMap } from 'rxjs';
+
+import type { DocsSearchService } from '../../docs-search';
+import type { WorkspaceService } from '../../workspace';
+import type { WorkspaceFlavoursService } from '../../workspace/services/flavours';
+
+interface HydratedBlobRecord extends ListedBlobRecord, Disposable {
+  url: string;
+  extension?: string;
+  type?: string;
+}
+
+export class UnusedBlobs extends Entity {
+  constructor(
+    private readonly flavoursService: WorkspaceFlavoursService,
+    private readonly workspaceService: WorkspaceService,
+    private readonly docsSearchService: DocsSearchService
+  ) {
+    super();
+  }
+
+  isLoading$ = new LiveData(false);
+  unusedBlobs$ = new LiveData<ListedBlobRecord[]>([]);
+
+  readonly revalidate = effect(
+    switchMap(() =>
+      fromPromise(async () => {
+        return await this.getUnusedBlobs();
+      }).pipe(
+        mergeMap(data => {
+          this.unusedBlobs$.setValue(data);
+          return EMPTY;
+        }),
+        onStart(() => this.isLoading$.setValue(true)),
+        onComplete(() => this.isLoading$.setValue(false))
+      )
+    )
+  );
+
+  private get flavourProvider() {
+    return this.flavoursService.flavours$.value.find(
+      f => f.flavour === this.workspaceService.workspace.flavour
+    );
+  }
+
+  private get localFlavourProvider() {
+    return this.flavoursService.flavours$.value.find(
+      f => f.flavour === 'local'
+    );
+  }
+
+  async listBlobs() {
+    const blobs = await this.flavourProvider?.listBlobs(
+      this.workspaceService.workspace.id
+    );
+    return blobs;
+  }
+
+  async getBlob(blobKey: string) {
+    const blob = await this.flavourProvider?.getWorkspaceBlob(
+      this.workspaceService.workspace.id,
+      blobKey
+    );
+    return blob;
+  }
+
+  async deleteBlob(blob: string, permanent: boolean) {
+    await this.flavourProvider?.deleteBlob(
+      this.workspaceService.workspace.id,
+      blob,
+      permanent
+    );
+
+    if (this.localFlavourProvider !== this.flavourProvider) {
+      await this.localFlavourProvider?.deleteBlob(
+        this.workspaceService.workspace.id,
+        blob,
+        permanent
+      );
+    }
+  }
+
+  async getUnusedBlobs(abortSignal?: AbortSignal) {
+    // wait for the indexer to finish
+    await this.docsSearchService.indexer.status$.waitFor(
+      status => status.remaining === undefined || status.remaining === 0,
+      abortSignal
+    );
+
+    const [blobs, usedBlobs] = await Promise.all([
+      this.listBlobs(),
+      this.getUsedBlobs(),
+    ]);
+
+    // ignore the workspace avatar
+    const workspaceAvatar = this.workspaceService.workspace.avatar$.value;
+
+    return (
+      blobs?.filter(
+        blob => !usedBlobs.includes(blob.key) && blob.key !== workspaceAvatar
+      ) ?? []
+    );
+  }
+
+  private async getUsedBlobs(): Promise<string[]> {
+    const result = await this.docsSearchService.indexer.blockIndex.aggregate(
+      {
+        type: 'boolean',
+        occur: 'must',
+        queries: [
+          {
+            type: 'exists',
+            field: 'blob',
+          },
+        ],
+      },
+      'blob'
+    );
+    return result.buckets.map(bucket => bucket.key);
+  }
+
+  async hydrateBlob(
+    record: ListedBlobRecord,
+    abortSignal?: AbortSignal
+  ): Promise<HydratedBlobRecord | null> {
+    const blob = await this.getBlob(record.key);
+
+    if (!blob || abortSignal?.aborted) {
+      return null;
+    }
+
+    const fileType = await fileTypeFromBuffer(await blob.arrayBuffer());
+
+    if (abortSignal?.aborted) {
+      return null;
+    }
+
+    const url = URL.createObjectURL(new Blob([blob]));
+    const mime = record.mime || fileType?.mime || 'unknown';
+    // todo(@pengx17): the following may not be sufficient
+    const extension = fileType?.ext;
+    const type = extension ?? (mime?.startsWith('text/') ? 'txt' : 'unknown');
+    return {
+      ...record,
+      url,
+      extension,
+      type,
+      mime,
+      [Symbol.dispose]: () => {
+        URL.revokeObjectURL(url);
+      },
+    };
+  }
+}

packages/frontend/core/src/modules/blob-management/index.ts

@@ -0,0 +1,18 @@
+import { type Framework } from '@toeverything/infra';
+
+import { DocsSearchService } from '../docs-search';
+import { WorkspaceScope, WorkspaceService } from '../workspace';
+import { WorkspaceFlavoursService } from '../workspace/services/flavours';
+import { UnusedBlobs } from './entity/unused-blobs';
+import { BlobManagementService } from './services';
+
+export function configureBlobManagementModule(framework: Framework) {
+  framework
+    .scope(WorkspaceScope)
+    .entity(UnusedBlobs, [
+      WorkspaceFlavoursService,
+      WorkspaceService,
+      DocsSearchService,
+    ])
+    .service(BlobManagementService);
+}

packages/frontend/core/src/modules/blob-management/services/index.ts

@@ -0,0 +1,11 @@
+import { Service } from '@toeverything/infra';
+
+import { UnusedBlobs } from '../entity/unused-blobs';
+
+export class BlobManagementService extends Service {
+  constructor() {
+    super();
+  }
+
+  unusedBlobs = this.framework.createEntity(UnusedBlobs);
+}

packages/frontend/core/src/modules/index.ts

@@ -7,6 +7,7 @@ import {
 } from './ai-button';
 import { configureAppSidebarModule } from './app-sidebar';
 import { configAtMenuConfigModule } from './at-menu-config';
+import { configureBlobManagementModule } from './blob-management';
 import { configureCloudModule } from './cloud';
 import { configureCollectionModule } from './collection';
 import { configureWorkspaceDBModule } from './db';
@@ -98,4 +99,5 @@ export function configureCommonModules(framework: Framework) {
   configureAINetworkSearchModule(framework);
   configureAIButtonModule(framework);
   configureTemplateDocModule(framework);
+  configureBlobManagementModule(framework);
 }

packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts

@@ -5,7 +5,11 @@ import {
   getWorkspaceInfoQuery,
   getWorkspacesQuery,
 } from '@affine/graphql';
-import type { BlobStorage, DocStorage } from '@affine/nbstore';
+import type {
+  BlobStorage,
+  DocStorage,
+  ListedBlobRecord,
+} from '@affine/nbstore';
 import { CloudBlobStorage, StaticCloudDocStorage } from '@affine/nbstore/cloud';
 import {
   IndexedDBBlobStorage,
@@ -364,6 +368,26 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider {
     return new Blob([cloudBlob.data], { type: cloudBlob.mime });
   }
 
+  async listBlobs(id: string): Promise<ListedBlobRecord[]> {
+    const cloudStorage = new CloudBlobStorage({
+      id,
+      serverBaseUrl: this.server.serverMetadata.baseUrl,
+    });
+    return cloudStorage.list();
+  }
+
+  async deleteBlob(
+    id: string,
+    blob: string,
+    permanent: boolean
+  ): Promise<void> {
+    const cloudStorage = new CloudBlobStorage({
+      id,
+      serverBaseUrl: this.server.serverMetadata.baseUrl,
+    });
+    await cloudStorage.delete(blob, permanent);
+  }
+
   onWorkspaceInitialized(workspace: Workspace): void {
     // bind the workspace to the affine cloud server
     workspace.scope.get(WorkspaceServerService).bindServer(this.server);

packages/frontend/core/src/modules/workspace-engine/impls/local.ts

@@ -2,6 +2,7 @@ import { DebugLogger } from '@affine/debug';
 import {
   type BlobStorage,
   type DocStorage,
+  type ListedBlobRecord,
   universalId,
 } from '@affine/nbstore';
 import {
@@ -276,6 +277,33 @@ class LocalWorkspaceFlavourProvider implements WorkspaceFlavourProvider {
     return blob ? new Blob([blob.data], { type: blob.mime }) : null;
   }
 
+  async listBlobs(id: string): Promise<ListedBlobRecord[]> {
+    const storage = new this.BlobStorageType({
+      id: id,
+      flavour: this.flavour,
+      type: 'workspace',
+    });
+    storage.connection.connect();
+    await storage.connection.waitForConnected();
+
+    return storage.list();
+  }
+
+  async deleteBlob(
+    id: string,
+    blob: string,
+    permanent: boolean
+  ): Promise<void> {
+    const storage = new this.BlobStorageType({
+      id: id,
+      flavour: this.flavour,
+      type: 'workspace',
+    });
+    storage.connection.connect();
+    await storage.connection.waitForConnected();
+    await storage.delete(blob, permanent);
+  }
+
   getEngineWorkerInitOptions(workspaceId: string): WorkerInitOptions {
     return {
       local: {

packages/frontend/core/src/modules/workspace/providers/flavour.ts

@@ -1,4 +1,8 @@
-import type { BlobStorage, DocStorage } from '@affine/nbstore';
+import type {
+  BlobStorage,
+  DocStorage,
+  ListedBlobRecord,
+} from '@affine/nbstore';
 import type { WorkerInitOptions } from '@affine/nbstore/worker/client';
 import type { Workspace as BSWorkspace } from '@blocksuite/affine/store';
 import { createIdentifier, type LiveData } from '@toeverything/infra';
@@ -41,6 +45,14 @@ export interface WorkspaceFlavourProvider {
 
   getWorkspaceBlob(id: string, blob: string): Promise<Blob | null>;
 
+  listBlobs(workspaceId: string): Promise<ListedBlobRecord[]>;
+
+  deleteBlob(
+    workspaceId: string,
+    blob: string,
+    permanent: boolean
+  ): Promise<void>;
+
   getEngineWorkerInitOptions(workspaceId: string): WorkerInitOptions;
 
   onWorkspaceInitialized?(workspace: Workspace): void;