feat(server): cleanup trashed doc's embedding (#13201)

fix AI-359

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added automated cleanup of embeddings for documents deleted or trashed
from workspaces.
* Introduced a new job to schedule and perform this cleanup per
workspace daily and on demand.
  * Added new GraphQL mutation to manually trigger the cleanup process.
* Added the ability to list workspaces with flexible filtering and
selection options.

* **Improvements**
* Enhanced document status handling to more accurately reflect embedding
presence.
* Refined internal methods for managing and checking document
embeddings.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/models/copilot-context.ts

@@ -84,11 +84,17 @@ export class CopilotContextModel extends BaseModel {
   }
 
   async mergeDocStatus(workspaceId: string, docs: ContextDoc[]) {
-    const docIds = Array.from(new Set(docs.map(doc => doc.id)));
-    const finishedDoc = await this.hasWorkspaceEmbedding(workspaceId, docIds);
+    const canEmbedding = await this.checkEmbeddingAvailable();
+    const finishedDoc = canEmbedding
+      ? await this.listWorkspaceEmbedding(
+          workspaceId,
+          Array.from(new Set(docs.map(doc => doc.id)))
+        )
+      : [];
+    const finishedDocSet = new Set(finishedDoc);
 
     for (const doc of docs) {
-      const status = finishedDoc.has(doc.id)
+      const status = finishedDocSet.has(doc.id)
         ? ContextEmbedStatus.finished
         : undefined;
       // NOTE: when the document has not been synchronized to the server or is in the embedding queue
@@ -120,24 +126,17 @@ export class CopilotContextModel extends BaseModel {
     return Number(count) === 2;
   }
 
-  async hasWorkspaceEmbedding(workspaceId: string, docIds: string[]) {
-    const canEmbedding = await this.checkEmbeddingAvailable();
-    if (!canEmbedding) {
-      return new Set();
-    }
-
+  async listWorkspaceEmbedding(workspaceId: string, docIds?: string[]) {
     const existsIds = await this.db.aiWorkspaceEmbedding
-      .findMany({
+      .groupBy({
         where: {
           workspaceId,
-          docId: { in: docIds },
-        },
-        select: {
-          docId: true,
+          docId: docIds ? { in: docIds } : undefined,
         },
+        by: ['docId'],
       })
       .then(r => r.map(r => r.docId));
-    return new Set(existsIds);
+    return existsIds;
   }
 
   private processEmbeddings(

packages/backend/server/src/models/workspace.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@nestjs/common';
 import { Transactional } from '@nestjs-cls/transactional';
-import { type Workspace } from '@prisma/client';
+import { Prisma, type Workspace } from '@prisma/client';
 
 import { EventBus } from '../base';
 import { BaseModel } from './base';
@@ -93,6 +93,19 @@ export class WorkspaceModel extends BaseModel {
     });
   }
 
+  async list<S extends Prisma.WorkspaceSelect>(
+    where: Prisma.WorkspaceWhereInput = {},
+    select?: S
+  ) {
+    return (await this.db.workspace.findMany({
+      where,
+      select,
+      orderBy: {
+        sid: 'asc',
+      },
+    })) as Prisma.WorkspaceGetPayload<{ select: S }>[];
+  }
+
   async delete(workspaceId: string) {
     const rawResult = await this.db.workspace.deleteMany({
       where: {