feat(server): trigger workspace embedding (#12328)

fix AI-127

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Added automated event handling for workspace updates and document embedding, streamlining document embedding workflows.
  - Introduced detection and queuing of documents needing embedding, excluding ignored documents.
- **Improvements**
  - Enhanced performance of embedding-related searches by filtering results at the database level.
  - Increased concurrency for embedding job processing to improve throughput.
- **Bug Fixes**
  - Improved error handling and fallback for missing document titles during embedding.
  - Added safeguards to skip invalid embedding jobs based on document identifiers.
- **Tests**
  - Expanded test coverage for document embedding and ignored document filtering.
  - Updated end-to-end tests to use dynamic content for improved reliability.
  - Added synchronization waits in document creation utilities to improve test stability.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/models/copilot-context.ts

@@ -119,6 +119,11 @@ export class CopilotContextModel extends BaseModel {
   }
 
   async hasWorkspaceEmbedding(workspaceId: string, docIds: string[]) {
+    const canEmbedding = await this.checkEmbeddingAvailable();
+    if (!canEmbedding) {
+      return new Set();
+    }
+
     const existsIds = await this.db.aiWorkspaceEmbedding
       .findMany({
         where: {
@@ -238,10 +243,11 @@ export class CopilotContextModel extends BaseModel {
       WHERE
         w."workspace_id" = ${workspaceId}
         AND i."doc_id" IS NULL
+        AND (w."embedding" <=> ${embedding}::vector) <= ${threshold}
       ORDER BY "distance" ASC
       LIMIT ${topK};
     `;
 
-    return similarityChunks.filter(c => Number(c.distance) <= threshold);
+    return similarityChunks;
   }
 }

packages/backend/server/src/models/copilot-workspace.ts

@@ -34,6 +34,33 @@ export class CopilotWorkspaceConfigModel extends BaseModel {
     });
   }
 
+  /**
+   * find docs to embed, excluding ignored and already embedded docs
+   * newer docs will be list first
+   * @param workspaceId id of the workspace
+   * @returns docIds
+   */
+  @Transactional()
+  async findDocsToEmbed(workspaceId: string): Promise<string[]> {
+    const docIds = await this.db.snapshot
+      .findMany({
+        where: {
+          workspaceId,
+          embedding: {
+            is: null,
+          },
+        },
+        select: { id: true },
+      })
+      .then(r => r.map(doc => doc.id));
+
+    const skipDocIds = await this.listIgnoredDocIds(workspaceId).then(
+      r => new Set(r.map(r => r.docId))
+    );
+
+    return docIds.filter(id => !skipDocIds.has(id));
+  }
+
   @Transactional()
   async updateIgnoredDocs(
     workspaceId: string,

packages/backend/server/src/models/workspace.ts

@@ -7,6 +7,7 @@ import { BaseModel } from './base';
 
 declare global {
   interface Events {
+    'workspace.updated': Workspace;
     'workspace.deleted': {
       id: string;
     };
@@ -58,6 +59,9 @@ export class WorkspaceModel extends BaseModel {
     this.logger.debug(
       `Updated workspace ${workspaceId} with data ${JSON.stringify(data)}`
     );
+
+    this.event.emit('workspace.updated', workspace);
+
     return workspace;
   }