feat(server): workspace embedding improve (#12022)

fix AI-10
fix AI-109
fix PD-2484

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Added a method to check if a document requires embedding, improving embedding efficiency.
  - Enhanced document embeddings with enriched metadata, including title, summary, creation/update dates, and author information.
  - Introduced a new type for document fragments with extended metadata fields.

- **Improvements**
  - Embedding logic now conditionally processes only documents needing updates.
  - Embedding content now includes document metadata for more informative context.
  - Expanded and improved test coverage for embedding scenarios and workspace behaviors.
  - Event emission added for workspace embedding updates on client version mismatch.
  - Job queueing enhanced with prioritization and explicit job IDs for better management.
  - Job queue calls updated to include priority and context identifiers in a structured format.

- **Bug Fixes**
  - Improved handling of ignored documents in embedding matches.
  - Fixed incorrect document ID assignment in embedding job queueing.

- **Tests**
  - Added and updated snapshot and behavioral tests for embedding and workspace document handling.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/__tests__/models/__snapshots__/copilot-context.spec.ts.md

@@ -4,19 +4,52 @@ The actual snapshot is saved in `copilot-context.spec.ts.snap`.
 
 Generated by [AVA](https://avajs.dev).
 
+## should get null for non-exist job
+
+> should return null for non-exist job
+
+    null
+
 ## should insert embedding by doc id
 
 > should match file embedding
 
     [
       {
-        chunk: 0,
-        content: 'content',
-        distance: 0,
         fileId: 'file-id',
       },
     ]
 
+> should return empty array when embedding is deleted
+
+    []
+
+> should match workspace embedding
+
+    [
+      {
+        docId: 'doc1',
+      },
+    ]
+
+> should return empty array when doc is ignored
+
+    []
+
+> should return workspace embedding
+
+    [
+      {
+        docId: 'doc1',
+      },
+    ]
+
 > should return empty array when embedding deleted
 
     []
+
+## should check embedding table
+
+> should return true when embedding table is available
+
+    true

packages/backend/server/src/__tests__/models/copilot-context.spec.ts

@@ -6,9 +6,11 @@ import ava, { TestFn } from 'ava';
 import { Config } from '../../base';
 import { CopilotContextModel } from '../../models/copilot-context';
 import { CopilotSessionModel } from '../../models/copilot-session';
+import { CopilotWorkspaceConfigModel } from '../../models/copilot-workspace';
 import { UserModel } from '../../models/user';
 import { WorkspaceModel } from '../../models/workspace';
 import { createTestingModule, type TestingModule } from '../utils';
+import { cleanObject } from '../utils/copilot';
 
 interface Context {
   config: Config;
@@ -18,6 +20,7 @@ interface Context {
   workspace: WorkspaceModel;
   copilotSession: CopilotSessionModel;
   copilotContext: CopilotContextModel;
+  copilotWorkspace: CopilotWorkspaceConfigModel;
 }
 
 const test = ava as TestFn<Context>;
@@ -28,6 +31,7 @@ test.before(async t => {
   t.context.workspace = module.get(WorkspaceModel);
   t.context.copilotSession = module.get(CopilotSessionModel);
   t.context.copilotContext = module.get(CopilotContextModel);
+  t.context.copilotWorkspace = module.get(CopilotWorkspaceConfigModel);
   t.context.db = module.get(PrismaClient);
   t.context.config = module.get(Config);
   t.context.module = module;
@@ -74,7 +78,7 @@ test('should create a copilot context', async t => {
 
 test('should get null for non-exist job', async t => {
   const job = await t.context.copilotContext.get('non-exist');
-  t.is(job, null);
+  t.snapshot(job, 'should return null for non-exist job');
 });
 
 test('should update context', async t => {
@@ -111,7 +115,10 @@ test('should insert embedding by doc id', async t => {
         1,
         1
       );
-      t.snapshot(ret, 'should match file embedding');
+      t.snapshot(
+        cleanObject(ret, ['chunk', 'content', 'distance']),
+        'should match file embedding'
+      );
     }
 
     {
@@ -122,7 +129,7 @@ test('should insert embedding by doc id', async t => {
         1,
         1
       );
-      t.is(ret.length, 0);
+      t.snapshot(ret, 'should return empty array when embedding is deleted');
     }
   }
 
@@ -155,7 +162,7 @@ test('should insert embedding by doc id', async t => {
         workspace.id,
         [docId]
       );
-      t.true(ret.has(docId), 'should return true when embedding exists');
+      t.true(ret.has(docId), 'should return doc id when embedding is inserted');
     }
 
     {
@@ -165,8 +172,39 @@ test('should insert embedding by doc id', async t => {
         1,
         1
       );
-      t.is(ret.length, 1);
-      t.is(ret[0].content, 'content');
+      t.snapshot(
+        cleanObject(ret, ['chunk', 'content', 'distance']),
+        'should match workspace embedding'
+      );
+    }
+
+    {
+      await t.context.copilotWorkspace.updateIgnoredDocs(workspace.id, [docId]);
+      const ret = await t.context.copilotContext.matchWorkspaceEmbedding(
+        Array.from({ length: 1024 }, () => 0.9),
+        workspace.id,
+        1,
+        1
+      );
+      t.snapshot(ret, 'should return empty array when doc is ignored');
+    }
+
+    {
+      await t.context.copilotWorkspace.updateIgnoredDocs(
+        workspace.id,
+        undefined,
+        [docId]
+      );
+      const ret = await t.context.copilotContext.matchWorkspaceEmbedding(
+        Array.from({ length: 1024 }, () => 0.9),
+        workspace.id,
+        1,
+        1
+      );
+      t.snapshot(
+        cleanObject(ret, ['chunk', 'content', 'distance']),
+        'should return workspace embedding'
+      );
     }
 
     {
@@ -188,7 +226,7 @@ test('should insert embedding by doc id', async t => {
 test('should check embedding table', async t => {
   {
     const ret = await t.context.copilotContext.checkEmbeddingAvailable();
-    t.true(ret, 'should return true when embedding table is available');
+    t.snapshot(ret, 'should return true when embedding table is available');
   }
 
   // {

packages/backend/server/src/__tests__/models/copilot-workspace.spec.ts

@@ -201,6 +201,68 @@ test('should insert and search embedding', async t => {
   }
 });
 
+test('should check need to be embedded', async t => {
+  const docId = randomUUID();
+
+  await t.context.doc.upsert({
+    spaceId: workspace.id,
+    docId,
+    blob: Uint8Array.from([1, 2, 3]),
+    timestamp: Date.now(),
+    editorId: user.id,
+  });
+
+  {
+    let needsEmbedding = await t.context.copilotWorkspace.checkDocNeedEmbedded(
+      workspace.id,
+      docId
+    );
+    t.true(needsEmbedding, 'document with no embedding should need embedding');
+  }
+
+  {
+    await t.context.copilotContext.insertWorkspaceEmbedding(
+      workspace.id,
+      docId,
+      [
+        {
+          index: 0,
+          content: 'content',
+          embedding: Array.from({ length: 1024 }, () => 1),
+        },
+      ]
+    );
+
+    let needsEmbedding = await t.context.copilotWorkspace.checkDocNeedEmbedded(
+      workspace.id,
+      docId
+    );
+    t.false(
+      needsEmbedding,
+      'document with recent embedding should not need embedding'
+    );
+  }
+
+  {
+    await t.context.doc.upsert({
+      spaceId: workspace.id,
+      docId,
+      blob: Uint8Array.from([4, 5, 6]),
+      timestamp: Date.now() + 1000, // Ensure timestamp is later
+      editorId: user.id,
+    });
+
+    let needsEmbedding = await t.context.copilotWorkspace.checkDocNeedEmbedded(
+      workspace.id,
+      docId
+    );
+    t.true(
+      needsEmbedding,
+      'document updated after embedding should need embedding'
+    );
+  }
+});
+
 test('should check embedding table', async t => {
   {
     const ret = await t.context.copilotWorkspace.checkEmbeddingAvailable();