feat(core): add matched context documents to ai prompt (#11148)

Close [BS-2834](https://linear.app/affine-design/issue/BS-2834).

### What Changed?
- Change `reference_index` from chip order to increasing positive integer.
- Add matched context documents to ai prompt.

packages/backend/server/src/__tests__/copilot-provider.spec.ts

@@ -311,10 +311,9 @@ const actions = [
           files: [
             {
               blobId: 'euclidean_distance',
-              refIndex: 1,
               fileName: 'euclidean_distance.rs',
               fileType: 'text/rust',
-              chunks: TestAssets.Code,
+              fileContent: TestAssets.Code,
             },
           ],
         },
@@ -339,10 +338,9 @@ const actions = [
           files: [
             {
               blobId: 'SSOT',
-              refIndex: 1,
               fileName: 'Single source of truth - Wikipedia',
               fileType: 'text/markdown',
-              chunks: TestAssets.SSOT,
+              fileContent: TestAssets.SSOT,
             },
           ],
         },

packages/backend/server/src/plugins/copilot/prompt/prompts.ts

@@ -1012,13 +1012,12 @@ Use the structure of the fragments to assess their relevance and provide the nec
 ## Content fragments format:
 - Document fragments, identified by a \`document_id\` and containing \`document_content\`.
 - File fragments, identified by a \`blob_id\` and containing \`file_content\`.
-- Each fragment has a \`reference_index\` that indicates its source.
 
 ## Citations Rules
 When referencing information from the provided documents or files in your response:
 1. Use markdown footnote format for citations
 2. Add citations immediately after the relevant sentence or paragraph
-3. Required format: [^reference_index] where reference_index is the numerical index of the source document or file
+3. Required format: [^reference_index] where reference_index is an increasing positive integer
 4. You MUST include citations at the end of your response in this exact format:
   - For documents: [^reference_index]:{"type":"doc","docId":"document_id"}
   - For files: [^reference_index]:{"type":"attachment","blobId":"blob_id","fileName":"file_name","fileType":"file_type"}
@@ -1045,22 +1044,20 @@ The following content is a relevant content segment:
 {{#docs}}
 ==========
 - type: document
-- reference_index: {{refIndex}}
-- document_id: {{docId}} 
+- document_id: {{docId}}
 - document_content:
-{{markdown}}
+{{docContent}}
 ==========
 {{/docs}}
 
 {{#files}}
 ==========
 - type: file
-- reference_index: {{refIndex}}
 - blob_id: {{blobId}}
 - file_name: {{fileName}}
 - file_type: {{fileType}}
 - file_content:
-{{chunks}}
+{{fileContent}}
 ==========
 {{/files}}
 

packages/frontend/core/src/blocksuite/ai/chat-panel/chat-context.ts

@@ -36,17 +36,15 @@ export type ChatStatus =
 
 export interface DocContext {
   docId: string;
-  refIndex: number;
-  markdown: string;
+  docContent: string;
 }
 
-export type FileContext = {
+export interface FileContext {
   blobId: string;
-  refIndex: number;
   fileName: string;
   fileType: string;
-  chunks: string;
-};
+  fileContent: string;
+}
 
 export type ChatContextValue = {
   // history messages of the chat

packages/frontend/core/src/blocksuite/ai/chat-panel/chat-panel-input.ts

@@ -23,6 +23,7 @@ import type {
   ChatContextValue,
   ChatMessage,
   DocContext,
+  FileChip,
   FileContext,
 } from './chat-context';
 import { isDocChip, isFileChip } from './components/utils';
@@ -556,43 +557,55 @@ export class ChatPanelInput extends SignalWatcher(WithDisposable(LitElement)) {
 
   private async _getMatchedContexts(userInput: string) {
     const contextId = await this.getContextId();
-    // TODO(@akumatus): adapt workspace docs
-    const { files: matched = [] } =
-      (contextId &&
-        (await AIProvider.context?.matchContext(contextId, userInput))) ||
-      {};
+    if (!contextId) {
+      return { files: [], docs: [] };
+    }
 
-    const contexts = this.chatContextValue.chips.reduce(
-      (acc, chip, index) => {
-        if (chip.state !== 'finished') {
-          return acc;
-        }
-        if (isDocChip(chip) && !!chip.markdown?.value) {
-          acc.docs.push({
-            docId: chip.docId,
-            refIndex: index + 1,
-            markdown: chip.markdown.value,
+    const docContexts = new Map<string, DocContext>();
+    const fileContexts = new Map<string, FileContext>();
+
+    const { files: matchedFiles = [], docs: matchedDocs = [] } =
+      (await AIProvider.context?.matchContext(contextId, userInput)) ?? {};
+
+    matchedDocs.forEach(doc => {
+      docContexts.set(doc.docId, {
+        docId: doc.docId,
+        docContent: doc.content,
+      });
+    });
+
+    matchedFiles.forEach(file => {
+      const context = fileContexts.get(file.fileId);
+      if (context) {
+        context.fileContent += `\n${file.content}`;
+      } else {
+        const fileChip = this.chatContextValue.chips.find(
+          chip => isFileChip(chip) && chip.fileId === file.fileId
+        ) as FileChip | undefined;
+        if (fileChip && fileChip.blobId) {
+          fileContexts.set(file.fileId, {
+            blobId: fileChip.blobId,
+            fileName: fileChip.file.name,
+            fileType: fileChip.file.type,
+            fileContent: file.content,
           });
         }
-        if (isFileChip(chip) && chip.blobId) {
-          const matchedChunks = matched
-            .filter(chunk => chunk.fileId === chip.fileId)
-            .map(chunk => chunk.content);
-          if (matchedChunks.length > 0) {
-            acc.files.push({
-              blobId: chip.blobId,
-              refIndex: index + 1,
-              fileName: chip.file.name,
-              fileType: chip.file.type,
-              chunks: matchedChunks.join('\n'),
-            });
-          }
-        }
-        return acc;
-      },
-      { docs: [], files: [] } as { docs: DocContext[]; files: FileContext[] }
-    );
-    return contexts;
+      }
+    });
+
+    this.chatContextValue.chips.forEach(chip => {
+      if (isDocChip(chip) && !!chip.markdown?.value) {
+        docContexts.set(chip.docId, {
+          docId: chip.docId,
+          docContent: chip.markdown.value,
+        });
+      }
+    });
+
+    return {
+      docs: Array.from(docContexts.values()),
+      files: Array.from(fileContexts.values()),
+    };
   }
 }