feat(server): use faster model in ci test (#13038)

fix AI-329

packages/backend/server/src/__tests__/copilot-provider.spec.ts

@@ -369,7 +369,7 @@ The term **“CRDT”** was first introduced by Marc Shapiro, Nuno Preguiça, Ca
               .map(c => JSON.parse(c.citationJson).type)
               .filter(type => ['attachment', 'doc'].includes(type)).length ===
               0,
-          'should not have citation'
+          `should not have citation: ${JSON.stringify(c, null, 2)}`
         );
       });
     },

packages/backend/server/src/plugins/copilot/embedding/client.ts

@@ -112,11 +112,14 @@ class ProductionEmbeddingClient extends EmbeddingClient {
     );
 
     try {
-      return ranks.map((score, chunk) => ({
-        chunk,
-        targetId: this.getTargetId(embeddings[chunk]),
-        score,
-      }));
+      return ranks.map((score, i) => {
+        const chunk = embeddings[i];
+        return {
+          chunk: chunk.chunk,
+          targetId: this.getTargetId(chunk),
+          score: Math.max(score, 1 - (chunk.distance || -Infinity)),
+        };
+      });
     } catch (error) {
       this.logger.error('Failed to parse rerank results', error);
       // silent error, will fallback to default sorting in parent method
@@ -148,7 +151,7 @@ class ProductionEmbeddingClient extends EmbeddingClient {
 
     const chunks = sortedEmbeddings.reduce(
       (acc, e) => {
-        const targetId = 'docId' in e ? e.docId : 'fileId' in e ? e.fileId : '';
+        const targetId = this.getTargetId(e);
         const key = `${targetId}:${e.chunk}`;
         acc[key] = e;
         return acc;
@@ -179,7 +182,10 @@ class ProductionEmbeddingClient extends EmbeddingClient {
         .filter(Boolean);
 
       this.logger.verbose(
-        `ReRank completed: ${highConfidenceChunks.length} high-confidence results found`
+        `ReRank completed: ${highConfidenceChunks.length} high-confidence results found, total ${sortedEmbeddings.length} embeddings`,
+        highConfidenceChunks.length !== sortedEmbeddings.length
+          ? JSON.stringify(ranks)
+          : undefined
       );
       return highConfidenceChunks.slice(0, topK);
     } catch (error) {

packages/backend/server/src/plugins/copilot/prompt/prompts.ts

@@ -338,7 +338,7 @@ Convert a multi-speaker audio recording into a structured JSON format by transcr
   {
     name: 'Rerank results',
     action: 'Rerank results',
-    model: 'gpt-4.1-mini',
+    model: 'gpt-4.1',
     messages: [
       {
         role: 'system',
@@ -1677,7 +1677,7 @@ This sentence contains information from the first source[^1]. This sentence refe
 Before starting Tool calling, you need to follow:
 - DO NOT explain what operation you will perform.
 - DO NOT embed a tool call mid-sentence.
-- When searching for unknown information or keyword, prioritize searching the user's workspace.
+- When searching for unknown information, personal information or keyword, prioritize searching the user's workspace rather than the web.
 - Depending on the complexity of the question and the information returned by the search tools, you can call different tools multiple times to search.
 </tool-calling-guidelines>
 

packages/backend/server/src/plugins/copilot/prompt/service.ts

@@ -53,8 +53,11 @@ export class PromptService implements OnApplicationBootstrap {
    * @returns prompt messages
    */
   async get(name: string): Promise<ChatPrompt | null> {
-    const cached = this.cache.get(name);
-    if (cached) return cached;
+    // skip cache in dev mode to ensure the latest prompt is always fetched
+    if (!env.dev) {
+      const cached = this.cache.get(name);
+      if (cached) return cached;
+    }
 
     const prompt = await this.db.aiPrompt.findUnique({
       where: {

packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts

@@ -62,6 +62,7 @@ export abstract class AnthropicProvider<T> extends CopilotProvider<T> {
 
     try {
       metrics.ai.counter('chat_text_calls').add(1, { model: model.id });
+
       const [system, msgs] = await chatToGPTMessage(messages, true, true);
 
       const modelInstance = this.instance(model.id);

packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts

@@ -88,6 +88,12 @@ export abstract class GeminiProvider<T> extends CopilotProvider<T> {
         system,
         messages: msgs,
         abortSignal: options.signal,
+        providerOptions: {
+          google: this.getGeminiOptions(options, model.id),
+        },
+        tools: await this.getTools(options, model.id),
+        maxSteps: this.MAX_STEPS,
+        experimental_continueSteps: true,
       });
 
       if (!text) throw new Error('Failed to generate text');
@@ -254,16 +260,16 @@ export abstract class GeminiProvider<T> extends CopilotProvider<T> {
   ) {
     const [system, msgs] = await chatToGPTMessage(messages);
     const { fullStream } = streamText({
-      model: this.instance(model.id, {
-        useSearchGrounding: this.useSearchGrounding(options),
-      }),
+      model: this.instance(model.id),
       system,
       messages: msgs,
       abortSignal: options.signal,
-      maxSteps: this.MAX_STEPS,
       providerOptions: {
         google: this.getGeminiOptions(options, model.id),
       },
+      tools: await this.getTools(options, model.id),
+      maxSteps: this.MAX_STEPS,
+      experimental_continueSteps: true,
     });
     return fullStream;
   }
@@ -282,8 +288,4 @@ export abstract class GeminiProvider<T> extends CopilotProvider<T> {
   private isReasoningModel(model: string) {
     return model.startsWith('gemini-2.5');
   }
-
-  private useSearchGrounding(options: CopilotChatOptions) {
-    return options?.tools?.includes('webSearch');
-  }
 }

packages/backend/server/src/plugins/copilot/providers/openai.ts

@@ -274,9 +274,11 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
   override getProviderSpecificTools(
     toolName: CopilotChatTools,
     model: string
-  ): [string, Tool] | undefined {
+  ): [string, Tool?] | undefined {
     if (toolName === 'webSearch' && !this.isReasoningModel(model)) {
       return ['web_search_preview', openai.tools.webSearchPreview()];
+    } else if (toolName === 'docEdit') {
+      return ['doc_edit', undefined];
     }
     return;
   }

packages/backend/server/src/plugins/copilot/providers/provider.ts

@@ -126,7 +126,7 @@ export abstract class CopilotProvider<C = any> {
   protected getProviderSpecificTools(
     _toolName: CopilotChatTools,
     _model: string
-  ): [string, Tool] | undefined {
+  ): [string, Tool?] | undefined {
     return;
   }
 
@@ -143,7 +143,10 @@ export abstract class CopilotProvider<C = any> {
       for (const tool of options.tools) {
         const toolDef = this.getProviderSpecificTools(tool, model);
         if (toolDef) {
-          tools[toolDef[0]] = toolDef[1];
+          // allow provider prevent tool creation
+          if (toolDef[1]) {
+            tools[toolDef[0]] = toolDef[1];
+          }
           continue;
         }
         switch (tool) {

packages/backend/server/src/plugins/copilot/tools/doc-semantic-search.ts

@@ -58,7 +58,7 @@ export const createDocSemanticSearchTool = (
 ) => {
   return tool({
     description:
-      'Retrieve conceptually related passages by performing vector-based semantic similarity search across embedded documents; use this tool only when exact keyword search fails or the user explicitly needs meaning-level matches (e.g., paraphrases, synonyms, broader concepts).',
+      'Retrieve conceptually related passages by performing vector-based semantic similarity search across embedded documents; use this tool only when exact keyword search fails or the user explicitly needs meaning-level matches (e.g., paraphrases, synonyms, broader concepts, recent documents).',
     parameters: z.object({
       query: z
         .string()

tests/affine-cloud-copilot/e2e/ai-action/image-processing.spec.ts

@@ -36,6 +36,8 @@ test.describe('AIAction/ImageProcessing', () => {
     await expect(answer.getByTestId('ai-answer-image')).toBeVisible();
     const insert = answer.getByTestId('answer-insert-below');
     await insert.click();
+    await page.reload();
+
     await utils.chatPanel.waitForHistory(page, [
       {
         role: 'action',

tests/affine-cloud-copilot/e2e/basic/chat.spec.ts

@@ -45,28 +45,30 @@ test.describe('AIBasic/Chat', () => {
     // Type and send a message
     await utils.chatPanel.makeChat(
       page,
-      'Introduce AFFiNE to me. Answer in 50 words.'
+      'Introduce AFFiNE to me. Answer in 500 words.'
     );
 
-    // AI is loading
-    await utils.chatPanel.waitForHistory(page, [
-      {
-        role: 'user',
-        content: 'Introduce AFFiNE to me. Answer in 50 words.',
-      },
-      {
-        role: 'assistant',
-        status: 'loading',
-      },
-    ]);
+    if (!(await page.getByTestId('ai-loading').isVisible())) {
+      // AI is loading
+      await utils.chatPanel.waitForHistory(page, [
+        {
+          role: 'user',
+          content: 'Introduce AFFiNE to me. Answer in 500 words.',
+        },
+        {
+          role: 'assistant',
+          status: 'loading',
+        },
+      ]);
 
-    await expect(page.getByTestId('ai-loading')).toBeVisible();
+      await expect(page.getByTestId('ai-loading')).toBeVisible();
+    }
 
     // AI Generating
     await utils.chatPanel.waitForHistory(page, [
       {
         role: 'user',
-        content: 'Introduce AFFiNE to me. Answer in 50 words.',
+        content: 'Introduce AFFiNE to me. Answer in 500 words.',
       },
       {
         role: 'assistant',
@@ -79,7 +81,7 @@ test.describe('AIBasic/Chat', () => {
     await utils.chatPanel.waitForHistory(page, [
       {
         role: 'user',
-        content: 'Introduce AFFiNE to me. Answer in 50 words.',
+        content: 'Introduce AFFiNE to me. Answer in 500 words.',
       },
       {
         role: 'assistant',
@@ -94,14 +96,14 @@ test.describe('AIBasic/Chat', () => {
   }) => {
     await utils.chatPanel.makeChat(
       page,
-      'Introduce AFFiNE to me. Answer in 50 words.'
+      'Introduce AFFiNE to me. Answer in 5000 words.'
     );
 
     // AI Generating
     await utils.chatPanel.waitForHistory(page, [
       {
         role: 'user',
-        content: 'Introduce AFFiNE to me. Answer in 50 words.',
+        content: 'Introduce AFFiNE to me. Answer in 5000 words.',
       },
       {
         role: 'assistant',
@@ -113,7 +115,7 @@ test.describe('AIBasic/Chat', () => {
     await utils.chatPanel.waitForHistory(page, [
       {
         role: 'user',
-        content: 'Introduce AFFiNE to me. Answer in 50 words.',
+        content: 'Introduce AFFiNE to me. Answer in 5000 words.',
       },
       {
         role: 'assistant',
@@ -183,13 +185,14 @@ test.describe('AIBasic/Chat', () => {
     // Type and send a message
     await utils.chatPanel.makeChat(
       page,
-      'Hello, write a poem about the moon. Answer in 50 words.'
+      'Hello, give a introduction about the moon. Answer in 500 words.'
     );
 
     await utils.chatPanel.waitForHistory(page, [
       {
         role: 'user',
-        content: 'Hello, write a poem about the moon. Answer in 50 words.',
+        content:
+          'Hello, give a introduction about the moon. Answer in 500 words.',
       },
       {
         role: 'assistant',

tests/affine-cloud-copilot/e2e/chat-with/attachments.spec.ts

@@ -2,9 +2,14 @@ import { expect } from '@playwright/test';
 
 import { test } from '../base/base-test';
 
+test.describe.configure({ mode: 'serial' });
+
 test.describe('AIChatWith/Attachments', () => {
   test.beforeEach(async ({ loggedInPage: page, utils }) => {
-    await utils.testUtils.setupTestEnvironment(page);
+    await utils.testUtils.setupTestEnvironment(
+      page,
+      'claude-sonnet-4@20250514'
+    );
     await utils.chatPanel.openChatPanel(page);
   });
 
@@ -48,8 +53,10 @@ test.describe('AIChatWith/Attachments', () => {
     loggedInPage: page,
     utils,
   }) => {
-    const textContent1 = 'AttachmentEEee is a cute cat';
-    const textContent2 = 'AttachmentFFff is a cute dog';
+    const randomStr1 = Math.random().toString(36).substring(2, 6);
+    const randomStr2 = Math.random().toString(36).substring(2, 6);
+    const textContent1 = `Attachment${randomStr1} is a cute cat`;
+    const textContent2 = `Attachment${randomStr2} is a cute dog`;
     const buffer1 = Buffer.from(textContent1);
     const buffer2 = Buffer.from(textContent2);
 
@@ -67,13 +74,13 @@ test.describe('AIChatWith/Attachments', () => {
           buffer: buffer2,
         },
       ],
-      'What is AttachmentEEee? What is AttachmentFFff?'
+      `What is Attachment${randomStr1}? What is Attachment${randomStr2}?`
     );
 
     await utils.chatPanel.waitForHistory(page, [
       {
         role: 'user',
-        content: 'What is AttachmentEEee? What is AttachmentFFff?',
+        content: `What is Attachment${randomStr1}? What is Attachment${randomStr2}?`,
       },
       {
         role: 'assistant',
@@ -84,8 +91,8 @@ test.describe('AIChatWith/Attachments', () => {
     await expect(async () => {
       const { content, message } =
         await utils.chatPanel.getLatestAssistantMessage(page);
-      expect(content).toMatch(/AttachmentEEee/);
-      expect(content).toMatch(/AttachmentFFff/);
+      expect(content).toMatch(new RegExp(`Attachment${randomStr1}`));
+      expect(content).toMatch(new RegExp(`Attachment${randomStr2}`));
       expect(await message.locator('affine-footnote-node').count()).toBe(2);
     }).toPass({ timeout: 20000 });
   });

tests/affine-cloud-copilot/e2e/chat-with/collections.spec.ts

@@ -6,7 +6,10 @@ test.describe.configure({ mode: 'serial' });
 
 test.describe('AIChatWith/Collections', () => {
   test.beforeEach(async ({ loggedInPage: page, utils }) => {
-    await utils.testUtils.setupTestEnvironment(page);
+    await utils.testUtils.setupTestEnvironment(
+      page,
+      'claude-sonnet-4@20250514'
+    );
     await utils.chatPanel.openChatPanel(page);
     await utils.editor.clearAllCollections(page);
 

tests/affine-cloud-copilot/e2e/settings/embedding.spec.ts

@@ -1,12 +1,16 @@
 import { createLocalWorkspace } from '@affine-test/kit/utils/workspace';
-import { faker } from '@faker-js/faker';
 import { expect } from '@playwright/test';
 
 import { test } from '../base/base-test';
 
+test.describe.configure({ mode: 'serial' });
+
 test.describe('AISettings/Embedding', () => {
   test.beforeEach(async ({ loggedInPage: page, utils }) => {
-    await utils.testUtils.setupTestEnvironment(page);
+    await utils.testUtils.setupTestEnvironment(
+      page,
+      'claude-sonnet-4@20250514'
+    );
     await utils.chatPanel.openChatPanel(page);
   });
 
@@ -246,7 +250,7 @@ test.describe('AISettings/Embedding', () => {
     await createLocalWorkspace({ name: 'test' }, page, false, 'affine-cloud');
     await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
-    const person = faker.person.fullName();
+    const person = 'test123';
 
     const hobby1 = Buffer.from(`${person} love climbing`);
     const hobby2 = Buffer.from(`${person} love skating`);

tests/affine-cloud-copilot/e2e/utils/chat-panel-utils.ts

@@ -201,7 +201,7 @@ export class ChatPanelUtils {
   public static async chatWithDoc(page: Page, docName: string) {
     const withButton = page.getByTestId('chat-panel-with-button');
     await withButton.hover();
-    await withButton.click();
+    await withButton.click({ delay: 200 });
     const withMenu = page.getByTestId('ai-add-popover');
     await withMenu.waitFor({ state: 'visible' });
     await withMenu.getByText(docName).click();
@@ -221,7 +221,7 @@ export class ChatPanelUtils {
       const fileChooserPromise = page.waitForEvent('filechooser');
       const withButton = page.getByTestId('chat-panel-with-button');
       await withButton.hover();
-      await withButton.click();
+      await withButton.click({ delay: 200 });
       const withMenu = page.getByTestId('ai-add-popover');
       await withMenu.waitFor({ state: 'visible' });
       await withMenu.getByTestId('ai-chat-with-files').click();
@@ -282,7 +282,7 @@ export class ChatPanelUtils {
     for (const tag of tags) {
       const withButton = page.getByTestId('chat-panel-with-button');
       await withButton.hover();
-      await withButton.click();
+      await withButton.click({ delay: 200 });
       const withMenu = page.getByTestId('ai-add-popover');
       await withMenu.waitFor({ state: 'visible' });
       await withMenu.getByTestId('ai-chat-with-tags').click();
@@ -299,7 +299,7 @@ export class ChatPanelUtils {
     for (const collection of collections) {
       const withButton = page.getByTestId('chat-panel-with-button');
       await withButton.hover();
-      await withButton.click();
+      await withButton.click({ delay: 200 });
       const withMenu = page.getByTestId('ai-add-popover');
       await withMenu.waitFor({ state: 'visible' });
       await withMenu.getByTestId('ai-chat-with-collections').click();

tests/affine-cloud-copilot/e2e/utils/test-utils.ts

@@ -1,5 +1,8 @@
 import { skipOnboarding } from '@affine-test/kit/playwright';
-import { createRandomAIUser } from '@affine-test/kit/utils/cloud';
+import {
+  createRandomAIUser,
+  switchDefaultChatModel,
+} from '@affine-test/kit/utils/cloud';
 import { openHomePage, setCoreUrl } from '@affine-test/kit/utils/load-page';
 import {
   clickNewPageButton,
@@ -58,7 +61,12 @@ export class TestUtils {
     await waitForEditorLoad(page);
   }
 
-  public async setupTestEnvironment(page: Page) {
+  public async setupTestEnvironment(
+    page: Page,
+    defaultModel = 'gemini-2.5-flash'
+  ) {
+    await switchDefaultChatModel(defaultModel);
+
     await skipOnboarding(page.context());
     await openHomePage(page);
     await this.createNewPage(page);

tests/kit/src/utils/cloud.ts

@@ -152,6 +152,22 @@ export async function createRandomUser(): Promise<{
   } as any;
 }
 
+export async function switchDefaultChatModel(model: string) {
+  await runPrisma(async client => {
+    const promptId = await client.aiPrompt
+      .findFirst({
+        where: { name: 'Chat With AFFiNE AI' },
+        select: { id: true },
+      })
+      .then(f => f!.id);
+
+    await client.aiPrompt.update({
+      where: { id: promptId },
+      data: { model },
+    });
+  });
+}
+
 export async function createRandomAIUser(): Promise<{
   name: string;
   email: string;