fix(server): rerank scores calc (#13016)

fix AI-257

tests/affine-cloud-copilot/e2e/settings/embedding.spec.ts

@@ -1,15 +1,13 @@
 import { createLocalWorkspace } from '@affine-test/kit/utils/workspace';
+import { faker } from '@faker-js/faker';
 import { expect } from '@playwright/test';
 
 import { test } from '../base/base-test';
 
-test.describe.configure({ mode: 'serial' });
-
 test.describe('AISettings/Embedding', () => {
   test.beforeEach(async ({ loggedInPage: page, utils }) => {
     await utils.testUtils.setupTestEnvironment(page);
     await utils.chatPanel.openChatPanel(page);
-    await utils.settings.openSettingsPanel(page);
   });
 
   test.afterEach(async ({ loggedInPage: page, utils }) => {
@@ -23,6 +21,7 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.waitForWorkspaceEmbeddingSwitchToBe(page, true);
   });
 
@@ -30,6 +29,7 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
     await utils.settings.disableWorkspaceEmbedding(page);
     await utils.settings.waitForWorkspaceEmbeddingSwitchToBe(page, false);
@@ -39,6 +39,7 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.disableWorkspaceEmbedding(page);
     await utils.settings.enableWorkspaceEmbedding(page);
     await utils.settings.waitForWorkspaceEmbeddingSwitchToBe(page, true);
@@ -99,6 +100,7 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
     await utils.settings.disableWorkspaceEmbedding(page);
     await utils.settings.waitForWorkspaceEmbeddingSwitchToBe(page, false);
@@ -116,6 +118,7 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
     await page.getByTestId('embedding-progress-wrapper');
 
@@ -134,9 +137,13 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await createLocalWorkspace({ name: 'test' }, page, false, 'affine-cloud');
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
-    const textContent1 = 'WorkspaceEBEEE is a cute cat';
-    const textContent2 = 'WorkspaceEBFFF is a cute dog';
+    const randomStr1 = Math.random().toString(36).substring(2, 6);
+    const randomStr2 = Math.random().toString(36).substring(2, 6);
+    const textContent1 = `Workspace${randomStr1} is a cute cat`;
+    const textContent2 = `Workspace${randomStr2} is a cute dog`;
     const buffer1 = Buffer.from(textContent1);
     const buffer2 = Buffer.from(textContent2);
     const attachments = [
@@ -164,15 +171,6 @@ test.describe('AISettings/Embedding', () => {
 
     await utils.settings.uploadWorkspaceEmbedding(page, attachments);
 
-    const attachmentList = await page.getByTestId(
-      'workspace-embedding-setting-attachment-list'
-    );
-
-    // Persisted
-    await expect(
-      attachmentList.getByTestId('workspace-embedding-setting-attachment-item')
-    ).toHaveCount(2);
-
     await client.send('Network.emulateNetworkConditions', {
       offline: false,
       latency: 0,
@@ -180,19 +178,19 @@ test.describe('AISettings/Embedding', () => {
       uploadThroughput: -1,
     });
 
-    await utils.settings.closeSettingsPanel(page);
+    await utils.settings.waitForFileEmbeddingReadiness(page, 2);
 
-    await page.waitForTimeout(5000); // wait for the embedding to be ready
+    await utils.settings.closeSettingsPanel(page);
 
     await utils.chatPanel.makeChat(
       page,
-      'What is WorkspaceEBEEE? What is WorkspaceEBFFF?'
+      `What is Workspace${randomStr1}? What is Workspace${randomStr2}?`
     );
 
     await utils.chatPanel.waitForHistory(page, [
       {
         role: 'user',
-        content: 'What is WorkspaceEBEEE? What is WorkspaceEBFFF?',
+        content: `What is Workspace${randomStr1}? What is Workspace${randomStr2}?`,
       },
       {
         role: 'assistant',
@@ -203,8 +201,8 @@ test.describe('AISettings/Embedding', () => {
     await expect(async () => {
       const { content, message } =
         await utils.chatPanel.getLatestAssistantMessage(page);
-      expect(content).toMatch(/WorkspaceEBEEE.*cat/);
-      expect(content).toMatch(/WorkspaceEBFFF.*dog/);
+      expect(content).toMatch(new RegExp(`Workspace${randomStr1}.*cat`));
+      expect(content).toMatch(new RegExp(`Workspace${randomStr2}.*dog`));
       expect(await message.locator('affine-footnote-node').count()).toBe(2);
     }).toPass({ timeout: 20000 });
   });
@@ -213,6 +211,8 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await createLocalWorkspace({ name: 'test' }, page, false, 'affine-cloud');
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
     const attachments = [
       {
@@ -226,7 +226,7 @@ test.describe('AISettings/Embedding', () => {
 
     await utils.settings.uploadWorkspaceEmbedding(page, attachments);
 
-    const attachmentList = await page.getByTestId(
+    const attachmentList = page.getByTestId(
       'workspace-embedding-setting-attachment-list'
     );
 
@@ -243,45 +243,42 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await createLocalWorkspace({ name: 'test' }, page, false, 'affine-cloud');
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
-    const hobby1 = Buffer.from('Jerry-Affine love climbing');
-    const hobby2 = Buffer.from('Jerry-Affine love skating');
+    const person = faker.person.fullName();
+
+    const hobby1 = Buffer.from(`${person} love climbing`);
+    const hobby2 = Buffer.from(`${person} love skating`);
     const attachments = [
       {
-        name: 'jerry-affine-hobby.txt',
+        name: 'hobby.txt',
         mimeType: 'text/plain',
         buffer: hobby1,
       },
     ];
     await utils.settings.uploadWorkspaceEmbedding(page, attachments);
 
-    const attachmentList = await page.getByTestId(
-      'workspace-embedding-setting-attachment-list'
-    );
-    await expect(
-      attachmentList.getByTestId('workspace-embedding-setting-attachment-item')
-    ).toHaveCount(1);
+    await utils.settings.waitForFileEmbeddingReadiness(page, 1);
 
     await utils.settings.closeSettingsPanel(page);
 
-    await page.waitForTimeout(5000); // wait for the embedding to be ready
-
     await utils.chatPanel.chatWithAttachments(
       page,
       [
         {
-          name: 'jerry-affine-hobby2.txt',
+          name: 'hobby2.txt',
           mimeType: 'text/plain',
           buffer: hobby2,
         },
       ],
-      'What is Jerry-Affine hobby?'
+      `What is ${person}'s hobby?`
     );
 
     await utils.chatPanel.waitForHistory(page, [
       {
         role: 'user',
-        content: 'What is Jerry-Affine hobby?',
+        content: `What is ${person}'s hobby?`,
       },
       {
         role: 'assistant',
@@ -302,6 +299,8 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await createLocalWorkspace({ name: 'test' }, page, false, 'affine-cloud');
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
     const attachments = Array.from({ length: 11 }, (_, i) => ({
       name: `document${i + 1}.txt`,
@@ -318,11 +317,11 @@ test.describe('AISettings/Embedding', () => {
     await expect(
       attachmentList.getByTestId('workspace-embedding-setting-attachment-item')
     ).toHaveCount(10);
-    const pagination = await attachmentList.getByRole('navigation');
-    const currentPage = await pagination.locator('li.active');
+    const pagination = attachmentList.getByRole('navigation');
+    const currentPage = pagination.locator('li.active');
     await expect(currentPage).toHaveText('1');
 
-    const page2 = await pagination.locator('li').nth(2);
+    const page2 = pagination.locator('li').nth(2);
     await page2.click();
 
     await expect(
@@ -339,8 +338,11 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await createLocalWorkspace({ name: 'test' }, page, false, 'affine-cloud');
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
-    const textContent = 'WorkspaceEBEEE is a cute cat';
+    const randomStr1 = Math.random().toString(36).substring(2, 6);
+    const textContent = `Workspace${randomStr1} is a cute cat`;
     const attachments = [
       {
         name: 'document1.txt',
@@ -350,7 +352,7 @@ test.describe('AISettings/Embedding', () => {
     ];
     await utils.settings.uploadWorkspaceEmbedding(page, attachments);
 
-    const attachmentList = await page.getByTestId(
+    const attachmentList = page.getByTestId(
       'workspace-embedding-setting-attachment-list'
     );
     await expect(
@@ -363,8 +365,11 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await createLocalWorkspace({ name: 'test' }, page, false, 'affine-cloud');
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
-    const textContent = 'WorkspaceEBEEE is a cute cat';
+    const randomStr1 = Math.random().toString(36).substring(2, 6);
+    const textContent = `Workspace${randomStr1} is a cute cat`;
     const attachments = [
       {
         name: 'document1.txt',
@@ -374,7 +379,7 @@ test.describe('AISettings/Embedding', () => {
     ];
     await utils.settings.uploadWorkspaceEmbedding(page, attachments);
 
-    const attachmentList = await page.getByTestId(
+    const attachmentList = page.getByTestId(
       'workspace-embedding-setting-attachment-list'
     );
     await expect(
@@ -393,8 +398,11 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await createLocalWorkspace({ name: 'test' }, page, false, 'affine-cloud');
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
-    const textContent = 'WorkspaceEBEEE is a cute cat';
+    const randomStr1 = Math.random().toString(36).substring(2, 6);
+    const textContent = `Workspace${randomStr1} is a cute cat`;
     const attachments = [
       {
         name: 'document1.txt',
@@ -413,6 +421,7 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
     await utils.settings.closeSettingsPanel(page);
     await utils.editor.createDoc(
@@ -426,7 +435,9 @@ test.describe('AISettings/Embedding', () => {
       'WBIgnoreFFF is a cute dog'
     );
 
-    await page.waitForTimeout(5000); // wait for the embedding to be ready
+    await utils.settings.openSettingsPanel(page);
+    await utils.settings.waitForEmbeddingComplete(page);
+    await utils.settings.closeSettingsPanel(page);
 
     await utils.chatPanel.makeChat(
       page,
@@ -487,6 +498,7 @@ test.describe('AISettings/Embedding', () => {
     loggedInPage: page,
     utils,
   }) => {
+    await utils.settings.openSettingsPanel(page);
     await utils.settings.enableWorkspaceEmbedding(page);
     await utils.settings.closeSettingsPanel(page);
 

tests/affine-cloud-copilot/e2e/utils/chat-panel-utils.ts

@@ -199,9 +199,11 @@ export class ChatPanelUtils {
   }
 
   public static async chatWithDoc(page: Page, docName: string) {
-    const withButton = await page.getByTestId('chat-panel-with-button');
+    const withButton = page.getByTestId('chat-panel-with-button');
+    await withButton.hover();
     await withButton.click();
-    const withMenu = await page.getByTestId('ai-add-popover');
+    const withMenu = page.getByTestId('ai-add-popover');
+    await withMenu.waitFor({ state: 'visible' });
     await withMenu.getByText(docName).click();
     await page.getByTestId('chat-panel-chips').getByText(docName);
   }
@@ -221,9 +223,20 @@ export class ChatPanelUtils {
       await withButton.hover();
       await withButton.click();
       const withMenu = page.getByTestId('ai-add-popover');
+      await withMenu.waitFor({ state: 'visible' });
       await withMenu.getByTestId('ai-chat-with-files').click();
       const fileChooser = await fileChooserPromise;
       await fileChooser.setFiles(attachment);
+
+      await expect(async () => {
+        const states = await page
+          .getByTestId('chat-panel-chip')
+          .evaluateAll(elements =>
+            elements.map(el => el.getAttribute('data-state'))
+          );
+
+        expect(states.every(state => state === 'finished')).toBe(true);
+      }).toPass({ timeout: 20000 });
     }
     await expect(async () => {
       const states = await page
@@ -267,9 +280,11 @@ export class ChatPanelUtils {
 
   public static async chatWithTags(page: Page, tags: string[]) {
     for (const tag of tags) {
-      const withButton = await page.getByTestId('chat-panel-with-button');
+      const withButton = page.getByTestId('chat-panel-with-button');
+      await withButton.hover();
       await withButton.click();
-      const withMenu = await page.getByTestId('ai-add-popover');
+      const withMenu = page.getByTestId('ai-add-popover');
+      await withMenu.waitFor({ state: 'visible' });
       await withMenu.getByTestId('ai-chat-with-tags').click();
       await withMenu.getByText(tag).click();
       await page.getByTestId('chat-panel-chips').getByText(tag);
@@ -282,9 +297,11 @@ export class ChatPanelUtils {
 
   public static async chatWithCollections(page: Page, collections: string[]) {
     for (const collection of collections) {
-      const withButton = await page.getByTestId('chat-panel-with-button');
+      const withButton = page.getByTestId('chat-panel-with-button');
+      await withButton.hover();
       await withButton.click();
-      const withMenu = await page.getByTestId('ai-add-popover');
+      const withMenu = page.getByTestId('ai-add-popover');
+      await withMenu.waitFor({ state: 'visible' });
       await withMenu.getByTestId('ai-chat-with-collections').click();
       await withMenu.getByText(collection).click();
       await page.getByTestId('chat-panel-chips').getByText(collection);

tests/affine-cloud-copilot/e2e/utils/settings-panel-utils.ts

@@ -207,4 +207,53 @@ export class SettingsPanelUtils {
       await searcher.getByTestId('doc-selector-confirm-button').click();
     }
   }
+
+  private static async waitForEmbeddingStatus(
+    page: Page,
+    timeout: number,
+    status = 'synced'
+  ) {
+    await expect(async () => {
+      await this.openSettingsPanel(page);
+      const title = page.getByTestId('embedding-progress-title');
+      // oxlint-disable-next-line prefer-dom-node-dataset
+      const progressAttr = await title.getAttribute('data-progress');
+      expect(progressAttr).not.toBe('loading');
+
+      expect(progressAttr).toBe(status);
+    }).toPass({ timeout });
+  }
+
+  public static async waitForEmbeddingComplete(page: Page, timeout = 30000) {
+    await this.waitForEmbeddingStatus(page, timeout);
+
+    // check embedding progress count
+    await expect(async () => {
+      const count = page.getByTestId('embedding-progress-count');
+      const countText = await count.textContent();
+      if (countText) {
+        const [embedded, total] = countText.split('/').map(Number);
+        expect(embedded).toBe(total);
+        expect(embedded).toBeGreaterThan(0);
+      }
+    }).toPass({ timeout });
+  }
+
+  public static async waitForFileEmbeddingReadiness(
+    page: Page,
+    expectedFileCount: number,
+    timeout = 30000
+  ) {
+    await expect(async () => {
+      const attachmentList = page.getByTestId(
+        'workspace-embedding-setting-attachment-list'
+      );
+      const attachmentItems = attachmentList.getByTestId(
+        'workspace-embedding-setting-attachment-item'
+      );
+      await expect(attachmentItems).toHaveCount(expectedFileCount);
+    }).toPass({ timeout });
+
+    await this.waitForEmbeddingComplete(page, timeout);
+  }
 }

tests/kit/src/utils/workspace.ts

@@ -16,7 +16,8 @@ export async function openWorkspaceListModal(page: Page) {
 export async function createLocalWorkspace(
   params: CreateWorkspaceParams,
   page: Page,
-  skipOpenWorkspaceListModal = false
+  skipOpenWorkspaceListModal = false,
+  serverId?: string
 ) {
   if (!skipOpenWorkspaceListModal) {
     await openWorkspaceListModal(page);
@@ -33,10 +34,9 @@ export async function createLocalWorkspace(
   await page.getByPlaceholder('Set a Workspace name').click();
   await page.getByPlaceholder('Set a Workspace name').fill(params.name);
 
-  // select local server
   await page.getByTestId('server-selector-trigger').click();
   const serverSelectorList = page.getByTestId('server-selector-list');
-  await serverSelectorList.getByTestId('local').click();
+  await serverSelectorList.getByTestId(serverId ?? 'local').click();
 
   // click create button
   await page.getByTestId('create-workspace-create-button').click({