diff --git a/tests/affine-cloud-copilot/e2e/base/base-test.ts b/tests/affine-cloud-copilot/e2e/base/base-test.ts index 8e92c6bf3e..b44cbcb0a5 100644 --- a/tests/affine-cloud-copilot/e2e/base/base-test.ts +++ b/tests/affine-cloud-copilot/e2e/base/base-test.ts @@ -4,6 +4,7 @@ import type { Page } from '@playwright/test'; import { ChatPanelUtils } from '../utils/chat-panel-utils'; import { EditorUtils } from '../utils/editor-utils'; +import { SettingsPanelUtils } from '../utils/settings-panel-utils'; import { TestUtils } from '../utils/test-utils'; interface TestUtilsFixtures { @@ -11,6 +12,7 @@ interface TestUtilsFixtures { testUtils: TestUtils; chatPanel: typeof ChatPanelUtils; editor: typeof EditorUtils; + settings: typeof SettingsPanelUtils; }; loggedInPage: Page; } @@ -22,6 +24,7 @@ export const test = base.extend({ testUtils, chatPanel: ChatPanelUtils, editor: EditorUtils, + settings: SettingsPanelUtils, }); }, loggedInPage: async ({ browser }, use) => { diff --git a/tests/affine-cloud-copilot/e2e/settings/embedding.spec.ts b/tests/affine-cloud-copilot/e2e/settings/embedding.spec.ts new file mode 100644 index 0000000000..e244a1d18c --- /dev/null +++ b/tests/affine-cloud-copilot/e2e/settings/embedding.spec.ts @@ -0,0 +1,302 @@ +import { expect } from '@playwright/test'; + +import { test } from '../base/base-test'; + +test.describe.configure({ mode: 'serial' }); + +test.describe.skip('AISettings/Embedding', () => { + test.beforeEach(async ({ loggedInPage: page, utils }) => { + await utils.testUtils.setupTestEnvironment(page); + await utils.chatPanel.openChatPanel(page); + await utils.settings.openSettingsPanel(page); + }); + + test.afterEach(async ({ loggedInPage: page, utils }) => { + await utils.settings.openSettingsPanel(page); + await utils.settings.clearAllIgnoredDocs(page); + await utils.settings.removeAllAttachments(page); + await utils.settings.closeSettingsPanel(page); + }); + + test('should show workspace embedding enabled status', async ({ + loggedInPage: page, + utils, + }) => { + await utils.settings.waitForWorkspaceEmbeddingSwitchToBe(page, true); + }); + + test('should support disable workspace embedding', async ({ + loggedInPage: page, + utils, + }) => { + await utils.settings.enableWorkspaceEmbedding(page); + await utils.settings.disableWorkspaceEmbedding(page); + await utils.settings.waitForWorkspaceEmbeddingSwitchToBe(page, false); + }); + + test('should support enable workspace embedding', async ({ + loggedInPage: page, + utils, + }) => { + await utils.settings.disableWorkspaceEmbedding(page); + await utils.settings.enableWorkspaceEmbedding(page); + await utils.settings.waitForWorkspaceEmbeddingSwitchToBe(page, true); + }); + + test('should allow manual attachment upload for embedding', async ({ + loggedInPage: page, + utils, + }) => { + await utils.settings.enableWorkspaceEmbedding(page); + const textContent1 = 'WorkspaceEBEEE is a cute cat'; + const textContent2 = 'WorkspaceEBFFF is a cute dog'; + const buffer1 = Buffer.from(textContent1); + const buffer2 = Buffer.from(textContent2); + const attachments = [ + { + name: 'document1.txt', + mimeType: 'text/plain', + buffer: buffer1, + }, + { + name: 'document2.txt', + mimeType: 'text/plain', + buffer: buffer2, + }, + ]; + await utils.settings.uploadWorkspaceEmbedding(page, attachments); + + const attachmentList = await page.getByTestId( + 'workspace-embedding-setting-attachment-list' + ); + await expect( + attachmentList.getByTestId('workspace-embedding-setting-attachment-item') + ).toHaveCount(2); + + await utils.settings.closeSettingsPanel(page); + + await page.waitForTimeout(5000); // wait for the embedding to be ready + + await utils.chatPanel.makeChat( + page, + 'What is WorkspaceEBEEE? What is WorkspaceEBFFF?' + ); + + await utils.chatPanel.waitForHistory(page, [ + { + role: 'user', + content: 'What is WorkspaceEBEEE? What is WorkspaceEBFFF?', + }, + { + role: 'assistant', + status: 'success', + }, + ]); + + await expect(async () => { + const { content, message } = + await utils.chatPanel.getLatestAssistantMessage(page); + expect(content).toMatch(/WorkspaceEBEEE.*cat/); + expect(content).toMatch(/WorkspaceEBFFF.*dog/); + expect(await message.locator('affine-footnote-node').count()).toBe(2); + }).toPass({ timeout: 20000 }); + }); + + test('should support hybrid search for both globally uploaded attachments and those uploaded in the current session', async ({ + loggedInPage: page, + utils, + }) => { + await utils.settings.enableWorkspaceEmbedding(page); + const hobby1 = Buffer.from('Jerry-Affine love climbing'); + const hobby2 = Buffer.from('Jerry-Affine love skating'); + const attachments = [ + { + name: 'jerry-affine-hobby.txt', + mimeType: 'text/plain', + buffer: hobby1, + }, + ]; + await utils.settings.uploadWorkspaceEmbedding(page, attachments); + + const attachmentList = await page.getByTestId( + 'workspace-embedding-setting-attachment-list' + ); + await expect( + attachmentList.getByTestId('workspace-embedding-setting-attachment-item') + ).toHaveCount(1); + + await utils.settings.closeSettingsPanel(page); + + await page.waitForTimeout(5000); // wait for the embedding to be ready + + await utils.chatPanel.chatWithAttachments( + page, + [ + { + name: 'jerry-affine-hobby2.txt', + mimeType: 'text/plain', + buffer: hobby2, + }, + ], + 'What is Jerry-Affine hobby?' + ); + + await utils.chatPanel.waitForHistory(page, [ + { + role: 'user', + content: 'What is Jerry-Affine hobby?', + }, + { + role: 'assistant', + status: 'success', + }, + ]); + + await expect(async () => { + const { content, message } = + await utils.chatPanel.getLatestAssistantMessage(page); + expect(content).toMatch(/climbing/i); + expect(content).toMatch(/skating/i); + expect(await message.locator('affine-footnote-node').count()).toBe(2); + }).toPass({ timeout: 20000 }); + }); + + test('should support attachments pagination', async ({ + loggedInPage: page, + utils, + }) => { + await utils.settings.enableWorkspaceEmbedding(page); + const attachments = Array.from({ length: 11 }, (_, i) => ({ + name: `document${i + 1}.txt`, + mimeType: 'text/plain', + buffer: Buffer.from('attachment content'), + })); + + await utils.settings.uploadWorkspaceEmbedding(page, attachments); + + const attachmentList = await page.getByTestId( + 'workspace-embedding-setting-attachment-list' + ); + + await expect( + attachmentList.getByTestId('workspace-embedding-setting-attachment-item') + ).toHaveCount(10); + const pagination = await attachmentList.getByRole('navigation'); + const currentPage = await pagination.locator('li.active'); + await expect(currentPage).toHaveText('1'); + + const page2 = await pagination.locator('li').nth(2); + await page2.click(); + + await expect( + attachmentList.getByTestId('workspace-embedding-setting-attachment-item') + ).toHaveCount(1); + await expect( + attachmentList + .getByTestId('workspace-embedding-setting-attachment-item') + .first() + ).toHaveText('document1.txt'); + }); + + test('should support remove attachment', async ({ + loggedInPage: page, + utils, + }) => { + await utils.settings.enableWorkspaceEmbedding(page); + const textContent = 'WorkspaceEBEEE is a cute cat'; + const attachments = [ + { + name: 'document1.txt', + mimeType: 'text/plain', + buffer: Buffer.from(textContent), + }, + ]; + await utils.settings.uploadWorkspaceEmbedding(page, attachments); + + const attachmentList = await page.getByTestId( + 'workspace-embedding-setting-attachment-list' + ); + await expect( + attachmentList.getByTestId('workspace-embedding-setting-attachment-item') + ).toHaveCount(1); + await utils.settings.removeAttachment(page, 'document1.txt'); + }); + + // FIXME: wait for indexer + test.skip('should support ignore docs for embedding', async ({ + loggedInPage: page, + utils, + }) => { + await utils.settings.enableWorkspaceEmbedding(page); + await utils.settings.closeSettingsPanel(page); + await utils.editor.createDoc( + page, + 'WBIgnoreDoc1', + 'WBIgnoreEEE is a cute cat' + ); + await utils.editor.createDoc( + page, + 'WBIgnoreDoc2', + 'WBIgnoreFFF is a cute dog' + ); + + await page.waitForTimeout(5000); // wait for the embedding to be ready + + await utils.chatPanel.makeChat( + page, + 'What is WBIgnoreEEE? What is WBIgnoreFFF?If you dont know, just say "I dont know"' + ); + + await utils.chatPanel.waitForHistory(page, [ + { + role: 'user', + content: + 'What is WBIgnoreEEE? What is WBIgnoreFFF?If you dont know, just say "I dont know"', + }, + { + role: 'assistant', + status: 'success', + }, + ]); + + await expect(async () => { + const { content, message } = + await utils.chatPanel.getLatestAssistantMessage(page); + expect(content).toMatch(/WBIgnoreEEE.*cat/); + expect(content).toMatch(/WBIgnoreFFF.*dog/); + expect(await message.locator('affine-footnote-node').count()).toBe(2); + }).toPass({ timeout: 20000 }); + + // Ignore docs + await utils.settings.openSettingsPanel(page); + await utils.settings.ignoreDocForEmbedding(page, 'WBIgnoreDoc1'); + await utils.settings.ignoreDocForEmbedding(page, 'WBIgnoreDoc2'); + + await utils.settings.closeSettingsPanel(page); + + // Clear history + await utils.chatPanel.clearChat(page); + + // Ignored docs should not be used for embedding + await utils.chatPanel.makeChat( + page, + 'What is WBIgnoreEEE? What is WBIgnoreFFF?If you dont know, just say "I dont know"' + ); + + await utils.chatPanel.waitForHistory(page, [ + { + role: 'user', + content: 'What is WBIgnoreEEE? What is WBIgnoreFFF?', + }, + { + role: 'assistant', + status: 'success', + }, + ]); + + await expect(async () => { + const { content } = await utils.chatPanel.getLatestAssistantMessage(page); + expect(content).toMatch(/I dont know/i); + }).toPass({ timeout: 20000 }); + }); +}); diff --git a/tests/affine-cloud-copilot/e2e/utils/editor-utils.ts b/tests/affine-cloud-copilot/e2e/utils/editor-utils.ts index adaf150dac..a57b0dc2b4 100644 --- a/tests/affine-cloud-copilot/e2e/utils/editor-utils.ts +++ b/tests/affine-cloud-copilot/e2e/utils/editor-utils.ts @@ -6,7 +6,10 @@ import { pressEscape, selectAllByKeyboard, } from '@affine-test/kit/utils/keyboard'; -import { getBlockSuiteEditorTitle } from '@affine-test/kit/utils/page-logic'; +import { + clickNewPageButton, + getBlockSuiteEditorTitle, +} from '@affine-test/kit/utils/page-logic'; import type { EdgelessRootBlockComponent } from '@blocksuite/affine/blocks/root'; import type { MindmapElementModel, @@ -288,6 +291,19 @@ export class EditorUtils { await page.waitForTimeout(100); } + public static async createDoc(page: Page, title: string, docContent: string) { + await clickNewPageButton(page); + await page.keyboard.insertText(title); + await this.focusToEditor(page); + const texts = docContent.split('\n'); + for (const [index, line] of texts.entries()) { + await page.keyboard.insertText(line); + if (index !== texts.length - 1) { + await page.keyboard.press('Enter'); + } + } + } + public static async createCollectionAndDoc( page: Page, collectionName: string, diff --git a/tests/affine-cloud-copilot/e2e/utils/settings-panel-utils.ts b/tests/affine-cloud-copilot/e2e/utils/settings-panel-utils.ts new file mode 100644 index 0000000000..5d7b58935f --- /dev/null +++ b/tests/affine-cloud-copilot/e2e/utils/settings-panel-utils.ts @@ -0,0 +1,175 @@ +import { expect, type Page } from '@playwright/test'; + +const WORKSPACE_EMBEDDING_SWITCH_TEST_ID = 'workspace-embedding-setting-switch'; + +export class SettingsPanelUtils { + public static async openSettingsPanel(page: Page) { + if ( + await page.getByTestId('workspace-setting:indexer-embedding').isHidden() + ) { + await page.getByTestId('slider-bar-workspace-setting-button').click(); + await page.getByTestId('workspace-setting:indexer-embedding').click(); + await page.getByTestId('workspace-embedding-setting-wrapper').waitFor({ + state: 'visible', + }); + } + } + + public static async closeSettingsPanel(page: Page) { + if ( + await page.getByTestId('workspace-embedding-setting-wrapper').isVisible() + ) { + await page.getByTestId('modal-close-button').click(); + await page.getByTestId('workspace-embedding-setting-wrapper').waitFor({ + state: 'hidden', + }); + } + } + + public static async isWorkspaceEmbeddingEnabled(page: Page) { + const input = await page + .getByTestId(WORKSPACE_EMBEDDING_SWITCH_TEST_ID) + .locator('input'); + return (await input.getAttribute('value')) === 'on'; + } + + public static async waitForWorkspaceEmbeddingSwitchToBe( + page: Page, + enabled: boolean + ) { + const input = await page + .getByTestId(WORKSPACE_EMBEDDING_SWITCH_TEST_ID) + .locator('input'); + await expect(input).toHaveAttribute('value', enabled ? 'on' : 'off'); + } + + public static async toggleWorkspaceEmbedding(page: Page) { + const input = await page.getByTestId(WORKSPACE_EMBEDDING_SWITCH_TEST_ID); + await input.click(); + } + + public static async enableWorkspaceEmbedding(page: Page) { + const enabled = await this.isWorkspaceEmbeddingEnabled(page); + if (!enabled) { + await this.toggleWorkspaceEmbedding(page); + } + await this.waitForWorkspaceEmbeddingSwitchToBe(page, true); + } + + public static async disableWorkspaceEmbedding(page: Page) { + const enabled = await this.isWorkspaceEmbeddingEnabled(page); + if (enabled) { + await this.toggleWorkspaceEmbedding(page); + } + await this.waitForWorkspaceEmbeddingSwitchToBe(page, false); + } + + public static async uploadWorkspaceEmbedding( + page: Page, + attachments: { name: string; mimeType: string; buffer: Buffer }[] + ) { + await page.evaluate(() => { + delete window.showOpenFilePicker; + }); + + for (const attachment of attachments) { + const fileChooserPromise = page.waitForEvent('filechooser'); + await page + .getByTestId('workspace-embedding-setting-upload-button') + .click(); + + const fileChooser = await fileChooserPromise; + await fileChooser.setFiles(attachment); + } + } + + public static async removeAllAttachments(page: Page) { + const itemId = 'workspace-embedding-setting-attachment-item'; + let count = await page.getByTestId(itemId).count(); + + while (count > 0) { + const attachmentItem = await page.getByTestId(itemId).first(); + await attachmentItem + .getByTestId('workspace-embedding-setting-attachment-delete-button') + .click(); + await page.getByTestId('confirm-modal-confirm').click(); + await page.waitForTimeout(1000); + count = await page.getByTestId(itemId).count(); + } + } + + public static async removeAttachment(page: Page, attachment: string) { + const attachmentItem = await page + .getByTestId('workspace-embedding-setting-attachment-item') + .filter({ hasText: attachment }); + await attachmentItem + .getByTestId('workspace-embedding-setting-attachment-delete-button') + .click(); + await page.getByTestId('confirm-modal-confirm').click(); + await page + .getByTestId('workspace-embedding-setting-attachment-item') + .filter({ hasText: attachment }) + .waitFor({ + state: 'hidden', + }); + } + + public static async ignoreDocForEmbedding(page: Page, doc: string) { + // Open Dos Searcher + const ignoreDocsButton = await page.getByTestId( + 'workspace-embedding-setting-ignore-docs-button' + ); + await ignoreDocsButton.click(); + // Search and select the doc + const searcher = await page.getByTestId('doc-selector-layout'); + const searchInput = await page.getByTestId('doc-selector-search-input'); + + await searchInput.focus(); + await page.keyboard.insertText(doc); + + const pageListItem = searcher.getByTestId('page-list-item'); + await expect(pageListItem).toHaveCount(1); + const pageListItemTitle = pageListItem.getByTestId( + 'page-list-item-title-text' + ); + await expect(pageListItemTitle).toHaveText(doc); + + await pageListItem.getByTestId('affine-checkbox').check(); + + await searcher.getByTestId('doc-selector-confirm-button').click(); + + const ignoredDocs = await page.getByTestId( + 'workspace-embedding-setting-ignore-docs-list' + ); + await expect( + ignoredDocs + .getByTestId('workspace-embedding-setting-ignore-docs-list-item') + .filter({ hasText: doc }) + ).toBeVisible(); + } + + public static async clearAllIgnoredDocs(page: Page) { + const ignoredDocs = await page.getByTestId('ignore-doc-title').all(); + for (const ignoredDoc of ignoredDocs) { + const doc = await ignoredDoc.innerText(); + // Open Dos Searcher + const ignoreDocsButton = await page.getByTestId( + 'workspace-embedding-setting-ignore-docs-button' + ); + await ignoreDocsButton.click(); + // Search and select the doc + const searcher = await page.getByTestId('doc-selector-layout'); + const searchInput = await page.getByTestId('doc-selector-search-input'); + + await searchInput.focus(); + await page.keyboard.insertText(doc); + + const pageListItem = searcher.getByTestId('page-list-item'); + await expect(pageListItem).toHaveCount(1); + + await pageListItem.getByTestId('affine-checkbox').uncheck(); + + await searcher.getByTestId('doc-selector-confirm-button').click(); + } + } +}