From fd3a2756f8a7ca1557b77cb1b704c5b83992d1a7 Mon Sep 17 00:00:00 2001 From: yoyoyohamapi <8338436+yoyoyohamapi@users.noreply.github.com> Date: Fri, 23 May 2025 06:18:28 +0000 Subject: [PATCH] fix(core): in edgeless mode, an error occurs when asking AI questions without selecting any content (#12437) ### TL;DR fix: in edgeless mode, an error occurs when asking AI questions without selecting any content > CLOSE AI-133 ## Summary by CodeRabbit - **New Features** - Added support for asking AI input in edgeless mode when no content is selected. - Enhanced AI panel behavior with improved input handling and chat message sending in edgeless mode. - **Bug Fixes** - Improved handling of AI chat input visibility and context extraction in edgeless mode. - **Tests** - Introduced new end-to-end tests to verify chat interactions with AI in edgeless mode. - Centralized editor content removal logic in test utilities for better maintainability. --- .../core/src/blocksuite/ai/utils/edgeless.ts | 19 ++--------- .../ai/utils/get-edgeless-copilot-widget.ts | 16 ++++++++++ .../blocksuite/ai/utils/selection-utils.ts | 2 +- .../ai/widgets/edgeless-copilot/constant.ts | 1 + .../ai/widgets/edgeless-copilot/index.ts | 32 +++++++++++++++++-- .../e2e/basic/chat.spec.ts | 27 ++++++++++++++++ .../e2e/utils/editor-utils.ts | 8 +++-- 7 files changed, 83 insertions(+), 22 deletions(-) create mode 100644 packages/frontend/core/src/blocksuite/ai/utils/get-edgeless-copilot-widget.ts create mode 100644 packages/frontend/core/src/blocksuite/ai/widgets/edgeless-copilot/constant.ts diff --git a/packages/frontend/core/src/blocksuite/ai/utils/edgeless.ts b/packages/frontend/core/src/blocksuite/ai/utils/edgeless.ts index 8da362a637..5c4fd05551 100644 --- a/packages/frontend/core/src/blocksuite/ai/utils/edgeless.ts +++ b/packages/frontend/core/src/blocksuite/ai/utils/edgeless.ts @@ -6,14 +6,9 @@ import { type ShapeElementModel, } from '@blocksuite/affine/model'; import { matchModels } from '@blocksuite/affine/shared/utils'; -import type { BlockComponent, EditorHost } from '@blocksuite/affine/std'; +import type { BlockComponent } from '@blocksuite/affine/std'; import type { GfxModel } from '@blocksuite/affine/std/gfx'; -import { - AFFINE_EDGELESS_COPILOT_WIDGET, - type EdgelessCopilotWidget, -} from '../widgets/edgeless-copilot'; - export function mindMapToMarkdown(mindmap: MindmapElementModel) { let markdownStr = ''; @@ -45,17 +40,7 @@ export function isMindmapChild(ele: GfxModel) { return ele?.group instanceof MindmapElementModel && !isMindMapRoot(ele); } -export function getEdgelessCopilotWidget( - host: EditorHost -): EdgelessCopilotWidget { - const rootBlockId = host.store.root?.id as string; - const copilotWidget = host.view.getWidget( - AFFINE_EDGELESS_COPILOT_WIDGET, - rootBlockId - ) as EdgelessCopilotWidget; - - return copilotWidget; -} +export { getEdgelessCopilotWidget } from './get-edgeless-copilot-widget'; export function findNoteBlockModel(blockElement: BlockComponent) { let curBlock = blockElement; diff --git a/packages/frontend/core/src/blocksuite/ai/utils/get-edgeless-copilot-widget.ts b/packages/frontend/core/src/blocksuite/ai/utils/get-edgeless-copilot-widget.ts new file mode 100644 index 0000000000..31d9cc294f --- /dev/null +++ b/packages/frontend/core/src/blocksuite/ai/utils/get-edgeless-copilot-widget.ts @@ -0,0 +1,16 @@ +import type { EditorHost } from '@blocksuite/affine/std'; + +import type { EdgelessCopilotWidget } from '../widgets/edgeless-copilot'; +import { AFFINE_EDGELESS_COPILOT_WIDGET } from '../widgets/edgeless-copilot/constant'; + +export function getEdgelessCopilotWidget( + host: EditorHost +): EdgelessCopilotWidget { + const rootBlockId = host.store.root?.id as string; + const copilotWidget = host.view.getWidget( + AFFINE_EDGELESS_COPILOT_WIDGET, + rootBlockId + ) as EdgelessCopilotWidget; + + return copilotWidget; +} diff --git a/packages/frontend/core/src/blocksuite/ai/utils/selection-utils.ts b/packages/frontend/core/src/blocksuite/ai/utils/selection-utils.ts index 3db98ce007..2ae7670d36 100644 --- a/packages/frontend/core/src/blocksuite/ai/utils/selection-utils.ts +++ b/packages/frontend/core/src/blocksuite/ai/utils/selection-utils.ts @@ -29,7 +29,7 @@ import { import { getContentFromSlice } from '../../utils'; import type { CopilotTool } from '../tool/copilot-tool'; -import { getEdgelessCopilotWidget } from './edgeless'; +import { getEdgelessCopilotWidget } from './get-edgeless-copilot-widget'; export async function selectedToCanvas(host: EditorHost) { const gfx = host.std.get(GfxControllerIdentifier); diff --git a/packages/frontend/core/src/blocksuite/ai/widgets/edgeless-copilot/constant.ts b/packages/frontend/core/src/blocksuite/ai/widgets/edgeless-copilot/constant.ts new file mode 100644 index 0000000000..94e51b8f8b --- /dev/null +++ b/packages/frontend/core/src/blocksuite/ai/widgets/edgeless-copilot/constant.ts @@ -0,0 +1 @@ +export const AFFINE_EDGELESS_COPILOT_WIDGET = 'affine-edgeless-copilot-widget'; diff --git a/packages/frontend/core/src/blocksuite/ai/widgets/edgeless-copilot/index.ts b/packages/frontend/core/src/blocksuite/ai/widgets/edgeless-copilot/index.ts index 95e296b914..ccf30178d0 100644 --- a/packages/frontend/core/src/blocksuite/ai/widgets/edgeless-copilot/index.ts +++ b/packages/frontend/core/src/blocksuite/ai/widgets/edgeless-copilot/index.ts @@ -27,13 +27,14 @@ import { styleMap } from 'lit/directives/style-map.js'; import { literal, unsafeStatic } from 'lit/static-html.js'; import type { AIItemGroupConfig } from '../../components/ai-item/types.js'; +import { AIProvider } from '../../provider/index.js'; +import { extractSelectedContent } from '../../utils/extract.js'; import { AFFINE_AI_PANEL_WIDGET, AffineAIPanelWidget, } from '../ai-panel/ai-panel.js'; import { EdgelessCopilotPanel } from '../edgeless-copilot-panel/index.js'; - -export const AFFINE_EDGELESS_COPILOT_WIDGET = 'affine-edgeless-copilot-widget'; +import { AFFINE_EDGELESS_COPILOT_WIDGET } from './constant.js'; export class EdgelessCopilotWidget extends WidgetComponent { static override styles = css` @@ -95,6 +96,31 @@ export class EdgelessCopilotWidget extends WidgetComponent { if (input instanceof AffineAIPanelWidget) { input.setState('input', referenceElement); + const aiPanel = input; + // TODO: @xiaojun refactor these scattered config overrides + if (aiPanel.config && !aiPanel.config.generateAnswer) { + aiPanel.config.generateAnswer = ({ finish, input }) => { + finish('success'); + aiPanel.hide(); + extractSelectedContent(this.host) + .then(context => { + AIProvider.slots.requestSendWithChat.next({ + input, + context, + host: this.host, + }); + }) + .catch(console.error); + }; + aiPanel.config.inputCallback = text => { + const panel = this.shadowRoot?.querySelector( + 'edgeless-copilot-panel' + ); + if (panel instanceof HTMLElement) { + panel.style.visibility = text ? 'hidden' : 'visible'; + } + }; + } requestAnimationFrame(() => { this._createCopilotPanel(); this._updateCopilotPanel(input); @@ -314,3 +340,5 @@ declare global { [AFFINE_EDGELESS_COPILOT_WIDGET]: EdgelessCopilotWidget; } } + +export * from './constant'; diff --git a/tests/affine-cloud-copilot/e2e/basic/chat.spec.ts b/tests/affine-cloud-copilot/e2e/basic/chat.spec.ts index ef97482f58..b33214e951 100644 --- a/tests/affine-cloud-copilot/e2e/basic/chat.spec.ts +++ b/tests/affine-cloud-copilot/e2e/basic/chat.spec.ts @@ -484,4 +484,31 @@ test.describe('AIBasic/Chat', () => { }, ]); }); + + test('should support chat with ask ai input in edgeless mode when nothing selected', async ({ + loggedInPage: page, + utils, + }) => { + await utils.chatPanel.closeChatPanel(page); + await utils.editor.switchToEdgelessMode(page); + await utils.editor.removeAll(page); + + await page.mouse.move(300, 300); + await page.mouse.down({ button: 'right' }); + await page.mouse.move(350, 350); + await page.mouse.up({ button: 'right' }); + + await page.keyboard.type('Who are you?'); + await page.keyboard.press('Enter'); + await utils.chatPanel.waitForHistory(page, [ + { + role: 'user', + content: 'Who are you?', + }, + { + role: 'assistant', + status: 'success', + }, + ]); + }); }); diff --git a/tests/affine-cloud-copilot/e2e/utils/editor-utils.ts b/tests/affine-cloud-copilot/e2e/utils/editor-utils.ts index bb661bcc05..f2ba407246 100644 --- a/tests/affine-cloud-copilot/e2e/utils/editor-utils.ts +++ b/tests/affine-cloud-copilot/e2e/utils/editor-utils.ts @@ -386,14 +386,18 @@ export class EditorUtils { ); } + public static async removeAll(page: Page) { + await selectAllByKeyboard(page); + await page.keyboard.press('Delete'); + } + public static async askAIWithEdgeless( page: Page, createBlock: () => Promise, afterSelected?: () => Promise ) { await this.switchToEdgelessMode(page); - await selectAllByKeyboard(page); - await page.keyboard.press('Delete'); + await this.removeAll(page); await createBlock(); await pressEscape(page, 5); await selectAllByKeyboard(page);