From a603c06fabeae062a32cc7b31aac58f55a9bd520 Mon Sep 17 00:00:00 2001 From: akumatus Date: Thu, 24 Apr 2025 12:23:05 +0000 Subject: [PATCH] feat(core): add web search tool and reasoning params (#11912) Close [AI-60](https://linear.app/affine-design/issue/AI-60). ### What changed? - Add Exa web search tool - Add reasoning params --- packages/backend/server/package.json | 1 + .../server/src/plugins/copilot/config.ts | 9 ++ .../server/src/plugins/copilot/controller.ts | 14 ++- .../src/plugins/copilot/prompt/prompts.ts | 19 ++-- .../plugins/copilot/providers/anthropic.ts | 90 +++++++++++++++++-- .../src/plugins/copilot/providers/types.ts | 6 +- .../server/src/plugins/copilot/tools/index.ts | 1 + .../src/plugins/copilot/tools/web-search.ts | 35 ++++++++ packages/frontend/admin/src/config.json | 4 + .../src/blocksuite/ai/actions/doc-handler.ts | 2 +- .../blocksuite/ai/actions/edgeless-handler.ts | 2 +- .../core/src/blocksuite/ai/actions/types.ts | 2 +- .../components/ai-chat-input/ai-chat-input.ts | 2 +- .../blocksuite/ai/provider/setup-provider.tsx | 22 ++--- .../ai-button/services/network-search.ts | 5 +- .../e2e/chat-with/network.spec.ts | 75 ---------------- .../e2e/utils/chat-panel-utils.ts | 3 - yarn.lock | 27 ++++++ 18 files changed, 200 insertions(+), 119 deletions(-) create mode 100644 packages/backend/server/src/plugins/copilot/tools/index.ts create mode 100644 packages/backend/server/src/plugins/copilot/tools/web-search.ts diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index c582c7af57..ca9f4f7f57 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -79,6 +79,7 @@ "date-fns": "^4.0.0", "dotenv": "^16.4.7", "eventemitter2": "^6.4.9", + "exa-js": "^1.6.13", "express": "^5.0.1", "fast-xml-parser": "^5.0.0", "get-stream": "^9.0.1", diff --git a/packages/backend/server/src/plugins/copilot/config.ts b/packages/backend/server/src/plugins/copilot/config.ts index 023a35949f..fb3f3c80ec 100644 --- a/packages/backend/server/src/plugins/copilot/config.ts +++ b/packages/backend/server/src/plugins/copilot/config.ts @@ -16,6 +16,9 @@ declare global { unsplash: ConfigItem<{ key: string; }>; + exa: ConfigItem<{ + key: string; + }>; storage: ConfigItem; providers: { openai: ConfigItem; @@ -70,6 +73,12 @@ defineModuleConfig('copilot', { key: '', }, }, + exa: { + desc: 'The config for the exa web search key.', + default: { + key: '', + }, + }, storage: { desc: 'The config for the storage provider.', default: { diff --git a/packages/backend/server/src/plugins/copilot/controller.ts b/packages/backend/server/src/plugins/copilot/controller.ts index 31dfff5ca7..04c54f21aa 100644 --- a/packages/backend/server/src/plugins/copilot/controller.ts +++ b/packages/backend/server/src/plugins/copilot/controller.ts @@ -176,9 +176,15 @@ export class CopilotController implements BeforeApplicationShutdown { const retry = Array.isArray(params.retry) ? Boolean(params.retry[0]) : Boolean(params.retry); + const reasoning = Array.isArray(params.reasoning) + ? Boolean(params.reasoning[0]) + : Boolean(params.reasoning); + delete params.messageId; delete params.retry; - return { messageId, retry, params }; + delete params.reasoning; + + return { messageId, retry, reasoning, params }; } private getSignal(req: Request) { @@ -226,7 +232,7 @@ export class CopilotController implements BeforeApplicationShutdown { const info: any = { sessionId, params }; try { - const { messageId, retry } = this.prepareParams(params); + const { messageId, retry, reasoning } = this.prepareParams(params); const provider = await this.chooseTextProvider( user.id, @@ -257,6 +263,7 @@ export class CopilotController implements BeforeApplicationShutdown { ...session.config.promptConfig, signal: this.getSignal(req), user: user.id, + reasoning, }); session.push({ @@ -289,7 +296,7 @@ export class CopilotController implements BeforeApplicationShutdown { const info: any = { sessionId, params, throwInStream: false }; try { - const { messageId, retry } = this.prepareParams(params); + const { messageId, retry, reasoning } = this.prepareParams(params); const provider = await this.chooseTextProvider( user.id, @@ -322,6 +329,7 @@ export class CopilotController implements BeforeApplicationShutdown { ...session.config.promptConfig, signal: this.getSignal(req), user: user.id, + reasoning, }) ).pipe( connect(shared$ => diff --git a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts index 9223dc2554..cf76088860 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts @@ -1044,21 +1044,27 @@ const chat: Prompt[] = [ content: `You are AFFiNE AI, a professional and humorous copilot within AFFiNE. You are powered by latest GPT model from OpenAI and AFFiNE. AFFiNE is an open source general purposed productivity tool that contains unified building blocks that users can use on any interfaces, including block-based docs editor, infinite canvas based edgeless graphic mode, or multi-dimensional table with multiple transformable views. Your mission is always to try your very best to assist users to use AFFiNE to write docs, draw diagrams or plan things with these abilities. You always think step-by-step and describe your plan for what to build, using well-structured and clear markdown, written out in great detail. Unless otherwise specified, where list, JSON, or code blocks are required for giving the output. Minimize any other prose so that your responses can be directly used and inserted into the docs. You are able to access to API of AFFiNE to finish your job. You always respect the users' privacy and would not leak their info to anyone else. AFFiNE is made by Toeverything .Pte .Ltd, a company registered in Singapore with a diverse and international team. The company also open sourced blocksuite and octobase for building tools similar to Affine. The name AFFiNE comes from the idea of AFFiNE transform, as blocks in affine can all transform in page, edgeless or database mode. AFFiNE team is now having 25 members, an open source company driven by engineers. Today is: {{affine::date}}, User's preferred language is {{affine::language}}. # Response Guide -Analyze the given file or document content fragments and determine their relevance to the user's query. -Use the structure of the fragments to assess their relevance and provide the necessary response with cite sources using the citation rules below. +Use the webSearch tool to gather information from the web. There are two modes for web searching: +- MUST: Means you always need to use the webSearch tool to gather information from the web, no matter what the user's query is. +- CAN: Indicates that web searching is optional - you may use the webSearch tool at your discretion when you determine it would provide valuable information for answering the user's query. +Currently, you are in the {{searchMode}} web searching mode. -## Content fragments format: +I will provide you with some content fragments. There are two types of content fragments: - Document fragments, identified by a \`document_id\` and containing \`document_content\`. - File fragments, identified by a \`blob_id\` and containing \`file_content\`. +You need to analyze web search results and content fragments, determine their relevance to the user's query, and combine them to answer the user's query. +Please cite all source links in your final answer according to the citations rules. + ## Citations Rules -When referencing information from the provided documents or files in your response: +When referencing information from the provided documents, files or web search results in your response: 1. Use markdown footnote format for citations 2. Add citations immediately after the relevant sentence or paragraph 3. Required format: [^reference_index] where reference_index is an increasing positive integer 4. You MUST include citations at the end of your response in this exact format: - For documents: [^reference_index]:{"type":"doc","docId":"document_id"} - For files: [^reference_index]:{"type":"attachment","blobId":"blob_id","fileName":"file_name","fileType":"file_type"} + - For web search results: [^reference_index]:{"type":"url","url":"url_path"} 5. Ensure citations adhere strictly to the required format. Do not add extra spaces in citations like [^ reference_index] or [ ^reference_index]. ### Citations Structure @@ -1068,16 +1074,17 @@ Your response MUST follow this structure: 3. Citations section with all referenced sources in the required format Example Output with Citations: -This is my response with a citation[^1]. Here is more content with another citation[^2]. +This is my response with a document citation[^1]. Here is more content with another file citation[^2]. And here is a web search result citation[^3]. [^1]:{"type":"doc","docId":"abc123"} [^2]:{"type":"attachment","blobId":"xyz789","fileName":"example.txt","fileType":"text"} +[^3]:{"type":"url","url":"https://affine.pro/"} `, }, { role: 'user', content: ` -The following content is a relevant content segment: +The following are some content fragments I provide for you: {{#docs}} ========== diff --git a/packages/backend/server/src/plugins/copilot/providers/anthropic.ts b/packages/backend/server/src/plugins/copilot/providers/anthropic.ts index e49c1c2009..8190a3fa0d 100644 --- a/packages/backend/server/src/plugins/copilot/providers/anthropic.ts +++ b/packages/backend/server/src/plugins/copilot/providers/anthropic.ts @@ -1,5 +1,6 @@ import { AnthropicProvider as AnthropicSDKProvider, + AnthropicProviderOptions, createAnthropic, } from '@ai-sdk/anthropic'; import { AISDKError, generateText, streamText } from 'ai'; @@ -10,6 +11,7 @@ import { metrics, UserFriendlyError, } from '../../../base'; +import { createExaTool } from '../tools'; import { CopilotProvider } from './provider'; import { ChatMessageRole, @@ -34,6 +36,10 @@ export class AnthropicProvider override readonly capabilities = [CopilotCapability.TextToText]; override readonly models = ['claude-3-7-sonnet-20250219']; + private readonly MAX_STEPS = 20; + + private toolResults: string[] = []; + #instance!: AnthropicSDKProvider; override configured(): boolean { @@ -120,15 +126,24 @@ export class AnthropicProvider const [system, msgs] = await chatToGPTMessage(messages); const modelInstance = this.#instance(model); - const { text } = await generateText({ + const { text, reasoning } = await generateText({ model: modelInstance, system, messages: msgs, abortSignal: options.signal, + providerOptions: { + anthropic: this.getAnthropicOptions(options), + }, + tools: { + webSearch: createExaTool(this.AFFiNEConfig), + }, + maxSteps: this.MAX_STEPS, + experimental_continueSteps: true, }); if (!text) throw new Error('Failed to generate text'); - return text.trim(); + + return reasoning ? `${reasoning}\n${text}` : text; } catch (e: any) { metrics.ai.counter('chat_text_errors').add(1, { model }); throw this.handleError(e); @@ -145,21 +160,52 @@ export class AnthropicProvider try { metrics.ai.counter('chat_text_stream_calls').add(1, { model }); const [system, msgs] = await chatToGPTMessage(messages); - - const { textStream } = streamText({ + const { fullStream } = streamText({ model: this.#instance(model), system, messages: msgs, abortSignal: options.signal, + providerOptions: { + anthropic: this.getAnthropicOptions(options), + }, + tools: { + webSearch: createExaTool(this.AFFiNEConfig), + }, + maxSteps: this.MAX_STEPS, + experimental_continueSteps: true, }); - for await (const message of textStream) { - if (message) { - yield message; - if (options.signal?.aborted) { - await textStream.cancel(); + for await (const message of fullStream) { + switch (message.type) { + case 'reasoning': { + yield message.textDelta; break; } + case 'tool-result': { + if (message.toolName === 'webSearch') { + this.toolResults.push(this.getWebSearchLinks(message.result)); + } + break; + } + case 'step-finish': { + if (message.finishReason === 'tool-calls') { + yield this.toolResults.join('\n'); + this.toolResults = []; + } + break; + } + case 'text-delta': { + yield message.textDelta; + break; + } + case 'error': { + const error = message.error as { type: string; message: string }; + throw new Error(error.message); + } + } + if (options.signal?.aborted) { + await fullStream.cancel(); + break; } } } catch (e: any) { @@ -167,4 +213,30 @@ export class AnthropicProvider throw this.handleError(e); } } + + private getAnthropicOptions( + options: CopilotChatOptions + ): AnthropicProviderOptions { + if (options?.reasoning) { + return { + thinking: { + type: 'enabled', + budgetTokens: 12000, + }, + }; + } + return {}; + } + + private getWebSearchLinks( + list: { + title: string | null; + url: string; + }[] + ): string { + const links = list.reduce((acc, result) => { + return acc + `\n[${result.title ?? result.url}](${result.url})\n`; + }, '\n'); + return links + '\n'; + } } diff --git a/packages/backend/server/src/plugins/copilot/providers/types.ts b/packages/backend/server/src/plugins/copilot/providers/types.ts index 37a4d83a83..813ddd0afa 100644 --- a/packages/backend/server/src/plugins/copilot/providers/types.ts +++ b/packages/backend/server/src/plugins/copilot/providers/types.ts @@ -78,7 +78,11 @@ const CopilotProviderOptionsSchema = z.object({ const CopilotChatOptionsSchema = CopilotProviderOptionsSchema.merge( PromptConfigStrictSchema -).optional(); +) + .extend({ + reasoning: z.boolean().optional(), + }) + .optional(); export type CopilotChatOptions = z.infer; diff --git a/packages/backend/server/src/plugins/copilot/tools/index.ts b/packages/backend/server/src/plugins/copilot/tools/index.ts new file mode 100644 index 0000000000..af14d9ae45 --- /dev/null +++ b/packages/backend/server/src/plugins/copilot/tools/index.ts @@ -0,0 +1 @@ +export * from './web-search'; diff --git a/packages/backend/server/src/plugins/copilot/tools/web-search.ts b/packages/backend/server/src/plugins/copilot/tools/web-search.ts new file mode 100644 index 0000000000..ebd45ba738 --- /dev/null +++ b/packages/backend/server/src/plugins/copilot/tools/web-search.ts @@ -0,0 +1,35 @@ +import { tool } from 'ai'; +import Exa from 'exa-js'; +import { z } from 'zod'; + +import { Config } from '../../../base'; + +export const createExaTool = (config: Config) => { + return tool({ + description: 'Search the web for information', + parameters: z.object({ + query: z.string().describe('The query to search the web for.'), + mode: z + .enum(['MUST', 'CAN']) + .optional() + .describe('The mode to search the web for.'), + }), + execute: async ({ query, mode }) => { + const { key } = config.copilot.exa; + const exa = new Exa(key); + const result = await exa.searchAndContents(query, { + numResults: 10, + summary: true, + livecrawl: mode === 'MUST' ? 'always' : undefined, + }); + return result.results.map(data => ({ + title: data.title, + url: data.url, + summary: data.summary, + favicon: data.favicon, + publishedDate: data.publishedDate, + author: data.author, + })); + }, + }); +}; diff --git a/packages/frontend/admin/src/config.json b/packages/frontend/admin/src/config.json index b2a658c9a4..72704cfb1f 100644 --- a/packages/frontend/admin/src/config.json +++ b/packages/frontend/admin/src/config.json @@ -241,6 +241,10 @@ "type": "Object", "desc": "The config for the unsplash key." }, + "exa": { + "type": "Object", + "desc": "The config for the exa web search key." + }, "storage": { "type": "Object", "desc": "The config for the storage provider." diff --git a/packages/frontend/core/src/blocksuite/ai/actions/doc-handler.ts b/packages/frontend/core/src/blocksuite/ai/actions/doc-handler.ts index 55cc45d764..41856b9423 100644 --- a/packages/frontend/core/src/blocksuite/ai/actions/doc-handler.ts +++ b/packages/frontend/core/src/blocksuite/ai/actions/doc-handler.ts @@ -109,7 +109,7 @@ function actionToStream( where, docId: host.doc.id, workspaceId: host.doc.workspace.id, - networkSearch: visible?.value && enabled?.value, + mustSearch: visible?.value && enabled?.value, } as Parameters[0]; // @ts-expect-error TODO(@Peng): maybe fix this stream = await action(options); diff --git a/packages/frontend/core/src/blocksuite/ai/actions/edgeless-handler.ts b/packages/frontend/core/src/blocksuite/ai/actions/edgeless-handler.ts index 272d421f1d..d2a284d651 100644 --- a/packages/frontend/core/src/blocksuite/ai/actions/edgeless-handler.ts +++ b/packages/frontend/core/src/blocksuite/ai/actions/edgeless-handler.ts @@ -196,7 +196,7 @@ function actionToStream( host, docId: host.doc.id, workspaceId: host.doc.workspace.id, - networkSearch: visible?.value && enabled?.value, + mustSearch: visible?.value && enabled?.value, } as Parameters[0]; const content = ctx.get().content; diff --git a/packages/frontend/core/src/blocksuite/ai/actions/types.ts b/packages/frontend/core/src/blocksuite/ai/actions/types.ts index d109a6bc54..01a87e61fb 100644 --- a/packages/frontend/core/src/blocksuite/ai/actions/types.ts +++ b/packages/frontend/core/src/blocksuite/ai/actions/types.ts @@ -133,7 +133,7 @@ declare global { interface ChatOptions extends AITextActionOptions { sessionId?: string; isRootSession?: boolean; - networkSearch?: boolean; + mustSearch?: boolean; contexts?: { docs: AIDocContextOption[]; files: AIFileContextOption[]; diff --git a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/ai-chat-input.ts b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/ai-chat-input.ts index a5fcc52332..29a3a6de62 100644 --- a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/ai-chat-input.ts +++ b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/ai-chat-input.ts @@ -544,7 +544,7 @@ export class AIChatInput extends SignalWatcher(WithDisposable(LitElement)) { isRootSession: this.isRootSession, where: this.trackOptions.where, control: this.trackOptions.control, - networkSearch: this._isNetworkActive, + mustSearch: this._isNetworkActive, }); for await (const text of stream) { diff --git a/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx b/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx index b9cce2d7cf..194aa9535d 100644 --- a/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx +++ b/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx @@ -82,28 +82,22 @@ export function setupAIProvider( //#region actions AIProvider.provide('chat', async options => { - const { input, contexts, attachments, networkSearch, retry } = options; - const disableSearch = - !!contexts?.files.length || - !!contexts?.docs.length || - !!attachments?.length; - const promptName = - networkSearch && !disableSearch - ? 'Search With AFFiNE AI' - : 'Chat With AFFiNE AI'; + const { input, contexts, mustSearch } = options; + const sessionId = await createSession({ - promptName, + promptName: 'Chat With AFFiNE AI', ...options, }); - if (!retry) { - await AIProvider.session?.updateSession(sessionId, promptName); - } return textToText({ ...options, client, sessionId, content: input, - params: contexts, + params: { + docs: contexts?.docs, + files: contexts?.files, + searchMode: mustSearch ? 'MUST' : 'CAN', + }, }); }); diff --git a/packages/frontend/core/src/modules/ai-button/services/network-search.ts b/packages/frontend/core/src/modules/ai-button/services/network-search.ts index e4c0b3242a..4c8885cff1 100644 --- a/packages/frontend/core/src/modules/ai-button/services/network-search.ts +++ b/packages/frontend/core/src/modules/ai-button/services/network-search.ts @@ -3,7 +3,6 @@ import { type Signal, } from '@blocksuite/affine/shared/utils'; import { LiveData, Service } from '@toeverything/infra'; -import { map } from 'rxjs'; import type { FeatureFlagService } from '../../feature-flag'; import type { GlobalStateService } from '../../storage'; @@ -42,9 +41,7 @@ export class AINetworkSearchService extends Service { this.featureFlagService.flags.enable_ai_network_search.$; private readonly _enabled$ = LiveData.from( - this.globalStateService.globalState - .watch(AI_NETWORK_SEARCH_KEY) - .pipe(map(v => (v === undefined ? true : v))), + this.globalStateService.globalState.watch(AI_NETWORK_SEARCH_KEY), undefined ); diff --git a/tests/affine-cloud-copilot/e2e/chat-with/network.spec.ts b/tests/affine-cloud-copilot/e2e/chat-with/network.spec.ts index 4a25510672..1b56922791 100644 --- a/tests/affine-cloud-copilot/e2e/chat-with/network.spec.ts +++ b/tests/affine-cloud-copilot/e2e/chat-with/network.spec.ts @@ -1,5 +1,3 @@ -import { expect } from '@playwright/test'; - import { test } from '../base/base-test'; test.describe('AIChatWith/Network', () => { @@ -27,78 +25,5 @@ test.describe('AIChatWith/Network', () => { status: 'success', }, ]); - - await expect(async () => { - const { message } = await utils.chatPanel.getLatestAssistantMessage(page); - expect( - await message.locator('affine-footnote-node').count() - ).toBeGreaterThan(0); - }).toPass({ timeout: 10000 }); - }); - - test('should disable chat with image if network search enabled', async ({ - loggedInPage: page, - utils, - }) => { - await utils.chatPanel.enableNetworkSearch(page); - const isImageUploadEnabled = - await utils.chatPanel.isImageUploadEnabled(page); - expect(isImageUploadEnabled).toBe(false); - }); - - test('should prevent network search if disabled', async ({ - loggedInPage: page, - utils, - }) => { - await utils.chatPanel.disableNetworkSearch(page); - await utils.chatPanel.makeChat( - page, - 'What is the weather like in Shanghai today?' - ); - await utils.chatPanel.waitForHistory( - page, - [ - { - role: 'user', - content: 'What is the weather like in Shanghai today?', - }, - { - role: 'assistant', - status: 'success', - }, - ], - 20000 - ); - const { message } = await utils.chatPanel.getLatestAssistantMessage(page); - await expect(message.locator('affine-footnote-node')).toHaveCount(0); - }); - - test('should disable network search when chating with image', async ({ - loggedInPage: page, - utils, - }) => { - const image = - '/9j/4AAQSkZJRgABAQEAAAAAAAD/4QAuRXhpZgAATU0AKgAAAAgAAkAAAAMAAAABAAAAAEABAAEAAAABAAAAAAAAAAD/2wBDAAoHBwkHBgoJCAkLCwoMDxkQDw4ODx4WFxIZJCAmJSMgIyIoLTkwKCo2KyIjMkQyNjs9QEBAJjBGS0U+Sjk/QD3/2wBDAQsLCw8NDx0QEB09KSMpPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT3/wAARCAD6APoDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD0yWZ/MPPb0phmf1/SmSk+YaKuwWEMr56/pSGZ/X9KaTSU2y0SCZ/X9KPNf1/QUztTwOKNkIkEr+v6UvnP/e/So6KQ0iTzn/vfpR5z/wB79KjpM0ATfaJPX9KabmT+9+gqEnPSkJxQIebh/X9KYZnz979BUZkUck1XkuVHTmgC4Zn/AL36CozcsP4/0FUTKx74p6IepouMsG5kI4P6U0Syk/e/QUgTkZqUIKQhySORy36U8SP6/pTEAz7VJsxzTAcJXx1/Sjzn24z+lMOBSGgCTznx1/SmGV/736CkplA2Mkmk/vfoKheV8fe/QU+Soz0pAiIytn736CozIwOSePwpx703HrQxksaSy4wDgnAbHFASfONjdcY2+lNWaRE2KxCZzinC4lGAGPHsKQF6zMoTdyVx1wKs+fJ/e/QVVtpGKbd3GOmKm5q47CFkJ8w02nSD94abQhWAk0g5pcZoxiiwBUlMFOzQx3FopMim5pAOJxUZkA5JpskgHeqFzcYBOaTdgsTyajHGcck1Sm1J3OFGB61myXBkc+lKDnHcVk5tvQvlSLcdyxcLIxyelWQKzZAZE2gkMOVNS6dfLMNj/LKvBU1SfcTRpRpzmpgCGHFMjKkDHOamBGcE1VyRwBp4Timoe/pUgOTQIAOfemksDjNWAAKZImeRVXAi5ptSADpSkYoEJjimmnU00AQPUR9Klf7pqPFIoicGoz1qZxmoiKQxo+9TqbTqALdscCp81UhI6VNketNMC3L/AK00ynT8SN9Kjyaq4rDqO1NzTu1CYWDNGaKQkAUNoQZ5qOSTHFRyS4qpJNnNK4x0svPX8Kz76QiAmrPJqlqJxHg1nPYqO6Mz7SYzzVmKRiQwBNV4/KLgOpb2J4q9HJFGMLFj3zkVhFM2k12JEIk6ZDDsaztWtpQwvLY4lU/MB3FXxeRbwpABPQ9KsAqcjA56itbGL0MG28SyxusTKTJ161pS6vOE3EY3DjFc9qds1vrEYRSVZgCce9dJNbKEjY7cAYP0oVwujOHjJrIlZELjOARRZ/EWKS8MckRCKM5BrjPEFwEuniT7ynt6msyztpDIIlyZZm2ii7JZ7Xo/iWz1VPkkG49s9K2I5klzsYHHvXjEVtPphkVZSjjCnHHHc1a0Txjc2F/GsjF4GO0Z7DPWi7Qj2Apk5FRE1gX/AIljs7VZ1+YEAkA9jUemeMLLUY+G2uTjaetaJodjos03qc0yOZJAArA5HrT/AKVVxETjJIqKpz0qIpzxQx9BjdqikAxmpiOaY5UDLHFICAikBycelQTX0UQ65Y9AKW2O9N38R5NJjLKnBqTzKipc0rgakv3zUdSTH94ajqxoKeDTKUuAOTQJjiQBVaWWmzXGBwaoySk9D1pNiHyS5amZqME55p4INF9AFBwaz9WkUR7s1cJA5NY+psJQVB5rOb0KhuVLWS2mO2SVQxPGa0o7N0I2TBoz/DXMx6ZLJIShwQc11eiRgRhWB3dwfWs4LozWbtsyaOyeQbWUMvoRVuOxPlnJxx19KuwhUGD07U+Q4HbArS1jBtsypNOQYZwGZSeSKzdbuhbJheSO34Vs38xjhJAwCRzXDaxfG4vDbo3TIZuvsaTBGBbW7X2pTXEmDGoLHPc1raHpzSahJeMpwDhAR0rWh0pbbRwQgLHJbA59hWvp1itnp4cjLbRj60JAcnrUbRyTfMdzKFGOnqa5oJmSJsEFT8uOnHet3xLcE6gY8nCoe/8AEf8A61Q+H7aO+1KOJhkKAW9xnJoe4jXhtp7+1KgZUYVQR696dD4alsozNuzLztHTGa6TQRHc3t2kaqFRgCQP0rd+wiRzlep447VSQ72Oc0xLm2GXZiBgc10NveCQAMcN6Ur6cACuB1qq0SxuQMlgevqaaugvc0iQR1pnGMk1Vh80gBVPPU9hVmONQcM2+Q9B2FVcQqQM/PQHuaebO2JBkJY/WnyGUjAwPwquUZRlpAT6UrjFfR7SU7lUKfUVShszZySxM27DZBI7Vajm2TBV3e+OafckGQN3KjNC1CzKxTAzSVKehpuKYzQl/wBYajp8v+sNRvIEHWmJsHcAelVJZuuKZNcj1qm7knOaGGoskhc02gGjNZgLSEgc+lGaimOI2PfFO4EU1yHcop6dayJnV7ofOV5pY5JEJ7kmq88MpcOq5x1FZN3ZolY2Yo1yGyM9mFX7dwHHAB7HFZWnFtg3DgHDKRWkf3TAj5l7VSJe5oiRg4JHyt19jQZicg/QVX8wGMMrE56fWp4drAccnlaogbfR77YqThtpIrz+w0ySXWyGUkZLHI/hr0e5AIyDn5c1jk21ncyS/KGIAX8//r0NK4XLlnpp8iNWHHU0/U41t7DauenQda1LaRJEGCOFzmsTxHcGKFmH3QAMeuaYjzTX5QbyVmGTgEe3bmpfCsqQ/abpyAVTGf6Vm6/KDeTKpPzNjk5xTrUNb6Ufm/1rZ2jv6VHmUej/AA8LTx3crdGc4x+p/Ou6jhA5x1rjPh3btHpILDBZufoK7kEYwKpbEsq3MbBDtHzVUjtY4xmbl85NaxGQMioJIVcknNXYVyhJIv3VGQOuOlLGFPRwrexp8qeUCCOD7VXMSy5JA59R/WoZaQtwlzGhZSzL7YNUXuJDnduBHqKvDzIRgM209c/MKoXLsX2jBB9DkVnNtbGsUhltJLJdDMeUzyRV+4/1nTgVHY2xjy7ZGKdJknPrV07tXZM7X0I+9L+FKB6U/HvWhBLcyBHPNZk1115pt9efvGGf84rOMpL5P4UNgiZ5Gc5oD+9RA5pam4yXPuKM89RUJOBQHzSuBMT6VVmdidoNSvJgZNQR/vJumM0mA9LHcAx6nriphZAcrg46g1fhHyY4NK6YHcUrIVzP8kg/KF+nrTowwJRlbB5API/OrQjJcED8xTzGwIYArk846U7BcZCUI8sLg9gf8asSQmOPcudobdkVC8bRjcFBUckD+da0cYltgwIIK8+4qkJmVLc5jAHXaRiub1q1kwJwzZU4OO+eldJJb/6TtUYJOcD9affaaskZXBYdgPXtQ0F7CaKJZIck4ULjjvjj+lYfi+5eIMpOQASw9DjA/LJNdjplqLawjTBBC4OfWvNfiTePHerEm796u0YHX/J/pQ9ETfU4Zo2vtQLRsW3Hqa1RHG9tDAqsZNw4A5bn/GoREttJHFajdLtAc4zyef8AP4V1fhvw5KZIbmYH5mLAkccdKnco7vwxZGw02OI8sFAJA7+ldDGMYqhZx+Wir6DFXwcirSESZB6Uw4Apc/hTH6igLFW6jMoI5zVARNEeTJj1HNaUqeYpG4qfUdqxbm2uQ+Y53YH1OCKllw10ZY84gYVw2exqAoskgIXD1AZLj7siq/8AtEYNWbONjN83X0qG76GtkkWyfLg29z1quRVibk4x04qEjFbJaGPUYBjk06kpcUrsZhX0UsU5Eikeh7Hiquea7G6tklLKyhlPYisO90R0y1sdw/unqKGiEzNBOKdn3phDRnaylW7g0mTSKQ4mjPem5pCalgR3MoA5as6TVlt5gwY8HvU2oyFYSwGfrXJXTiaQ8MpHcHipb1KW1z0uxvlmiVlKtkdqtmbfxtrzPSbq5t3CgttY43Aniu2sZp5IR5gYnHUjGaadybGzHKp+UnBHrUySKVKsACfQ5rn769ltoSQ23A9M155qPinV7l5Ugu2hhVjucYXHsMVSYmrK56+XUckjI9RkH61o2JURlUb5TyF9K+f7PxlrWnPtttVnZM5KyAOP1rvvDHj5rx44NSjWC6YbkcDCyjvT2EtdDubeNTezE8FflFaghUg8dSKyYZVMxkHO7BHuMVsRPvH4UwaYECOM+mK8u8URNc6rJc3DARQgrHkdW6kfoBn3r1G5P7sjHJFcZeaGbyQLMx5fdkjqDyaTBGH4S8LtJeC5ulVlGSfdu5/pXcFIkmEUfBQBcAcVd07T0sbUKqgYGTj1qtbwia6kkUtgsSOOBTSsO5oRAYH0qcEAc1WlmS2j3MyqB3JqC2vJbqTcEZY+x9adxGh19qCPxojHbv6VKEFAinIGxxn6CojGD/Ft9zWgYlPNVbm2V0Khc1LKTMiYxxTA7wxPYVehj81wfugDPFU4tOxc735x90CtZEEKE4xxUpO+pTatoVZ48OahI9ammILZB4qHNaC3EwN2afj2poGalwPU0IZclQecQfSmlAWokkJmO1S36U4BiOcL+tMyKN5Y28yYkUfWufvtOS2BkhkygPKmrupi6hm/fO7Rk/Ke1ULmTNsVz71EilcpZ9KQnjpTAc9KXJpFXKGphzbEr0FYtnYfai7D7o6jFdPIiyDDCizhQPtVcYqGru5SZX0fTY4nwyrtPXIyK6yG1TYAMYA4qhFCqPuPGe1acMqYC5wPpmqRLMnX7YQ6JezKuXSIkZHOa8UvPk0q1C5/e5Zj6nNfQNzbx3ttJbuQUlUqfUZrxXW9Fl0SSaw1KJ0iDFrecDIwex9qtWuQ07HL+xrrtR1+bVfDljJcqizaeQsTooU7e68duBxXPx2cCPme4i8vr8p61cmkF4ix2sD/AGSM5yAcuf6CtUkk79SNW1boe1aJN9s0a2lXBLKDXRWZycevX6VzPgizmh8LWsdwNrhc7e4zziuptoygz+FZo2nuOuBvcDtUEsQyBtHBFXHjyOe1QyDCd8mixCGySAR7ScZHWq4ljthtUjPcmuX8f+Mk8NWq21qFl1OcfIp5EYP8RHf2FeN3+pajNIz32oTvK/LKJD9cccD6UXHex9C+VBdThppPMccqpIwPoK0I4gBxn86+fNH/AOEmtbE6lpz3q2qZ3OSWj4POR9a9S8DeOV1yM214ohvogA6HgN7ii1txJp7HeRjCAdaf70yNwQCDmnUAxaikzTyQB1qpc3Kp1ODQwQoXnPSm3D8BTzRDKJB6mnyAEcjmpGU3GBUNWJRkVDjNUlcsdHweakyKb0SmeYPWnYDWkUCQ4AFIKWRgZDninYpmRWubZLmExuoINcdqdrJYyFGyUP3W9RXckcVn6tpy39m0bDDYyrehpNXQJ2OCjfNSZzUDxvbTtFMMOpwRUgPrWZpYnjwTggc1bht9j7goA9c1SRwCMVpW0gIGTxQDJAR0PJ9hUscmwn5QR6d6sIFcfKuPwqWO15zzz15oFcdG4cYKlfxptzapcwmG4hE8TcYIBzV2GJU/hUj3GasgoCCQBVJCuc7Z+CtFiJlXRrfzCeN67sfnV250O0uECTxRrHG2QEAUfpW15yn+IYHvVa5KlCFw3qKYJu4kMUUZ3cADAVR0Aq7GgOGU5rnrbU4ppGUPhVbGP7p961ba9UnbwMdCO9F0U4PsaJTORjrWH4j1iPRNNluCN0i8Kg6sx6CtlJATyetZ2taImrWs0edrsMq2PusOh/Ch3Ekr6nzxfXlzqPiGO8vnZ5rgknd/Cc4C47Y9Kx7ot9qkLj5txzmvV/FXwzuLmzF7pLM9/D80sBAHmnuynsT6d681nEVzMRcE29yDtkV1xyOv0PtQiJb3LOj+KdS0uH7Jb3EggfKvHn5WVuoI6d663wnZmTxJZCPiRonDZX7wUZGfxrkraxsYpAqTtdTN91IVJJPYV678P/CVzYk6nqi7bhkEcUQ5ESenuSetU9rMmNr3OttjNGgUrkD9KlN5gfMrCrZQAcCqlyg2Ejg1BpcrXGogIecZ7VkXF75r7VJ+tTXEZGduCx6EnpWPcB/OwCfr2qWykkdFYyADhs+9aHmAgAZzWTpKeXCBu578YrYA796L6CIJRgdKgyAatypkc1WkiBQmnzFxVyGaUbDgiqe9vapzDnuaZ9mWjnNfZHQSjMhyKVCRwamkA3moyOKs5B/amkZFIMjg08cimByPirScoLyFfnXhgO4rm4wxHPHua9LuYVljKsAVYYIrzfUbdrDUJYGDbQcqc9RWclYuL6Cjah9TVm2mJcAcD2qiJF7D8SasQuxPy/Knc1IzftycDrVtC2etY1tdoPlZ8n0FakEocfKtMTLqOe5qYberbmPtVYGniQgYzTQh7uvGIC3PftUYkbOWiCfQ5p2TjO41SuZliBLMxyMdafQaVzH1uyNnexalCGVFJEoHRlPrT7W+ubyNTYoryM38ZOFHrxUOpamv2WRV3MWUrtPQ8VY8NTC2s4lRV2gDcB1z3qUtTtUrUtVqdTpUN0I1N0yFgMcA8/nWjVSGYlAY8NjqpPOKeJRJzG2GByyHrV2OJu7uSPGHcMOGHUjvXPa14F0LW5/tF9p6NcEYaVCULfUjrW+JhnByp9CKeTnvSJuc1pvhPRdDfda2CIw6OfmP610MbqRwQfoaHjz1OfemmNR2xjvSux7jy+BVebLj/wCtUhAH0FRnkZBxj3pgZ1zCvJ2jJrEuQwkwMdeuK6SaMnPIx7VWjsVd+VBqWhp2G6TbSYDOcg+lbJHA46UQxLEgUDpTz92gV9SrJ1IqNBnIp8p2EnFV47r95tCGlY3g9CORMHGOlMxU1zKA3CEk1U+0S/8APE0G0ZaHSNjeaPlPWmSN+8NJkda1OAcUI57UYPapAcjmgj0oAiIyOa5TxlpxlhjuY1+dOG9xXXkVQ1aFZrORGGQVoeqBOzPM4dp5ZsqOTinS3DScL8sY6KKjuozaTtAP72frQAsY3MRu9PSsjVE0JKEFiQOwHU1t2d5gBXIX0UH+dc49wUIwMyHoO9SQ3Pkvy26Q9eeF/wDr1KdgavqdnHMCAenoKnDg1gWVwxG5ifxrTiuM8k1ZFjQByKimtlk5Iz6ZpY3BxU4IPGaEFzJutJjlTGMH271RGmSWEnmwMQBjK9jXT4BIqMxqc5Gc0y1UdrDLGYyxq6kq+MfX2NX8CQZ+5IOhFUY4xGeOOamE38LAYHejmILbkyABjtkH3WHQ0yOboGIDDioftTj5WXKmmvGxbdnKt0x3/wADRcRfEnPPFIZAB/KqQDhOWJX9RTgT0bv+tFwJXkUjggN6VXMpzgjk1KY80gh/Ee9JjGR8g5PWp4YSDuzkGpI4VA6VOABTFcBwOaQ807FMJApAVpsADjvUQHz5AqWQqc00YyKqxrF2HSAFOMZqPyz61NTqdh3ZPKBvPNMwexp8ijzDSYqkYDQWFSI/rTMc5yaWgCbINV7pCYzxkYp4JHSn+YDwwpAeb+JYfJk81V78msISA/MxzjnHrXoniPRGvbZmtl3HHK/4VwMmlX1sSJraQYPUDIArJpmkXoVi5j+bP7xu/wDdH+NOtuHB259vWojkOdwK+xFPSUhwqjpWZdjoLN22ZbqR0rRiOMHNZNm5IBJya04smrRBcjmNWY5uvNVEHpUoFMTLwl460vmHPWqgJ4Gafk5+lFwsWSc80YJIIpYyCMHFTAAUCGpHkc9v5VPHHgYPQ9RRFjkVLkA9aaEwEYH+etJsAan5BApucnFPQQ4x8cUoTFKM4paGMUCl6U0yKByaqTXqgYBpgk2WZJlA61Ukuc8A1TkmaQ9ePSmFyOtCRoo2LBmwetKk2TyaoPIR/wDrpI5j607mljXEmR1p3me9Z6THFO8+i5Njdk/1ppuPenSf6w03NUtjATFLiloosAmKMUAU6gTbGYNNMKv95QfwqcUuKkZmXGi2dz/rLdD+FZc3g3T5CWWPaevHFdRikwKHZgm0coPCKxH93Iw+tO/sGeL7rAiupwKQilZDuzl/sM0fVCfpSeUw+8CPwrqCgNRm2R+qijlQXZz8Y9qlCDFa72MZz8v5VWk0xXB2swHpmpaY0ykhVBnOeeMVIHaQfKPxNTCxMYwFzTgMcdMUrBcZEHABY9akCMX5zinDBOB2qYDNMTY0AU/gUHCdaqTXHJCmmloCTZZMyp3qCS6GODVGSUnqaiMmB1qkjRQ7k8twx6nFU3kJPWkkkNQPJgUmzSMSwJBSGUdjVbJJ69aQk+tJNl2HSSEnANEbkmq5yTmpYzg0uo7KxdQ8U7JqJH4pfMqiDqZD+8NMpJR+8PNIKtbHKPFA60lOpgLTCcCgnFQySdqQE8ZyamqtDzg1ZH3aTAKKM0nWkAZo6mkp1ABijFFIaAFppxQTRSbCwUwxqeqilzTS/vQkxjfsy9QMUhQoOlPEgHU1FNMoBOadkIp3VxjgGs55vU0+4k3OSKqOfmpPQ3irDzJmkJO2ocgd6N4x1ouaWEkOM1GTmkkfioc0mxpImzRvz3qB3PSm7znrSHYs0oIHWmpkilxTBjw+Kd5hqI5peaLisdjN/rD6U1OtPm++aaAB0rRbHHYdS5pKa5AHWmAyWQAVU8zJpZ5OSKbHgnNIC9CelWPpVaLgVYB9aTAU0U6mnikAvFHam5pQfWgLDu1NP3qCaKGAUh6U49KYTStqMic4FVpJiO9TynANZtzJgHmm2CVwe8IPWoZLliDzVZ5KjkkI/Glc1jEcZOTzULkGmEknNN3Gg2SAmojJjvUhIIqvNkHOKlsaQGXNGc9/pVQyEGpEkyOaVy2iU0g6ioy49aeCMjmmIuR4wKf+FQQh5HCojMT6AkZ9M+tPEgJ4NNEPcfRxURkApvm0wsdvJ/rDQKO6/wC6P5U7vVHGhhOBVaWTA61O/eq1xTQFOSQk1JCeRUJ+/UsPagOhpRdKlB4qGP7lT0rgOJPagHIptLQAEkGlFJRQA6lHSmVJUoBpNRucVJUcnSqQFSZ+DWVcydRWnN0NZNz1NDKiVi/aoZJBnk041Wk71mzogO3ikMgBqOmGmaIn8z6VFI+RTO1MkqWBC55NIH7USU0UGnQlBqQDvUaVOtCJZYtbqa2BWFyocfMMdf8AOamkvZZEKssZDKVJ24PJz2qutOP3ashxVxtxM0xDMFUqMfIMZ9z71XzUj96UdBQVFaH/2Q=='; - - const buffer = Buffer.from(image, 'base64'); - await page.evaluate(() => { - delete window.showOpenFilePicker; - }, buffer); - - const fileChooserPromise = page.waitForEvent('filechooser'); - // Open file upload dialog - await page.getByTestId('chat-panel-input-image-upload').click(); - - const fileChooser = await fileChooserPromise; - await fileChooser.setFiles({ - name: 'image1.jpeg', - mimeType: 'image/jpeg', - buffer: buffer, - }); - - await page.waitForSelector('ai-chat-input img'); - const isNetworkSearchEnabled = - await utils.chatPanel.isNetworkSearchEnabled(page); - await expect(isNetworkSearchEnabled).toBe(false); }); }); diff --git a/tests/affine-cloud-copilot/e2e/utils/chat-panel-utils.ts b/tests/affine-cloud-copilot/e2e/utils/chat-panel-utils.ts index 13c6e1a1bf..a6c11b9498 100644 --- a/tests/affine-cloud-copilot/e2e/utils/chat-panel-utils.ts +++ b/tests/affine-cloud-copilot/e2e/utils/chat-panel-utils.ts @@ -37,9 +37,6 @@ export class ChatPanelUtils { } await page.getByTestId('sidebar-tab-chat').click(); await expect(page.getByTestId('sidebar-tab-content-chat')).toBeVisible(); - // TODO: remove this - // after network search is disabled by default - await this.disableNetworkSearch(page); } public static async closeChatPanel(page: Page) { diff --git a/yarn.lock b/yarn.lock index f7c865f5aa..f7c87715ba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -961,6 +961,7 @@ __metadata: date-fns: "npm:^4.0.0" dotenv: "npm:^16.4.7" eventemitter2: "npm:^6.4.9" + exa-js: "npm:^1.6.13" express: "npm:^5.0.1" fast-xml-parser: "npm:^5.0.0" get-stream: "npm:^9.0.1" @@ -19232,6 +19233,15 @@ __metadata: languageName: node linkType: hard +"cross-fetch@npm:~4.1.0": + version: 4.1.0 + resolution: "cross-fetch@npm:4.1.0" + dependencies: + node-fetch: "npm:^2.7.0" + checksum: 10/07624940607b64777d27ec9c668ddb6649e8c59ee0a5a10e63a51ce857e2bbb1294a45854a31c10eccb91b65909a5b199fcb0217339b44156f85900a7384f489 + languageName: node + linkType: hard + "cross-inspect@npm:1.0.1": version: 1.0.1 resolution: "cross-inspect@npm:1.0.1" @@ -20077,6 +20087,13 @@ __metadata: languageName: node linkType: hard +"dotenv@npm:~16.4.7": + version: 16.4.7 + resolution: "dotenv@npm:16.4.7" + checksum: 10/f13bfe97db88f0df4ec505eeffb8925ec51f2d56a3d0b6d916964d8b4af494e6fb1633ba5d09089b552e77ab2a25de58d70259b2c5ed45ec148221835fc99a0c + languageName: node + linkType: hard + "ds-store@npm:^0.1.5": version: 0.1.6 resolution: "ds-store@npm:0.1.6" @@ -21191,6 +21208,16 @@ __metadata: languageName: node linkType: hard +"exa-js@npm:^1.6.13": + version: 1.6.13 + resolution: "exa-js@npm:1.6.13" + dependencies: + cross-fetch: "npm:~4.1.0" + dotenv: "npm:~16.4.7" + checksum: 10/17135b8d586032e8500f00c9b5f4e45cbe49ffb30a0ada77b42112ce71f5405bcdebfd8aed2edda175a5d0b15f57524836c825406cf07a8d71c14d37796056e1 + languageName: node + linkType: hard + "execa@npm:^1.0.0": version: 1.0.0 resolution: "execa@npm:1.0.0"