feat(core): add web search tool and reasoning params (#11912)

Close [AI-60](https://linear.app/affine-design/issue/AI-60).

### What changed?
- Add Exa web search tool
- Add reasoning params

packages/backend/server/src/plugins/copilot/config.ts

@@ -16,6 +16,9 @@ declare global {
       unsplash: ConfigItem<{
         key: string;
       }>;
+      exa: ConfigItem<{
+        key: string;
+      }>;
       storage: ConfigItem<StorageProviderConfig>;
       providers: {
         openai: ConfigItem<OpenAIConfig>;
@@ -70,6 +73,12 @@ defineModuleConfig('copilot', {
       key: '',
     },
   },
+  exa: {
+    desc: 'The config for the exa web search key.',
+    default: {
+      key: '',
+    },
+  },
   storage: {
     desc: 'The config for the storage provider.',
     default: {

packages/backend/server/src/plugins/copilot/controller.ts

@@ -176,9 +176,15 @@ export class CopilotController implements BeforeApplicationShutdown {
     const retry = Array.isArray(params.retry)
       ? Boolean(params.retry[0])
       : Boolean(params.retry);
+    const reasoning = Array.isArray(params.reasoning)
+      ? Boolean(params.reasoning[0])
+      : Boolean(params.reasoning);
+
     delete params.messageId;
     delete params.retry;
-    return { messageId, retry, params };
+    delete params.reasoning;
+
+    return { messageId, retry, reasoning, params };
   }
 
   private getSignal(req: Request) {
@@ -226,7 +232,7 @@ export class CopilotController implements BeforeApplicationShutdown {
     const info: any = { sessionId, params };
 
     try {
-      const { messageId, retry } = this.prepareParams(params);
+      const { messageId, retry, reasoning } = this.prepareParams(params);
 
       const provider = await this.chooseTextProvider(
         user.id,
@@ -257,6 +263,7 @@ export class CopilotController implements BeforeApplicationShutdown {
         ...session.config.promptConfig,
         signal: this.getSignal(req),
         user: user.id,
+        reasoning,
       });
 
       session.push({
@@ -289,7 +296,7 @@ export class CopilotController implements BeforeApplicationShutdown {
     const info: any = { sessionId, params, throwInStream: false };
 
     try {
-      const { messageId, retry } = this.prepareParams(params);
+      const { messageId, retry, reasoning } = this.prepareParams(params);
 
       const provider = await this.chooseTextProvider(
         user.id,
@@ -322,6 +329,7 @@ export class CopilotController implements BeforeApplicationShutdown {
           ...session.config.promptConfig,
           signal: this.getSignal(req),
           user: user.id,
+          reasoning,
         })
       ).pipe(
         connect(shared$ =>

packages/backend/server/src/plugins/copilot/prompt/prompts.ts

@@ -1044,21 +1044,27 @@ const chat: Prompt[] = [
         content: `You are AFFiNE AI, a professional and humorous copilot within AFFiNE. You are powered by latest GPT model from OpenAI and AFFiNE. AFFiNE is an open source general purposed productivity tool that contains unified building blocks that users can use on any interfaces, including block-based docs editor, infinite canvas based edgeless graphic mode, or multi-dimensional table with multiple transformable views. Your mission is always to try your very best to assist users to use AFFiNE to write docs, draw diagrams or plan things with these abilities. You always think step-by-step and describe your plan for what to build, using well-structured and clear markdown, written out in great detail. Unless otherwise specified, where list, JSON, or code blocks are required for giving the output. Minimize any other prose so that your responses can be directly used and inserted into the docs. You are able to access to API of AFFiNE to finish your job. You always respect the users' privacy and would not leak their info to anyone else. AFFiNE is made by Toeverything .Pte .Ltd, a company registered in Singapore with a diverse and international team. The company also open sourced blocksuite and octobase for building tools similar to Affine. The name AFFiNE comes from the idea of AFFiNE transform, as blocks in affine can all transform in page, edgeless or database mode. AFFiNE team is now having 25 members, an open source company driven by engineers. Today is: {{affine::date}}, User's preferred language is {{affine::language}}.
 
 # Response Guide
-Analyze the given file or document content fragments and determine their relevance to the user's query.
-Use the structure of the fragments to assess their relevance and provide the necessary response with cite sources using the citation rules below.
+Use the webSearch tool to gather information from the web. There are two modes for web searching:
+- MUST: Means you always need to use the webSearch tool to gather information from the web, no matter what the user's query is.
+- CAN: Indicates that web searching is optional - you may use the webSearch tool at your discretion when you determine it would provide valuable information for answering the user's query.
+Currently, you are in the {{searchMode}} web searching mode.
 
-## Content fragments format:
+I will provide you with some content fragments. There are two types of content fragments:
 - Document fragments, identified by a \`document_id\` and containing \`document_content\`.
 - File fragments, identified by a \`blob_id\` and containing \`file_content\`.
 
+You need to analyze web search results and content fragments, determine their relevance to the user's query, and combine them to answer the user's query.
+Please cite all source links in your final answer according to the citations rules.
+
 ## Citations Rules
-When referencing information from the provided documents or files in your response:
+When referencing information from the provided documents, files or web search results in your response:
 1. Use markdown footnote format for citations
 2. Add citations immediately after the relevant sentence or paragraph
 3. Required format: [^reference_index] where reference_index is an increasing positive integer
 4. You MUST include citations at the end of your response in this exact format:
   - For documents: [^reference_index]:{"type":"doc","docId":"document_id"}
   - For files: [^reference_index]:{"type":"attachment","blobId":"blob_id","fileName":"file_name","fileType":"file_type"}
+  - For web search results: [^reference_index]:{"type":"url","url":"url_path"}
 5. Ensure citations adhere strictly to the required format. Do not add extra spaces in citations like [^ reference_index] or [ ^reference_index].
 
 ### Citations Structure
@@ -1068,16 +1074,17 @@ Your response MUST follow this structure:
 3. Citations section with all referenced sources in the required format
 
 Example Output with Citations:
-This is my response with a citation[^1]. Here is more content with another citation[^2].
+This is my response with a document citation[^1]. Here is more content with another file citation[^2]. And here is a web search result citation[^3].
 
 [^1]:{"type":"doc","docId":"abc123"}
 [^2]:{"type":"attachment","blobId":"xyz789","fileName":"example.txt","fileType":"text"}
+[^3]:{"type":"url","url":"https://affine.pro/"}
 `,
       },
       {
         role: 'user',
         content: `
-The following content is a relevant content segment:
+The following are some content fragments I provide for you:
 
 {{#docs}}
 ==========

packages/backend/server/src/plugins/copilot/providers/anthropic.ts

@@ -1,5 +1,6 @@
 import {
   AnthropicProvider as AnthropicSDKProvider,
+  AnthropicProviderOptions,
   createAnthropic,
 } from '@ai-sdk/anthropic';
 import { AISDKError, generateText, streamText } from 'ai';
@@ -10,6 +11,7 @@ import {
   metrics,
   UserFriendlyError,
 } from '../../../base';
+import { createExaTool } from '../tools';
 import { CopilotProvider } from './provider';
 import {
   ChatMessageRole,
@@ -34,6 +36,10 @@ export class AnthropicProvider
   override readonly capabilities = [CopilotCapability.TextToText];
   override readonly models = ['claude-3-7-sonnet-20250219'];
 
+  private readonly MAX_STEPS = 20;
+
+  private toolResults: string[] = [];
+
   #instance!: AnthropicSDKProvider;
 
   override configured(): boolean {
@@ -120,15 +126,24 @@ export class AnthropicProvider
       const [system, msgs] = await chatToGPTMessage(messages);
 
       const modelInstance = this.#instance(model);
-      const { text } = await generateText({
+      const { text, reasoning } = await generateText({
         model: modelInstance,
         system,
         messages: msgs,
         abortSignal: options.signal,
+        providerOptions: {
+          anthropic: this.getAnthropicOptions(options),
+        },
+        tools: {
+          webSearch: createExaTool(this.AFFiNEConfig),
+        },
+        maxSteps: this.MAX_STEPS,
+        experimental_continueSteps: true,
       });
 
       if (!text) throw new Error('Failed to generate text');
-      return text.trim();
+
+      return reasoning ? `${reasoning}\n${text}` : text;
     } catch (e: any) {
       metrics.ai.counter('chat_text_errors').add(1, { model });
       throw this.handleError(e);
@@ -145,21 +160,52 @@ export class AnthropicProvider
     try {
       metrics.ai.counter('chat_text_stream_calls').add(1, { model });
       const [system, msgs] = await chatToGPTMessage(messages);
-
-      const { textStream } = streamText({
+      const { fullStream } = streamText({
         model: this.#instance(model),
         system,
         messages: msgs,
         abortSignal: options.signal,
+        providerOptions: {
+          anthropic: this.getAnthropicOptions(options),
+        },
+        tools: {
+          webSearch: createExaTool(this.AFFiNEConfig),
+        },
+        maxSteps: this.MAX_STEPS,
+        experimental_continueSteps: true,
       });
 
-      for await (const message of textStream) {
-        if (message) {
-          yield message;
-          if (options.signal?.aborted) {
-            await textStream.cancel();
+      for await (const message of fullStream) {
+        switch (message.type) {
+          case 'reasoning': {
+            yield message.textDelta;
             break;
           }
+          case 'tool-result': {
+            if (message.toolName === 'webSearch') {
+              this.toolResults.push(this.getWebSearchLinks(message.result));
+            }
+            break;
+          }
+          case 'step-finish': {
+            if (message.finishReason === 'tool-calls') {
+              yield this.toolResults.join('\n');
+              this.toolResults = [];
+            }
+            break;
+          }
+          case 'text-delta': {
+            yield message.textDelta;
+            break;
+          }
+          case 'error': {
+            const error = message.error as { type: string; message: string };
+            throw new Error(error.message);
+          }
+        }
+        if (options.signal?.aborted) {
+          await fullStream.cancel();
+          break;
         }
       }
     } catch (e: any) {
@@ -167,4 +213,30 @@ export class AnthropicProvider
       throw this.handleError(e);
     }
   }
+
+  private getAnthropicOptions(
+    options: CopilotChatOptions
+  ): AnthropicProviderOptions {
+    if (options?.reasoning) {
+      return {
+        thinking: {
+          type: 'enabled',
+          budgetTokens: 12000,
+        },
+      };
+    }
+    return {};
+  }
+
+  private getWebSearchLinks(
+    list: {
+      title: string | null;
+      url: string;
+    }[]
+  ): string {
+    const links = list.reduce((acc, result) => {
+      return acc + `\n[${result.title ?? result.url}](${result.url})\n`;
+    }, '\n');
+    return links + '\n';
+  }
 }

packages/backend/server/src/plugins/copilot/providers/types.ts

@@ -78,7 +78,11 @@ const CopilotProviderOptionsSchema = z.object({
 
 const CopilotChatOptionsSchema = CopilotProviderOptionsSchema.merge(
   PromptConfigStrictSchema
-).optional();
+)
+  .extend({
+    reasoning: z.boolean().optional(),
+  })
+  .optional();
 
 export type CopilotChatOptions = z.infer<typeof CopilotChatOptionsSchema>;
 

packages/backend/server/src/plugins/copilot/tools/index.ts

@@ -0,0 +1 @@
+export * from './web-search';

packages/backend/server/src/plugins/copilot/tools/web-search.ts

@@ -0,0 +1,35 @@
+import { tool } from 'ai';
+import Exa from 'exa-js';
+import { z } from 'zod';
+
+import { Config } from '../../../base';
+
+export const createExaTool = (config: Config) => {
+  return tool({
+    description: 'Search the web for information',
+    parameters: z.object({
+      query: z.string().describe('The query to search the web for.'),
+      mode: z
+        .enum(['MUST', 'CAN'])
+        .optional()
+        .describe('The mode to search the web for.'),
+    }),
+    execute: async ({ query, mode }) => {
+      const { key } = config.copilot.exa;
+      const exa = new Exa(key);
+      const result = await exa.searchAndContents(query, {
+        numResults: 10,
+        summary: true,
+        livecrawl: mode === 'MUST' ? 'always' : undefined,
+      });
+      return result.results.map(data => ({
+        title: data.title,
+        url: data.url,
+        summary: data.summary,
+        favicon: data.favicon,
+        publishedDate: data.publishedDate,
+        author: data.author,
+      }));
+    },
+  });
+};