diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index ec9dd7ee74..c534723c00 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -29,7 +29,7 @@ "@affine/server-native": "workspace:*", "@ai-sdk/anthropic": "^1.2.10", "@ai-sdk/google": "^1.2.10", - "@ai-sdk/openai": "^1.3.18", + "@ai-sdk/openai": "^1.3.21", "@ai-sdk/perplexity": "^1.1.6", "@apollo/server": "^4.11.3", "@aws-sdk/client-s3": "^3.779.0", diff --git a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts index deb0779e08..6c29859500 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts @@ -1037,24 +1037,26 @@ Finally, please only send us the content of your continuation in Markdown Format const chat: Prompt[] = [ { name: 'Chat With AFFiNE AI', - model: 'gpt-4.1', + model: 'o4-mini', messages: [ { role: 'system', content: `You are AFFiNE AI, a professional and humorous copilot within AFFiNE. You are powered by latest GPT model from OpenAI and AFFiNE. AFFiNE is an open source general purposed productivity tool that contains unified building blocks that users can use on any interfaces, including block-based docs editor, infinite canvas based edgeless graphic mode, or multi-dimensional table with multiple transformable views. Your mission is always to try your very best to assist users to use AFFiNE to write docs, draw diagrams or plan things with these abilities. You always think step-by-step and describe your plan for what to build, using well-structured and clear markdown, written out in great detail. Unless otherwise specified, where list, JSON, or code blocks are required for giving the output. Minimize any other prose so that your responses can be directly used and inserted into the docs. You are able to access to API of AFFiNE to finish your job. You always respect the users' privacy and would not leak their info to anyone else. AFFiNE is made by Toeverything .Pte .Ltd, a company registered in Singapore with a diverse and international team. The company also open sourced blocksuite and octobase for building tools similar to Affine. The name AFFiNE comes from the idea of AFFiNE transform, as blocks in affine can all transform in page, edgeless or database mode. AFFiNE team is now having 25 members, an open source company driven by engineers. Today is: {{affine::date}}, User's preferred language is {{affine::language}}. # Response Guide -Use the web search tool to gather information from the web. There are two modes for web searching: +Use the web search tool to gather information from the web if you have been equipped with it. There are two modes for web searching: - MUST: Means you always need to use the web search tool to gather information from the web, no matter what the user's query is. -- CAN: Indicates that web searching is optional - you may use the web search tool at your discretion when you determine it would provide valuable information for answering the user's query. +- AUTO: Indicates that web searching is optional - you may use the web search tool at your discretion when you determine it would provide valuable information for answering the user's query. If your own knowledge can directly answer the user's questions, there is no need to use web search tool. Currently, you are in the {{searchMode}} web searching mode. I will provide you with some content fragments. There are two types of content fragments: - Document fragments, identified by a \`document_id\` and containing \`document_content\`. - File fragments, identified by a \`blob_id\` and containing \`file_content\`. -You need to analyze web search results and content fragments, determine their relevance to the user's query, and combine them to answer the user's query. -Please cite all source links in your final answer according to the citations rules. +You need to analyze web search results and content fragments, determine their relevance to the user's query, and combine them with your own knowledge to answer the user's query. +Please cite all source links in your final answer according to the citations rules. Don't make up citations that don't exist. + +If you’re unsure, tell them you’re unsure so they don’t fall into hallucinations. ## Citations Rules When referencing information from the provided documents, files or web search results in your response: diff --git a/packages/backend/server/src/plugins/copilot/providers/anthropic.ts b/packages/backend/server/src/plugins/copilot/providers/anthropic.ts index 83559ab652..605ad560b3 100644 --- a/packages/backend/server/src/plugins/copilot/providers/anthropic.ts +++ b/packages/backend/server/src/plugins/copilot/providers/anthropic.ts @@ -177,6 +177,12 @@ export class AnthropicProvider yield message.textDelta; break; } + case 'tool-call': { + if (message.toolName === 'web_search') { + yield '\n' + `Searching the web "${message.args.query}"` + '\n'; + } + break; + } case 'tool-result': { if (message.toolName === 'web_search') { this.toolResults.push(this.getWebSearchLinks(message.result)); @@ -219,18 +225,15 @@ export class AnthropicProvider return undefined; } - private getAnthropicOptions( - options: CopilotChatOptions - ): AnthropicProviderOptions { + private getAnthropicOptions(options: CopilotChatOptions) { + const result: AnthropicProviderOptions = {}; if (options?.reasoning) { - return { - thinking: { - type: 'enabled', - budgetTokens: 12000, - }, + result.thinking = { + type: 'enabled', + budgetTokens: 12000, }; } - return {}; + return result; } private getWebSearchLinks( diff --git a/packages/backend/server/src/plugins/copilot/providers/openai.ts b/packages/backend/server/src/plugins/copilot/providers/openai.ts index 905a5e58f1..be70de4754 100644 --- a/packages/backend/server/src/plugins/copilot/providers/openai.ts +++ b/packages/backend/server/src/plugins/copilot/providers/openai.ts @@ -2,6 +2,7 @@ import { createOpenAI, openai, type OpenAIProvider as VercelOpenAIProvider, + OpenAIResponsesProviderOptions, } from '@ai-sdk/openai'; import { AISDKError, @@ -10,7 +11,6 @@ import { generateObject, generateText, streamText, - ToolSet, } from 'ai'; import { @@ -19,6 +19,7 @@ import { metrics, UserFriendlyError, } from '../../../base'; +import { createExaTool } from '../tools'; import { CopilotProvider } from './provider'; import { ChatMessageRole, @@ -42,6 +43,11 @@ export type OpenAIConfig = { baseUrl?: string; }; +type OpenAITools = { + web_search_preview: ReturnType; + web_search: ReturnType; +}; + export class OpenAIProvider extends CopilotProvider implements @@ -68,7 +74,7 @@ export class OpenAIProvider 'gpt-4.1-2025-04-14', 'gpt-4.1-mini', 'o1', - 'o3-mini', + 'o4-mini', // embeddings 'text-embedding-3-large', 'text-embedding-3-small', @@ -81,6 +87,8 @@ export class OpenAIProvider 'gpt-image-1', ]; + private readonly MAX_STEPS = 20; + #instance!: VercelOpenAIProvider; override configured(): boolean { @@ -179,21 +187,28 @@ export class OpenAIProvider } } - private getTools(options: CopilotChatOptions): ToolSet | undefined { + private getTools( + options: CopilotChatOptions, + model: string + ): Partial { + const tools: Partial = {}; if (options?.tools?.length) { - const tools: ToolSet = {}; for (const tool of options.tools) { switch (tool) { case 'webSearch': { - tools.web_search_preview = openai.tools.webSearchPreview(); + // o series reasoning models + if (model.startsWith('o')) { + tools.web_search = createExaTool(this.AFFiNEConfig); + } else { + tools.web_search_preview = openai.tools.webSearchPreview(); + } break; } } } return tools; } - - return undefined; + return tools; } // ====== text to text ====== @@ -231,10 +246,10 @@ export class OpenAIProvider : await generateText({ ...commonParams, providerOptions: { - openai: options.user ? { user: options.user } : {}, + openai: this.getOpenAIOptions(options), }, - toolChoice: options.webSearch ? 'required' : 'auto', - tools: this.getTools(options), + tools: this.getTools(options, model), + maxSteps: this.MAX_STEPS, }); return text.trim(); @@ -258,14 +273,16 @@ export class OpenAIProvider const modelInstance = this.#instance.responses(model); + const tools = this.getTools(options, model); const { fullStream } = streamText({ model: modelInstance, system, messages: msgs, - tools: this.getTools(options), providerOptions: { - openai: options.user ? { user: options.user } : {}, + openai: this.getOpenAIOptions(options), }, + tools: tools as OpenAITools, + maxSteps: this.MAX_STEPS, frequencyPenalty: options.frequencyPenalty || 0, presencePenalty: options.presencePenalty || 0, temperature: options.temperature || 0, @@ -274,25 +291,53 @@ export class OpenAIProvider }); const parser = new CitationParser(); + let lastType; for await (const chunk of fullStream) { if (chunk) { switch (chunk.type) { case 'text-delta': { + if (lastType !== chunk.type) { + yield `\n`; + } const result = parser.parse(chunk.textDelta); yield result; break; } + case 'reasoning': { + if (lastType !== chunk.type) { + yield `\n`; + } + yield chunk.textDelta; + break; + } + case 'tool-call': { + if (chunk.toolName === 'web_search') { + yield '\n' + `Searching the web "${chunk.args.query}"` + '\n'; + } + break; + } + case 'tool-result': { + if (chunk.toolName === 'web_search') { + yield '\n' + this.getWebSearchLinks(chunk.result) + '\n'; + } + break; + } case 'step-finish': { const result = parser.end(); yield result; break; } + case 'error': { + const error = chunk.error as { type: string; message: string }; + throw new Error(error.message); + } } if (options.signal?.aborted) { await fullStream.cancel(); break; } + lastType = chunk.type; } } } catch (e: any) { @@ -380,4 +425,28 @@ export class OpenAIProvider throw e; } } + + private getOpenAIOptions(options: CopilotChatOptions) { + const result: OpenAIResponsesProviderOptions = {}; + if (options?.reasoning) { + result.reasoningEffort = 'medium'; + result.reasoningSummary = 'auto'; + } + if (options?.user) { + result.user = options.user; + } + return result; + } + + private getWebSearchLinks( + list: { + title: string | null; + url: string; + }[] + ): string { + const links = list.reduce((acc, result) => { + return acc + `\n[${result.title ?? result.url}](${result.url})\n`; + }, '\n'); + return links + '\n'; + } } diff --git a/packages/backend/server/src/plugins/copilot/tools/web-search.ts b/packages/backend/server/src/plugins/copilot/tools/web-search.ts index ebd45ba738..80aa8e14b7 100644 --- a/packages/backend/server/src/plugins/copilot/tools/web-search.ts +++ b/packages/backend/server/src/plugins/copilot/tools/web-search.ts @@ -10,8 +10,7 @@ export const createExaTool = (config: Config) => { parameters: z.object({ query: z.string().describe('The query to search the web for.'), mode: z - .enum(['MUST', 'CAN']) - .optional() + .enum(['MUST', 'AUTO']) .describe('The mode to search the web for.'), }), execute: async ({ query, mode }) => { diff --git a/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx b/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx index aef8211695..afe788ff12 100644 --- a/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx +++ b/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx @@ -96,7 +96,7 @@ export function setupAIProvider( params: { docs: contexts?.docs, files: contexts?.files, - searchMode: webSearch ? 'MUST' : 'CAN', + searchMode: webSearch ? 'MUST' : 'AUTO', }, }); }); diff --git a/yarn.lock b/yarn.lock index 7b633d8e74..ba0e0a68e3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -893,7 +893,7 @@ __metadata: "@affine/server-native": "workspace:*" "@ai-sdk/anthropic": "npm:^1.2.10" "@ai-sdk/google": "npm:^1.2.10" - "@ai-sdk/openai": "npm:^1.3.18" + "@ai-sdk/openai": "npm:^1.3.21" "@ai-sdk/perplexity": "npm:^1.1.6" "@apollo/server": "npm:^4.11.3" "@aws-sdk/client-s3": "npm:^3.779.0" @@ -1080,7 +1080,7 @@ __metadata: languageName: node linkType: hard -"@ai-sdk/openai@npm:^1.3.18": +"@ai-sdk/openai@npm:^1.3.21": version: 1.3.21 resolution: "@ai-sdk/openai@npm:1.3.21" dependencies: