From f8fada0b108ea587974952012ff29353c418c928 Mon Sep 17 00:00:00 2001 From: akumatus Date: Thu, 8 May 2025 03:42:21 +0000 Subject: [PATCH] feat(core): mark reasoning summary as markdown callout (#12176) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close [AI-75](https://linear.app/affine-design/issue/AI-75) ![截屏2025-05-07 17.40.32.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/e5e7cf4c-0f80-41de-bb22-79a25fdfef48.png) ## Summary by CodeRabbit - **New Features** - Improved formatting for AI-generated reasoning and web search results, displaying them as callout blocks for enhanced readability. - Expanded support for rendering callout blocks in the text renderer. - **Style** - Adjusted layout to ensure callout blocks retain appropriate spacing and appearance. - **Refactor** - Simplified and unified the integration of web search tools for AI providers. --- .../plugins/copilot/providers/anthropic.ts | 89 ++++++++++++------- .../src/plugins/copilot/providers/openai.ts | 55 ++++++++---- .../src/plugins/copilot/tools/web-search.ts | 2 +- .../blocksuite/ai/components/text-renderer.ts | 5 +- 4 files changed, 98 insertions(+), 53 deletions(-) diff --git a/packages/backend/server/src/plugins/copilot/providers/anthropic.ts b/packages/backend/server/src/plugins/copilot/providers/anthropic.ts index 605ad560b3..e7cbd748d5 100644 --- a/packages/backend/server/src/plugins/copilot/providers/anthropic.ts +++ b/packages/backend/server/src/plugins/copilot/providers/anthropic.ts @@ -11,7 +11,7 @@ import { metrics, UserFriendlyError, } from '../../../base'; -import { createExaTool } from '../tools'; +import { createExaSearchTool } from '../tools'; import { CopilotProvider } from './provider'; import { ChatMessageRole, @@ -38,7 +38,7 @@ export class AnthropicProvider private readonly MAX_STEPS = 20; - private toolResults: string[] = []; + private readonly CALLOUT_PREFIX = '\n> [!]\n> '; #instance!: AnthropicSDKProvider; @@ -134,7 +134,7 @@ export class AnthropicProvider providerOptions: { anthropic: this.getAnthropicOptions(options), }, - tools: this.getTools(options), + tools: this.getTools(), maxSteps: this.MAX_STEPS, experimental_continueSteps: true, }); @@ -166,42 +166,63 @@ export class AnthropicProvider providerOptions: { anthropic: this.getAnthropicOptions(options), }, - tools: this.getTools(options), + tools: this.getTools(), maxSteps: this.MAX_STEPS, experimental_continueSteps: true, }); - for await (const message of fullStream) { - switch (message.type) { + let lastType; + // reasoning, tool-call, tool-result need to mark as callout + let prefix: string | null = this.CALLOUT_PREFIX; + for await (const chunk of fullStream) { + switch (chunk.type) { + case 'text-delta': { + if (!prefix) { + prefix = this.CALLOUT_PREFIX; + } + let result = chunk.textDelta; + if (lastType !== chunk.type) { + result = '\n\n' + result; + } + yield result; + break; + } case 'reasoning': { - yield message.textDelta; + if (prefix) { + yield prefix; + prefix = null; + } + let result = chunk.textDelta; + if (lastType !== chunk.type) { + result = '\n\n' + result; + } + yield this.markAsCallout(result); break; } case 'tool-call': { - if (message.toolName === 'web_search') { - yield '\n' + `Searching the web "${message.args.query}"` + '\n'; + if (prefix) { + yield prefix; + prefix = null; + } + if (chunk.toolName === 'web_search') { + yield this.markAsCallout( + `\nSearching the web "${chunk.args.query}"\n` + ); } break; } case 'tool-result': { - if (message.toolName === 'web_search') { - this.toolResults.push(this.getWebSearchLinks(message.result)); + if (chunk.toolName === 'web_search') { + if (prefix) { + yield prefix; + prefix = null; + } + yield this.markAsCallout(this.getWebSearchLinks(chunk.result)); } break; } - case 'step-finish': { - if (message.finishReason === 'tool-calls') { - yield this.toolResults.join('\n'); - this.toolResults = []; - } - break; - } - case 'text-delta': { - yield message.textDelta; - break; - } case 'error': { - const error = message.error as { type: string; message: string }; + const error = chunk.error as { type: string; message: string }; throw new Error(error.message); } } @@ -209,6 +230,7 @@ export class AnthropicProvider await fullStream.cancel(); break; } + lastType = chunk.type; } } catch (e: any) { metrics.ai.counter('chat_text_stream_errors').add(1, { model }); @@ -216,13 +238,10 @@ export class AnthropicProvider } } - private getTools(options: CopilotChatOptions) { - if (options?.webSearch) { - return { - web_search: createExaTool(this.AFFiNEConfig), - }; - } - return undefined; + private getTools() { + return { + web_search: createExaSearchTool(this.AFFiNEConfig), + }; } private getAnthropicOptions(options: CopilotChatOptions) { @@ -243,8 +262,12 @@ export class AnthropicProvider }[] ): string { const links = list.reduce((acc, result) => { - return acc + `\n[${result.title ?? result.url}](${result.url})\n`; - }, '\n'); - return links + '\n'; + return acc + `\n[${result.title ?? result.url}](${result.url})\n\n`; + }, ''); + return links; + } + + private markAsCallout(text: string) { + return text.replaceAll('\n', '\n> '); } } diff --git a/packages/backend/server/src/plugins/copilot/providers/openai.ts b/packages/backend/server/src/plugins/copilot/providers/openai.ts index be70de4754..f9edfc5f54 100644 --- a/packages/backend/server/src/plugins/copilot/providers/openai.ts +++ b/packages/backend/server/src/plugins/copilot/providers/openai.ts @@ -19,7 +19,7 @@ import { metrics, UserFriendlyError, } from '../../../base'; -import { createExaTool } from '../tools'; +import { createExaSearchTool } from '../tools'; import { CopilotProvider } from './provider'; import { ChatMessageRole, @@ -45,7 +45,7 @@ export type OpenAIConfig = { type OpenAITools = { web_search_preview: ReturnType; - web_search: ReturnType; + web_search_exa: ReturnType; }; export class OpenAIProvider @@ -89,6 +89,8 @@ export class OpenAIProvider private readonly MAX_STEPS = 20; + private readonly CALLOUT_PREFIX = '\n> [!]\n> '; + #instance!: VercelOpenAIProvider; override configured(): boolean { @@ -198,7 +200,7 @@ export class OpenAIProvider case 'webSearch': { // o series reasoning models if (model.startsWith('o')) { - tools.web_search = createExaTool(this.AFFiNEConfig); + tools.web_search_exa = createExaSearchTool(this.AFFiNEConfig); } else { tools.web_search_preview = openai.tools.webSearchPreview(); } @@ -292,37 +294,52 @@ export class OpenAIProvider const parser = new CitationParser(); let lastType; + // reasoning, tool-call, tool-result need to mark as callout + let prefix: string | null = this.CALLOUT_PREFIX; for await (const chunk of fullStream) { if (chunk) { switch (chunk.type) { case 'text-delta': { + let result = parser.parse(chunk.textDelta); if (lastType !== chunk.type) { - yield `\n`; + result = '\n\n' + result; } - const result = parser.parse(chunk.textDelta); yield result; break; } case 'reasoning': { - if (lastType !== chunk.type) { - yield `\n`; + if (prefix) { + yield prefix; + prefix = null; } - yield chunk.textDelta; + let result = chunk.textDelta; + if (lastType !== chunk.type) { + result = '\n\n' + result; + } + yield this.markAsCallout(result); break; } case 'tool-call': { - if (chunk.toolName === 'web_search') { - yield '\n' + `Searching the web "${chunk.args.query}"` + '\n'; + if (prefix) { + yield prefix; + prefix = null; + } + if (chunk.toolName === 'web_search_exa') { + yield this.markAsCallout( + `\nSearching the web "${chunk.args.query}"\n` + ); } break; } case 'tool-result': { - if (chunk.toolName === 'web_search') { - yield '\n' + this.getWebSearchLinks(chunk.result) + '\n'; + if (chunk.toolName === 'web_search_exa') { + yield this.markAsCallout( + `\n${this.getWebSearchLinks(chunk.result)}\n` + ); } break; } - case 'step-finish': { + case 'finish': { const result = parser.end(); yield result; break; @@ -430,7 +447,7 @@ export class OpenAIProvider const result: OpenAIResponsesProviderOptions = {}; if (options?.reasoning) { result.reasoningEffort = 'medium'; - result.reasoningSummary = 'auto'; + result.reasoningSummary = 'detailed'; } if (options?.user) { result.user = options.user; @@ -445,8 +462,12 @@ export class OpenAIProvider }[] ): string { const links = list.reduce((acc, result) => { - return acc + `\n[${result.title ?? result.url}](${result.url})\n`; - }, '\n'); - return links + '\n'; + return acc + `\n[${result.title ?? result.url}](${result.url})\n\n`; + }, ''); + return links; + } + + private markAsCallout(text: string) { + return text.replaceAll('\n', '\n> '); } } diff --git a/packages/backend/server/src/plugins/copilot/tools/web-search.ts b/packages/backend/server/src/plugins/copilot/tools/web-search.ts index 80aa8e14b7..5bb32034e9 100644 --- a/packages/backend/server/src/plugins/copilot/tools/web-search.ts +++ b/packages/backend/server/src/plugins/copilot/tools/web-search.ts @@ -4,7 +4,7 @@ import { z } from 'zod'; import { Config } from '../../../base'; -export const createExaTool = (config: Config) => { +export const createExaSearchTool = (config: Config) => { return tool({ description: 'Search the web for information', parameters: z.object({ diff --git a/packages/frontend/core/src/blocksuite/ai/components/text-renderer.ts b/packages/frontend/core/src/blocksuite/ai/components/text-renderer.ts index dbcdf580b9..60ef94f085 100644 --- a/packages/frontend/core/src/blocksuite/ai/components/text-renderer.ts +++ b/packages/frontend/core/src/blocksuite/ai/components/text-renderer.ts @@ -124,8 +124,8 @@ export class TextRenderer extends WithDisposable(ShadowlessElement) { .ai-answer-text-editor { .affine-note-block-container { > .affine-block-children-container { - > :first-child, - > :first-child * { + > :first-child:not(affine-callout), + > :first-child:not(affine-callout) * { margin-top: 0 !important; } > :last-child, @@ -225,6 +225,7 @@ export class TextRenderer extends WithDisposable(ShadowlessElement) { 'affine:table', 'affine:surface', 'affine:paragraph', + 'affine:callout', 'affine:code', 'affine:list', 'affine:divider',