feat(server): migrate copilot to native (#14620)

#### PR Dependency Tree


* **PR #14620** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Native LLM workflows: structured outputs, embeddings, and reranking
plus richer multimodal attachments (images, audio, files) and improved
remote-attachment inlining.

* **Refactor**
* Tooling API unified behind a local tool-definition helper;
provider/adapters reorganized to route through native dispatch paths.

* **Chores**
* Dependency updates, removed legacy Google SDK integrations, and
increased front memory allocation.

* **Tests**
* Expanded end-to-end and streaming tests exercising native provider
flows, attachments, and rerank/structured scenarios.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/__tests__/copilot/copilot-provider.spec.ts

@@ -225,6 +225,20 @@ const checkStreamObjects = (result: string) => {
   }
 };
 
+const parseStreamObjects = (result: string): StreamObject[] => {
+  const streamObjects = JSON.parse(result);
+  return z.array(StreamObjectSchema).parse(streamObjects);
+};
+
+const getStreamObjectText = (result: string) =>
+  parseStreamObjects(result)
+    .filter(
+      (chunk): chunk is Extract<StreamObject, { type: 'text-delta' }> =>
+        chunk.type === 'text-delta'
+    )
+    .map(chunk => chunk.textDelta)
+    .join('');
+
 const retry = async (
   action: string,
   t: ExecutionContext<Tester>,
@@ -444,6 +458,49 @@ The term **“CRDT”** was first introduced by Marc Shapiro, Nuno Preguiça, Ca
     },
     type: 'object' as const,
   },
+  {
+    name: 'Gemini native text',
+    promptName: ['Chat With AFFiNE AI'],
+    messages: [
+      {
+        role: 'user' as const,
+        content:
+          'In one short sentence, explain what AFFiNE AI is and mention AFFiNE by name.',
+      },
+    ],
+    config: { model: 'gemini-2.5-flash' },
+    verifier: (t: ExecutionContext<Tester>, result: string) => {
+      assertNotWrappedInCodeBlock(t, result);
+      t.assert(
+        result.toLowerCase().includes('affine'),
+        'should mention AFFiNE'
+      );
+    },
+    prefer: CopilotProviderType.Gemini,
+    type: 'text' as const,
+  },
+  {
+    name: 'Gemini native stream objects',
+    promptName: ['Chat With AFFiNE AI'],
+    messages: [
+      {
+        role: 'user' as const,
+        content:
+          'Respond with one short sentence about AFFiNE AI and mention AFFiNE by name.',
+      },
+    ],
+    config: { model: 'gemini-2.5-flash' },
+    verifier: (t: ExecutionContext<Tester>, result: string) => {
+      t.truthy(checkStreamObjects(result), 'should be valid stream objects');
+      const assembledText = getStreamObjectText(result);
+      t.assert(
+        assembledText.toLowerCase().includes('affine'),
+        'should mention AFFiNE'
+      );
+    },
+    prefer: CopilotProviderType.Gemini,
+    type: 'object' as const,
+  },
   {
     name: 'Should transcribe short audio',
     promptName: ['Transcript audio'],
@@ -716,14 +773,13 @@ for (const {
         const { factory, prompt: promptService } = t.context;
         const prompt = (await promptService.get(promptName))!;
         t.truthy(prompt, 'should have prompt');
-        const provider = (await factory.getProviderByModel(prompt.model, {
+        const finalConfig = Object.assign({}, prompt.config, config);
+        const modelId = finalConfig.model || prompt.model;
+        const provider = (await factory.getProviderByModel(modelId, {
           prefer,
         }))!;
         t.truthy(provider, 'should have provider');
         await retry(`action: ${promptName}`, t, async t => {
-          const finalConfig = Object.assign({}, prompt.config, config);
-          const modelId = finalConfig.model || prompt.model;
-
           switch (type) {
             case 'text': {
               const result = await provider.text(
@@ -891,7 +947,7 @@ test(
   'should be able to rerank message chunks',
   runIfCopilotConfigured,
   async t => {
-    const { factory, prompt } = t.context;
+    const { factory } = t.context;
 
     await retry('rerank', t, async t => {
       const query = 'Is this content relevant to programming?';
@@ -908,14 +964,18 @@ test(
         'The stock market is experiencing significant fluctuations.',
       ];
 
-      const p = (await prompt.get('Rerank results'))!;
-      t.assert(p, 'should have prompt for rerank');
-      const provider = (await factory.getProviderByModel(p.model))!;
+      const provider = (await factory.getProviderByModel('gpt-5.2'))!;
       t.assert(provider, 'should have provider for rerank');
 
       const scores = await provider.rerank(
-        { modelId: p.model },
-        embeddings.map(e => p.finish({ query, doc: e }))
+        { modelId: 'gpt-5.2' },
+        {
+          query,
+          candidates: embeddings.map((text, index) => ({
+            id: String(index),
+            text,
+          })),
+        }
       );
 
       t.is(scores.length, 10, 'should return scores for all chunks');

packages/backend/server/src/__tests__/copilot/copilot.spec.ts

@@ -33,10 +33,7 @@ import {
   ModelOutputType,
   OpenAIProvider,
 } from '../../plugins/copilot/providers';
-import {
-  CitationParser,
-  TextStreamParser,
-} from '../../plugins/copilot/providers/utils';
+import { TextStreamParser } from '../../plugins/copilot/providers/utils';
 import { ChatSessionService } from '../../plugins/copilot/session';
 import { CopilotStorage } from '../../plugins/copilot/storage';
 import { CopilotTranscriptionService } from '../../plugins/copilot/transcript';
@@ -660,6 +657,55 @@ test('should be able to generate with message id', async t => {
   }
 });
 
+test('should preserve file handle attachments when merging user content into prompt', async t => {
+  const { prompt, session } = t.context;
+
+  await prompt.set(promptName, 'model', [
+    { role: 'user', content: '{{content}}' },
+  ]);
+
+  const sessionId = await session.create({
+    docId: 'test',
+    workspaceId: 'test',
+    userId,
+    promptName,
+    pinned: false,
+  });
+  const s = (await session.get(sessionId))!;
+
+  const message = await session.createMessage({
+    sessionId,
+    content: 'Summarize this file',
+    attachments: [
+      {
+        kind: 'file_handle',
+        fileHandle: 'file_123',
+        mimeType: 'application/pdf',
+      },
+    ],
+  });
+
+  await s.pushByMessageId(message);
+  const finalMessages = s.finish({});
+
+  t.deepEqual(finalMessages, [
+    {
+      role: 'user',
+      content: 'Summarize this file',
+      attachments: [
+        {
+          kind: 'file_handle',
+          fileHandle: 'file_123',
+          mimeType: 'application/pdf',
+        },
+      ],
+      params: {
+        content: 'Summarize this file',
+      },
+    },
+  ]);
+});
+
 test('should save message correctly', async t => {
   const { prompt, session } = t.context;
 
@@ -1225,149 +1271,6 @@ test('should be able to run image executor', async t => {
   Sinon.restore();
 });
 
-test('CitationParser should replace citation placeholders with URLs', t => {
-  const content =
-    'This is [a] test sentence with [citations [1]] and [[2]] and [3].';
-  const citations = ['https://example1.com', 'https://example2.com'];
-
-  const parser = new CitationParser();
-  for (const citation of citations) {
-    parser.push(citation);
-  }
-
-  const result = parser.parse(content) + parser.end();
-
-  const expected = [
-    'This is [a] test sentence with [citations [^1]] and [^2] and [3].',
-    `[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
-    `[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
-  ].join('\n');
-
-  t.is(result, expected);
-});
-
-test('CitationParser should replace chunks of citation placeholders with URLs', t => {
-  const contents = [
-    '[[]]',
-    'This is [',
-    'a] test sentence ',
-    'with citations [1',
-    '] and [',
-    '[2]] and [[',
-    '3]] and [[4',
-    ']] and [[5]',
-    '] and [[6]]',
-    ' and [7',
-  ];
-  const citations = [
-    'https://example1.com',
-    'https://example2.com',
-    'https://example3.com',
-    'https://example4.com',
-    'https://example5.com',
-    'https://example6.com',
-    'https://example7.com',
-  ];
-
-  const parser = new CitationParser();
-  for (const citation of citations) {
-    parser.push(citation);
-  }
-
-  let result = contents.reduce((acc, current) => {
-    return acc + parser.parse(current);
-  }, '');
-  result += parser.end();
-
-  const expected = [
-    '[[]]This is [a] test sentence with citations [^1] and [^2] and [^3] and [^4] and [^5] and [^6] and [7',
-    `[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
-    `[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
-    `[^3]: {"type":"url","url":"${encodeURIComponent(citations[2])}"}`,
-    `[^4]: {"type":"url","url":"${encodeURIComponent(citations[3])}"}`,
-    `[^5]: {"type":"url","url":"${encodeURIComponent(citations[4])}"}`,
-    `[^6]: {"type":"url","url":"${encodeURIComponent(citations[5])}"}`,
-    `[^7]: {"type":"url","url":"${encodeURIComponent(citations[6])}"}`,
-  ].join('\n');
-  t.is(result, expected);
-});
-
-test('CitationParser should not replace citation already with URLs', t => {
-  const content =
-    'This is [a] test sentence with citations [1](https://example1.com) and [[2]](https://example2.com) and [[3](https://example3.com)].';
-  const citations = [
-    'https://example4.com',
-    'https://example5.com',
-    'https://example6.com',
-  ];
-
-  const parser = new CitationParser();
-  for (const citation of citations) {
-    parser.push(citation);
-  }
-
-  const result = parser.parse(content) + parser.end();
-
-  const expected = [
-    content,
-    `[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
-    `[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
-    `[^3]: {"type":"url","url":"${encodeURIComponent(citations[2])}"}`,
-  ].join('\n');
-  t.is(result, expected);
-});
-
-test('CitationParser should not replace chunks of citation already with URLs', t => {
-  const contents = [
-    'This is [a] test sentence with citations [1',
-    '](https://example1.com) and [[2]',
-    '](https://example2.com) and [[3](https://example3.com)].',
-  ];
-  const citations = [
-    'https://example4.com',
-    'https://example5.com',
-    'https://example6.com',
-  ];
-
-  const parser = new CitationParser();
-  for (const citation of citations) {
-    parser.push(citation);
-  }
-
-  let result = contents.reduce((acc, current) => {
-    return acc + parser.parse(current);
-  }, '');
-  result += parser.end();
-
-  const expected = [
-    contents.join(''),
-    `[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
-    `[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
-    `[^3]: {"type":"url","url":"${encodeURIComponent(citations[2])}"}`,
-  ].join('\n');
-  t.is(result, expected);
-});
-
-test('CitationParser should replace openai style reference chunks', t => {
-  const contents = [
-    'This is [a] test sentence with citations ',
-    '([example1.com](https://example1.com))',
-  ];
-
-  const parser = new CitationParser();
-
-  let result = contents.reduce((acc, current) => {
-    return acc + parser.parse(current);
-  }, '');
-  result += parser.end();
-
-  const expected = [
-    contents[0] + '[^1]',
-    `[^1]: {"type":"url","url":"${encodeURIComponent('https://example1.com')}"}`,
-  ].join('\n');
-  t.is(result, expected);
-});
-
 test('TextStreamParser should format different types of chunks correctly', t => {
   // Define interfaces for fixtures
   interface BaseFixture {

packages/backend/server/src/__tests__/copilot/native-adapter.spec.ts

@@ -1,210 +0,0 @@
-import test from 'ava';
-import { z } from 'zod';
-
-import type { NativeLlmRequest, NativeLlmStreamEvent } from '../../native';
-import {
-  buildNativeRequest,
-  NativeProviderAdapter,
-} from '../../plugins/copilot/providers/native';
-
-const mockDispatch = () =>
-  (async function* (): AsyncIterableIterator<NativeLlmStreamEvent> {
-    yield { type: 'text_delta', text: 'Use [^1] now' };
-    yield { type: 'citation', index: 1, url: 'https://affine.pro' };
-    yield { type: 'done', finish_reason: 'stop' };
-  })();
-
-test('NativeProviderAdapter streamText should append citation footnotes', async t => {
-  const adapter = new NativeProviderAdapter(mockDispatch, {}, 3);
-  const chunks: string[] = [];
-  for await (const chunk of adapter.streamText({
-    model: 'gpt-5-mini',
-    stream: true,
-    messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }],
-  })) {
-    chunks.push(chunk);
-  }
-
-  const text = chunks.join('');
-  t.true(text.includes('Use [^1] now'));
-  t.true(
-    text.includes('[^1]: {"type":"url","url":"https%3A%2F%2Faffine.pro"}')
-  );
-});
-
-test('NativeProviderAdapter streamObject should append citation footnotes', async t => {
-  const adapter = new NativeProviderAdapter(mockDispatch, {}, 3);
-  const chunks = [];
-  for await (const chunk of adapter.streamObject({
-    model: 'gpt-5-mini',
-    stream: true,
-    messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }],
-  })) {
-    chunks.push(chunk);
-  }
-
-  t.deepEqual(
-    chunks.map(chunk => chunk.type),
-    ['text-delta', 'text-delta']
-  );
-  const text = chunks
-    .filter(chunk => chunk.type === 'text-delta')
-    .map(chunk => chunk.textDelta)
-    .join('');
-  t.true(text.includes('Use [^1] now'));
-  t.true(
-    text.includes('[^1]: {"type":"url","url":"https%3A%2F%2Faffine.pro"}')
-  );
-});
-
-test('NativeProviderAdapter streamObject should append fallback attachment footnotes', async t => {
-  const dispatch = () =>
-    (async function* (): AsyncIterableIterator<NativeLlmStreamEvent> {
-      yield {
-        type: 'tool_result',
-        call_id: 'call_1',
-        name: 'blob_read',
-        arguments: { blob_id: 'blob_1' },
-        output: {
-          blobId: 'blob_1',
-          fileName: 'a.txt',
-          fileType: 'text/plain',
-          content: 'A',
-        },
-      };
-      yield {
-        type: 'tool_result',
-        call_id: 'call_2',
-        name: 'blob_read',
-        arguments: { blob_id: 'blob_2' },
-        output: {
-          blobId: 'blob_2',
-          fileName: 'b.txt',
-          fileType: 'text/plain',
-          content: 'B',
-        },
-      };
-      yield { type: 'text_delta', text: 'Answer from files.' };
-      yield { type: 'done', finish_reason: 'stop' };
-    })();
-
-  const adapter = new NativeProviderAdapter(dispatch, {}, 3);
-  const chunks = [];
-  for await (const chunk of adapter.streamObject({
-    model: 'gpt-5-mini',
-    stream: true,
-    messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }],
-  })) {
-    chunks.push(chunk);
-  }
-
-  const text = chunks
-    .filter(chunk => chunk.type === 'text-delta')
-    .map(chunk => chunk.textDelta)
-    .join('');
-  t.true(text.includes('Answer from files.'));
-  t.true(text.includes('[^1][^2]'));
-  t.true(
-    text.includes(
-      '[^1]: {"type":"attachment","blobId":"blob_1","fileName":"a.txt","fileType":"text/plain"}'
-    )
-  );
-  t.true(
-    text.includes(
-      '[^2]: {"type":"attachment","blobId":"blob_2","fileName":"b.txt","fileType":"text/plain"}'
-    )
-  );
-});
-
-test('NativeProviderAdapter streamObject should map tool and text events', async t => {
-  let round = 0;
-  const dispatch = (_request: NativeLlmRequest) =>
-    (async function* (): AsyncIterableIterator<NativeLlmStreamEvent> {
-      round += 1;
-      if (round === 1) {
-        yield {
-          type: 'tool_call',
-          call_id: 'call_1',
-          name: 'doc_read',
-          arguments: { doc_id: 'a1' },
-        };
-        yield { type: 'done', finish_reason: 'tool_calls' };
-        return;
-      }
-      yield { type: 'text_delta', text: 'ok' };
-      yield { type: 'done', finish_reason: 'stop' };
-    })();
-
-  const adapter = new NativeProviderAdapter(
-    dispatch,
-    {
-      doc_read: {
-        inputSchema: z.object({ doc_id: z.string() }),
-        execute: async () => ({ markdown: '# a1' }),
-      },
-    },
-    4
-  );
-
-  const events = [];
-  for await (const event of adapter.streamObject({
-    model: 'gpt-5-mini',
-    stream: true,
-    messages: [{ role: 'user', content: [{ type: 'text', text: 'read' }] }],
-  })) {
-    events.push(event);
-  }
-
-  t.deepEqual(
-    events.map(event => event.type),
-    ['tool-call', 'tool-result', 'text-delta']
-  );
-  t.deepEqual(events[0], {
-    type: 'tool-call',
-    toolCallId: 'call_1',
-    toolName: 'doc_read',
-    args: { doc_id: 'a1' },
-  });
-});
-
-test('buildNativeRequest should include rust middleware from profile', async t => {
-  const { request } = await buildNativeRequest({
-    model: 'gpt-5-mini',
-    messages: [{ role: 'user', content: 'hello' }],
-    tools: {},
-    middleware: {
-      rust: {
-        request: ['normalize_messages', 'clamp_max_tokens'],
-        stream: ['stream_event_normalize', 'citation_indexing'],
-      },
-      node: {
-        text: ['callout'],
-      },
-    },
-  });
-
-  t.deepEqual(request.middleware, {
-    request: ['normalize_messages', 'clamp_max_tokens'],
-    stream: ['stream_event_normalize', 'citation_indexing'],
-  });
-});
-
-test('NativeProviderAdapter streamText should skip citation footnotes when disabled', async t => {
-  const adapter = new NativeProviderAdapter(mockDispatch, {}, 3, {
-    nodeTextMiddleware: ['callout'],
-  });
-  const chunks: string[] = [];
-  for await (const chunk of adapter.streamText({
-    model: 'gpt-5-mini',
-    stream: true,
-    messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }],
-  })) {
-    chunks.push(chunk);
-  }
-
-  const text = chunks.join('');
-  t.true(text.includes('Use [^1] now'));
-  t.false(
-    text.includes('[^1]: {"type":"url","url":"https%3A%2F%2Faffine.pro"}')
-  );
-});

packages/backend/server/src/__tests__/copilot/provider-native.spec.ts

@@ -1,9 +1,13 @@
+import serverNativeModule from '@affine/server-native';
 import test from 'ava';
 
+import type { NativeLlmRerankRequest } from '../../native';
 import { ProviderMiddlewareConfig } from '../../plugins/copilot/config';
-import { normalizeOpenAIOptionsForModel } from '../../plugins/copilot/providers/openai';
+import {
+  normalizeOpenAIOptionsForModel,
+  OpenAIProvider,
+} from '../../plugins/copilot/providers/openai';
 import { CopilotProvider } from '../../plugins/copilot/providers/provider';
-import { normalizeRerankModel } from '../../plugins/copilot/providers/rerank';
 import {
   CopilotProviderType,
   ModelInputType,
@@ -46,6 +50,33 @@ class TestOpenAIProvider extends CopilotProvider<{ apiKey: string }> {
   }
 }
 
+class NativeRerankProtocolProvider extends OpenAIProvider {
+  override readonly models = [
+    {
+      id: 'gpt-5.2',
+      capabilities: [
+        {
+          input: [ModelInputType.Text],
+          output: [ModelOutputType.Text, ModelOutputType.Rerank],
+          defaultForOutputType: true,
+        },
+      ],
+    },
+  ];
+
+  override get config() {
+    return {
+      apiKey: 'test-key',
+      baseURL: 'https://api.openai.com/v1',
+      oldApiStyle: false,
+    };
+  }
+
+  override configured() {
+    return true;
+  }
+}
+
 function createProvider(profileMiddleware?: ProviderMiddlewareConfig) {
   const provider = new TestOpenAIProvider();
   (provider as any).AFFiNEConfig = {
@@ -126,14 +157,44 @@ test('normalizeOpenAIOptionsForModel should keep options for gpt-4.1', t => {
   );
 });
 
-test('normalizeOpenAIRerankModel should keep supported rerank models', t => {
-  t.is(normalizeRerankModel('gpt-4.1'), 'gpt-4.1');
-  t.is(normalizeRerankModel('gpt-4.1-mini'), 'gpt-4.1-mini');
-  t.is(normalizeRerankModel('gpt-5.2'), 'gpt-5.2');
-});
+test('OpenAI rerank should always use chat-completions native protocol', async t => {
+  const provider = new NativeRerankProtocolProvider();
+  let capturedProtocol: string | undefined;
+  let capturedRequest: NativeLlmRerankRequest | undefined;
 
-test('normalizeOpenAIRerankModel should fall back for unsupported models', t => {
-  t.is(normalizeRerankModel('gpt-5-mini'), 'gpt-5.2');
-  t.is(normalizeRerankModel('gemini-2.5-flash'), 'gpt-5.2');
-  t.is(normalizeRerankModel(undefined), 'gpt-5.2');
+  const original = (serverNativeModule as any).llmRerankDispatch;
+  (serverNativeModule as any).llmRerankDispatch = (
+    protocol: string,
+    _backendConfigJson: string,
+    requestJson: string
+  ) => {
+    capturedProtocol = protocol;
+    capturedRequest = JSON.parse(requestJson) as NativeLlmRerankRequest;
+    return JSON.stringify({ model: 'gpt-5.2', scores: [0.9, 0.1] });
+  };
+  t.teardown(() => {
+    (serverNativeModule as any).llmRerankDispatch = original;
+  });
+
+  const scores = await provider.rerank(
+    { modelId: 'gpt-5.2' },
+    {
+      query: 'programming',
+      candidates: [
+        { id: 'react', text: 'React is a UI library.' },
+        { id: 'weather', text: 'The weather is sunny today.' },
+      ],
+    }
+  );
+
+  t.deepEqual(scores, [0.9, 0.1]);
+  t.is(capturedProtocol, 'openai_chat');
+  t.deepEqual(capturedRequest, {
+    model: 'gpt-5.2',
+    query: 'programming',
+    candidates: [
+      { id: 'react', text: 'React is a UI library.' },
+      { id: 'weather', text: 'The weather is sunny today.' },
+    ],
+  });
 });

packages/backend/server/src/__tests__/copilot/tool-call-loop.spec.ts

@@ -34,6 +34,56 @@ test('ToolCallAccumulator should merge deltas and complete tool call', t => {
     id: 'call_1',
     name: 'doc_read',
     args: { doc_id: 'a1' },
+    rawArgumentsText: '{"doc_id":"a1"}',
+    thought: undefined,
+  });
+});
+
+test('ToolCallAccumulator should preserve invalid JSON instead of swallowing it', t => {
+  const accumulator = new ToolCallAccumulator();
+
+  accumulator.feedDelta({
+    type: 'tool_call_delta',
+    call_id: 'call_1',
+    name: 'doc_read',
+    arguments_delta: '{"doc_id":',
+  });
+
+  const pending = accumulator.drainPending();
+
+  t.is(pending.length, 1);
+  t.deepEqual(pending[0]?.id, 'call_1');
+  t.deepEqual(pending[0]?.name, 'doc_read');
+  t.deepEqual(pending[0]?.args, {});
+  t.is(pending[0]?.rawArgumentsText, '{"doc_id":');
+  t.truthy(pending[0]?.argumentParseError);
+});
+
+test('ToolCallAccumulator should prefer native canonical tool arguments metadata', t => {
+  const accumulator = new ToolCallAccumulator();
+
+  accumulator.feedDelta({
+    type: 'tool_call_delta',
+    call_id: 'call_1',
+    name: 'doc_read',
+    arguments_delta: '{"stale":true}',
+  });
+
+  const completed = accumulator.complete({
+    type: 'tool_call',
+    call_id: 'call_1',
+    name: 'doc_read',
+    arguments: {},
+    arguments_text: '{"doc_id":"a1"}',
+    arguments_error: 'invalid json',
+  });
+
+  t.deepEqual(completed, {
+    id: 'call_1',
+    name: 'doc_read',
+    args: {},
+    rawArgumentsText: '{"doc_id":"a1"}',
+    argumentParseError: 'invalid json',
     thought: undefined,
   });
 });
@@ -71,6 +121,8 @@ test('ToolSchemaExtractor should convert zod schema to json schema', t => {
 
 test('ToolCallLoop should execute tool call and continue to next round', async t => {
   const dispatchRequests: NativeLlmRequest[] = [];
+  const originalMessages = [{ role: 'user', content: 'read doc' }] as const;
+  const signal = new AbortController().signal;
 
   const dispatch = (request: NativeLlmRequest) => {
     dispatchRequests.push(request);
@@ -100,13 +152,17 @@ test('ToolCallLoop should execute tool call and continue to next round', async t
   };
 
   let executedArgs: Record<string, unknown> | null = null;
+  let executedMessages: unknown;
+  let executedSignal: AbortSignal | undefined;
   const loop = new ToolCallLoop(
     dispatch,
     {
       doc_read: {
         inputSchema: z.object({ doc_id: z.string() }),
-        execute: async args => {
+        execute: async (args, options) => {
           executedArgs = args;
+          executedMessages = options.messages;
+          executedSignal = options.signal;
           return { markdown: '# doc' };
         },
       },
@@ -114,6 +170,92 @@ test('ToolCallLoop should execute tool call and continue to next round', async t
     4
   );
 
+  const events: NativeLlmStreamEvent[] = [];
+  for await (const event of loop.run(
+    {
+      model: 'gpt-5-mini',
+      stream: true,
+      messages: [
+        { role: 'user', content: [{ type: 'text', text: 'read doc' }] },
+      ],
+    },
+    signal,
+    [...originalMessages]
+  )) {
+    events.push(event);
+  }
+
+  t.deepEqual(executedArgs, { doc_id: 'a1' });
+  t.deepEqual(executedMessages, originalMessages);
+  t.is(executedSignal, signal);
+  t.true(
+    dispatchRequests[1]?.messages.some(message => message.role === 'tool')
+  );
+  t.deepEqual(dispatchRequests[1]?.messages[1]?.content, [
+    {
+      type: 'tool_call',
+      call_id: 'call_1',
+      name: 'doc_read',
+      arguments: { doc_id: 'a1' },
+      arguments_text: '{"doc_id":"a1"}',
+      arguments_error: undefined,
+      thought: undefined,
+    },
+  ]);
+  t.deepEqual(dispatchRequests[1]?.messages[2]?.content, [
+    {
+      type: 'tool_result',
+      call_id: 'call_1',
+      name: 'doc_read',
+      arguments: { doc_id: 'a1' },
+      arguments_text: '{"doc_id":"a1"}',
+      arguments_error: undefined,
+      output: { markdown: '# doc' },
+      is_error: undefined,
+    },
+  ]);
+  t.deepEqual(
+    events.map(event => event.type),
+    ['tool_call', 'tool_result', 'text_delta', 'done']
+  );
+});
+
+test('ToolCallLoop should surface invalid JSON as tool error without executing', async t => {
+  let executed = false;
+  let round = 0;
+  const loop = new ToolCallLoop(
+    request => {
+      round += 1;
+      const hasToolResult = request.messages.some(
+        message => message.role === 'tool'
+      );
+      return (async function* (): AsyncIterableIterator<NativeLlmStreamEvent> {
+        if (!hasToolResult && round === 1) {
+          yield {
+            type: 'tool_call_delta',
+            call_id: 'call_1',
+            name: 'doc_read',
+            arguments_delta: '{"doc_id":',
+          };
+          yield { type: 'done', finish_reason: 'tool_calls' };
+          return;
+        }
+
+        yield { type: 'done', finish_reason: 'stop' };
+      })();
+    },
+    {
+      doc_read: {
+        inputSchema: z.object({ doc_id: z.string() }),
+        execute: async () => {
+          executed = true;
+          return { markdown: '# doc' };
+        },
+      },
+    },
+    2
+  );
+
   const events: NativeLlmStreamEvent[] = [];
   for await (const event of loop.run({
     model: 'gpt-5-mini',
@@ -123,12 +265,24 @@ test('ToolCallLoop should execute tool call and continue to next round', async t
     events.push(event);
   }
 
-  t.deepEqual(executedArgs, { doc_id: 'a1' });
-  t.true(
-    dispatchRequests[1]?.messages.some(message => message.role === 'tool')
-  );
-  t.deepEqual(
-    events.map(event => event.type),
-    ['tool_call', 'tool_result', 'text_delta', 'done']
-  );
+  t.false(executed);
+  t.true(events[0]?.type === 'tool_result');
+  t.deepEqual(events[0], {
+    type: 'tool_result',
+    call_id: 'call_1',
+    name: 'doc_read',
+    arguments: {},
+    arguments_text: '{"doc_id":',
+    arguments_error:
+      events[0]?.type === 'tool_result' ? events[0].arguments_error : undefined,
+    output: {
+      message: 'Invalid tool arguments JSON',
+      rawArguments: '{"doc_id":',
+      error:
+        events[0]?.type === 'tool_result'
+          ? events[0].arguments_error
+          : undefined,
+    },
+    is_error: true,
+  });
 });

packages/backend/server/src/__tests__/copilot/utils.spec.ts

@@ -1,12 +1,6 @@
 import test from 'ava';
-import { z } from 'zod';
 
-import {
-  chatToGPTMessage,
-  CitationFootnoteFormatter,
-  CitationParser,
-  StreamPatternParser,
-} from '../../plugins/copilot/providers/utils';
+import { CitationFootnoteFormatter } from '../../plugins/copilot/providers/utils';
 
 test('CitationFootnoteFormatter should format sorted footnotes from citation events', t => {
   const formatter = new CitationFootnoteFormatter();
@@ -50,67 +44,3 @@ test('CitationFootnoteFormatter should overwrite duplicated index with latest ur
     '[^1]: {"type":"url","url":"https%3A%2F%2Fexample.com%2Fnew"}'
   );
 });
-
-test('StreamPatternParser should keep state across chunks', t => {
-  const parser = new StreamPatternParser(pattern => {
-    if (pattern.kind === 'wrappedLink') {
-      return `[^${pattern.url}]`;
-    }
-    if (pattern.kind === 'index') {
-      return `[#${pattern.value}]`;
-    }
-    return `[${pattern.text}](${pattern.url})`;
-  });
-
-  const first = parser.write('ref ([AFFiNE](https://affine.pro');
-  const second = parser.write(')) and [2]');
-
-  t.is(first, 'ref ');
-  t.is(second, '[^https://affine.pro] and [#2]');
-  t.is(parser.end(), '');
-});
-
-test('CitationParser should convert wrapped links to numbered footnotes', t => {
-  const parser = new CitationParser();
-
-  const output = parser.parse('Use ([AFFiNE](https://affine.pro)) now');
-  t.is(output, 'Use [^1] now');
-  t.regex(
-    parser.end(),
-    /\[\^1\]: \{"type":"url","url":"https%3A%2F%2Faffine.pro"\}/
-  );
-});
-
-test('chatToGPTMessage should not mutate input and should keep system schema', async t => {
-  const schema = z.object({
-    query: z.string(),
-  });
-  const messages = [
-    {
-      role: 'system' as const,
-      content: 'You are helper',
-      params: { schema },
-    },
-    {
-      role: 'user' as const,
-      content: '',
-      attachments: ['https://example.com/a.png'],
-    },
-  ];
-  const firstRef = messages[0];
-  const secondRef = messages[1];
-  const [system, normalized, parsedSchema] = await chatToGPTMessage(
-    messages,
-    false
-  );
-
-  t.is(system, 'You are helper');
-  t.is(parsedSchema, schema);
-  t.is(messages.length, 2);
-  t.is(messages[0], firstRef);
-  t.is(messages[1], secondRef);
-  t.deepEqual(normalized[0], {
-    role: 'user',
-    content: [{ type: 'text', text: '[no content]' }],
-  });
-});

packages/backend/server/src/__tests__/mocks/copilot.mock.ts

@@ -33,7 +33,7 @@ export class MockCopilotProvider extends OpenAIProvider {
       id: 'test-image',
       capabilities: [
         {
-          input: [ModelInputType.Text],
+          input: [ModelInputType.Text, ModelInputType.Image],
           output: [ModelOutputType.Image],
           defaultForOutputType: true,
         },