feat(core): add stream object api (#12841)

Close [AI-193](https://linear.app/affine-design/issue/AI-193)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Added support for streaming structured AI chat responses as objects,
enabling richer and more interactive chat experiences.
- Chat messages now include a new field displaying structured stream
objects, such as reasoning steps, text deltas, tool calls, and tool
results.
- GraphQL APIs and queries updated to expose these structured streaming
objects in chat histories.
  - Introduced a new streaming chat endpoint for object-based responses.

- **Bug Fixes**
- Improved error handling for streaming responses to ensure more robust
and informative error reporting.

- **Refactor**
- Centralized and streamlined session preparation and streaming logic
for AI chat providers.
  - Unified streaming setup across multiple AI model providers.

- **Tests**
- Extended test coverage for streaming object responses to ensure
reliability and correctness.

- **Documentation**
- Updated type definitions and schemas to reflect new streaming object
capabilities in both backend and frontend code.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Co-authored-by: DarkSky <25152247+darkskygit@users.noreply.github.com>

packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts

@@ -21,11 +21,17 @@ import { CopilotProvider } from '../provider';
 import type {
   CopilotChatOptions,
   CopilotImageOptions,
+  CopilotProviderModel,
   ModelConditions,
   PromptMessage,
+  StreamObject,
 } from '../types';
 import { ModelOutputType } from '../types';
-import { chatToGPTMessage, TextStreamParser } from '../utils';
+import {
+  chatToGPTMessage,
+  StreamObjectParser,
+  TextStreamParser,
+} from '../utils';
 
 export const DEFAULT_DIMENSIONS = 256;
 
@@ -150,21 +156,7 @@ export abstract class GeminiProvider<T> extends CopilotProvider<T> {
 
     try {
       metrics.ai.counter('chat_text_stream_calls').add(1, { model: model.id });
-      const [system, msgs] = await chatToGPTMessage(messages);
-
-      const { fullStream } = streamText({
-        model: this.instance(model.id, {
-          useSearchGrounding: this.useSearchGrounding(options),
-        }),
-        system,
-        messages: msgs,
-        abortSignal: options.signal,
-        maxSteps: this.MAX_STEPS,
-        providerOptions: {
-          google: this.getGeminiOptions(options, model.id),
-        },
-      });
-
+      const fullStream = await this.getFullStream(model, messages, options);
       const parser = new TextStreamParser();
       for await (const chunk of fullStream) {
         const result = parser.parse(chunk);
@@ -180,6 +172,60 @@ export abstract class GeminiProvider<T> extends CopilotProvider<T> {
     }
   }
 
+  override async *streamObject(
+    cond: ModelConditions,
+    messages: PromptMessage[],
+    options: CopilotChatOptions = {}
+  ): AsyncIterable<StreamObject> {
+    const fullCond = { ...cond, outputType: ModelOutputType.Object };
+    await this.checkParams({ cond: fullCond, messages, options });
+    const model = this.selectModel(fullCond);
+
+    try {
+      metrics.ai
+        .counter('chat_object_stream_calls')
+        .add(1, { model: model.id });
+      const fullStream = await this.getFullStream(model, messages, options);
+      const parser = new StreamObjectParser();
+      for await (const chunk of fullStream) {
+        const result = parser.parse(chunk);
+        if (result) {
+          yield result;
+        }
+        if (options.signal?.aborted) {
+          await fullStream.cancel();
+          break;
+        }
+      }
+    } catch (e: any) {
+      metrics.ai
+        .counter('chat_object_stream_errors')
+        .add(1, { model: model.id });
+      throw this.handleError(e);
+    }
+  }
+
+  private async getFullStream(
+    model: CopilotProviderModel,
+    messages: PromptMessage[],
+    options: CopilotChatOptions = {}
+  ) {
+    const [system, msgs] = await chatToGPTMessage(messages);
+    const { fullStream } = streamText({
+      model: this.instance(model.id, {
+        useSearchGrounding: this.useSearchGrounding(options),
+      }),
+      system,
+      messages: msgs,
+      abortSignal: options.signal,
+      maxSteps: this.MAX_STEPS,
+      providerOptions: {
+        google: this.getGeminiOptions(options, model.id),
+      },
+    });
+    return fullStream;
+  }
+
   private getGeminiOptions(options: CopilotChatOptions, model: string) {
     const result: GoogleGenerativeAIProviderOptions = {};
     if (options?.reasoning && this.isReasoningModel(model)) {

packages/backend/server/src/plugins/copilot/providers/gemini/generative.ts

@@ -25,7 +25,11 @@ export class GeminiGenerativeProvider extends GeminiProvider<GeminiGenerativeCon
             ModelInputType.Image,
             ModelInputType.Audio,
           ],
-          output: [ModelOutputType.Text, ModelOutputType.Structured],
+          output: [
+            ModelOutputType.Text,
+            ModelOutputType.Object,
+            ModelOutputType.Structured,
+          ],
           defaultForOutputType: true,
         },
       ],
@@ -40,7 +44,11 @@ export class GeminiGenerativeProvider extends GeminiProvider<GeminiGenerativeCon
             ModelInputType.Image,
             ModelInputType.Audio,
           ],
-          output: [ModelOutputType.Text, ModelOutputType.Structured],
+          output: [
+            ModelOutputType.Text,
+            ModelOutputType.Object,
+            ModelOutputType.Structured,
+          ],
         },
       ],
     },
@@ -54,7 +62,11 @@ export class GeminiGenerativeProvider extends GeminiProvider<GeminiGenerativeCon
             ModelInputType.Image,
             ModelInputType.Audio,
           ],
-          output: [ModelOutputType.Text, ModelOutputType.Structured],
+          output: [
+            ModelOutputType.Text,
+            ModelOutputType.Object,
+            ModelOutputType.Structured,
+          ],
         },
       ],
     },

packages/backend/server/src/plugins/copilot/providers/gemini/vertex.ts

@@ -23,7 +23,11 @@ export class GeminiVertexProvider extends GeminiProvider<GeminiVertexConfig> {
             ModelInputType.Image,
             ModelInputType.Audio,
           ],
-          output: [ModelOutputType.Text, ModelOutputType.Structured],
+          output: [
+            ModelOutputType.Text,
+            ModelOutputType.Object,
+            ModelOutputType.Structured,
+          ],
         },
       ],
     },
@@ -37,7 +41,11 @@ export class GeminiVertexProvider extends GeminiProvider<GeminiVertexConfig> {
             ModelInputType.Image,
             ModelInputType.Audio,
           ],
-          output: [ModelOutputType.Text, ModelOutputType.Structured],
+          output: [
+            ModelOutputType.Text,
+            ModelOutputType.Object,
+            ModelOutputType.Structured,
+          ],
         },
       ],
     },