feat(core): add stream object api (#12841)

Close [AI-193](https://linear.app/affine-design/issue/AI-193)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Added support for streaming structured AI chat responses as objects,
enabling richer and more interactive chat experiences.
- Chat messages now include a new field displaying structured stream
objects, such as reasoning steps, text deltas, tool calls, and tool
results.
- GraphQL APIs and queries updated to expose these structured streaming
objects in chat histories.
  - Introduced a new streaming chat endpoint for object-based responses.

- **Bug Fixes**
- Improved error handling for streaming responses to ensure more robust
and informative error reporting.

- **Refactor**
- Centralized and streamlined session preparation and streaming logic
for AI chat providers.
  - Unified streaming setup across multiple AI model providers.

- **Tests**
- Extended test coverage for streaming object responses to ensure
reliability and correctness.

- **Documentation**
- Updated type definitions and schemas to reflect new streaming object
capabilities in both backend and frontend code.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Co-authored-by: DarkSky <25152247+darkskygit@users.noreply.github.com>

packages/backend/server/src/__tests__/copilot-provider.spec.ts

@@ -1,5 +1,6 @@
 import type { ExecutionContext, TestFn } from 'ava';
 import ava from 'ava';
+import { z } from 'zod';
 
 import { ServerFeature, ServerService } from '../core';
 import { AuthService } from '../core/auth';
@@ -9,6 +10,8 @@ import { prompts, PromptService } from '../plugins/copilot/prompt';
 import {
   CopilotProviderFactory,
   CopilotProviderType,
+  StreamObject,
+  StreamObjectSchema,
 } from '../plugins/copilot/providers';
 import { TranscriptionResponseSchema } from '../plugins/copilot/transcript/types';
 import {
@@ -183,6 +186,16 @@ const checkUrl = (url: string) => {
   }
 };
 
+const checkStreamObjects = (result: string) => {
+  try {
+    const streamObjects = JSON.parse(result);
+    z.array(StreamObjectSchema).parse(streamObjects);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
 const retry = async (
   action: string,
   t: ExecutionContext<Tester>,
@@ -387,6 +400,20 @@ The term **“CRDT”** was first introduced by Marc Shapiro, Nuno Preguiça, Ca
     },
     type: 'text' as const,
   },
+  {
+    name: 'stream objects',
+    promptName: ['Chat With AFFiNE AI'],
+    messages: [
+      {
+        role: 'user' as const,
+        content: 'what is AFFiNE AI',
+      },
+    ],
+    verifier: (t: ExecutionContext<Tester>, result: string) => {
+      t.truthy(checkStreamObjects(result), 'should be valid stream objects');
+    },
+    type: 'object' as const,
+  },
   {
     name: 'Should transcribe short audio',
     promptName: ['Transcript audio'],
@@ -680,6 +707,27 @@ for (const {
               verifier?.(t, result);
               break;
             }
+            case 'object': {
+              const streamObjects: StreamObject[] = [];
+              for await (const chunk of provider.streamObject(
+                { modelId: prompt.model },
+                [
+                  ...prompt.finish(
+                    messages.reduce(
+                      (acc, m) => Object.assign(acc, (m as any).params || {}),
+                      {}
+                    )
+                  ),
+                  ...messages,
+                ],
+                finalConfig
+              )) {
+                streamObjects.push(chunk);
+              }
+              t.truthy(streamObjects, 'should return result');
+              verifier?.(t, JSON.stringify(streamObjects));
+              break;
+            }
             case 'image': {
               const finalMessage = [...messages];
               const params = {};

packages/backend/server/src/__tests__/copilot.e2e.ts

@@ -39,6 +39,7 @@ import {
   array2sse,
   audioTranscription,
   chatWithImages,
+  chatWithStreamObject,
   chatWithText,
   chatWithTextStream,
   chatWithWorkflow,
@@ -512,6 +513,28 @@ test('should be able to chat with api', async t => {
     );
   }
 
+  {
+    const sessionId = await createCopilotSession(
+      app,
+      id,
+      randomUUID(),
+      textPromptName
+    );
+    const messageId = await createCopilotMessage(app, sessionId);
+
+    const ret4 = await chatWithStreamObject(app, sessionId, messageId);
+
+    const objects = Array.from('generate text to object stream').map(data =>
+      JSON.stringify({ type: 'text-delta', textDelta: data })
+    );
+
+    t.is(
+      ret4,
+      textToEventStream(objects, messageId),
+      'should be able to chat with stream object'
+    );
+  }
+
   Sinon.restore();
 });
 

packages/backend/server/src/__tests__/mocks/copilot.mock.ts

@@ -9,6 +9,7 @@ import {
   ModelInputType,
   ModelOutputType,
   PromptMessage,
+  StreamObject,
 } from '../../plugins/copilot/providers';
 import {
   DEFAULT_DIMENSIONS,
@@ -23,7 +24,7 @@ export class MockCopilotProvider extends OpenAIProvider {
       capabilities: [
         {
           input: [ModelInputType.Text],
-          output: [ModelOutputType.Text],
+          output: [ModelOutputType.Text, ModelOutputType.Object],
           defaultForOutputType: true,
         },
       ],
@@ -43,7 +44,7 @@ export class MockCopilotProvider extends OpenAIProvider {
       capabilities: [
         {
           input: [ModelInputType.Text, ModelInputType.Image],
-          output: [ModelOutputType.Text],
+          output: [ModelOutputType.Text, ModelOutputType.Object],
         },
       ],
     },
@@ -52,7 +53,7 @@ export class MockCopilotProvider extends OpenAIProvider {
       capabilities: [
         {
           input: [ModelInputType.Text, ModelInputType.Image],
-          output: [ModelOutputType.Text],
+          output: [ModelOutputType.Text, ModelOutputType.Object],
         },
       ],
     },
@@ -61,7 +62,7 @@ export class MockCopilotProvider extends OpenAIProvider {
       capabilities: [
         {
           input: [ModelInputType.Text, ModelInputType.Image],
-          output: [ModelOutputType.Text],
+          output: [ModelOutputType.Text, ModelOutputType.Object],
         },
       ],
     },
@@ -70,7 +71,7 @@ export class MockCopilotProvider extends OpenAIProvider {
       capabilities: [
         {
           input: [ModelInputType.Text, ModelInputType.Image],
-          output: [ModelOutputType.Text],
+          output: [ModelOutputType.Text, ModelOutputType.Object],
         },
       ],
     },
@@ -79,7 +80,11 @@ export class MockCopilotProvider extends OpenAIProvider {
       capabilities: [
         {
           input: [ModelInputType.Text, ModelInputType.Image],
-          output: [ModelOutputType.Text, ModelOutputType.Structured],
+          output: [
+            ModelOutputType.Text,
+            ModelOutputType.Object,
+            ModelOutputType.Structured,
+          ],
         },
       ],
     },
@@ -98,7 +103,11 @@ export class MockCopilotProvider extends OpenAIProvider {
       capabilities: [
         {
           input: [ModelInputType.Text, ModelInputType.Image],
-          output: [ModelOutputType.Text, ModelOutputType.Structured],
+          output: [
+            ModelOutputType.Text,
+            ModelOutputType.Object,
+            ModelOutputType.Structured,
+          ],
         },
       ],
     },
@@ -195,4 +204,24 @@ export class MockCopilotProvider extends OpenAIProvider {
     await sleep(100);
     return [Array.from(randomBytes(options.dimensions)).map(v => v % 128)];
   }
+
+  override async *streamObject(
+    cond: ModelConditions,
+    messages: PromptMessage[],
+    options: CopilotChatOptions = {}
+  ): AsyncIterable<StreamObject> {
+    const fullCond = { ...cond, outputType: ModelOutputType.Object };
+    await this.checkParams({ messages, cond: fullCond, options });
+
+    // make some time gap for history test case
+    await sleep(100);
+
+    const result = 'generate text to object stream';
+    for (const data of result) {
+      yield { type: 'text-delta', textDelta: data } as const;
+      if (options.signal?.aborted) {
+        break;
+      }
+    }
+  }
 }

packages/backend/server/src/__tests__/utils/copilot.ts

@@ -582,6 +582,14 @@ export async function chatWithImages(
   return chatWithText(app, sessionId, messageId, '/images');
 }
 
+export async function chatWithStreamObject(
+  app: TestingApp,
+  sessionId: string,
+  messageId?: string
+) {
+  return chatWithText(app, sessionId, messageId, '/stream-object');
+}
+
 export async function unsplashSearch(
   app: TestingApp,
   params: Record<string, string> = {}