feat(server): refactor provider interface (#11665)

fix AI-4
fix AI-18

better provider/model choose to allow fallback to similar models (e.g., self-hosted) when the provider is not fully configured
split functions of different output types

packages/backend/server/src/__tests__/mocks/copilot.mock.ts

@@ -1,9 +1,13 @@
 import { randomBytes } from 'node:crypto';
 
 import {
-  CopilotCapability,
   CopilotChatOptions,
   CopilotEmbeddingOptions,
+  CopilotImageOptions,
+  CopilotStructuredOptions,
+  ModelConditions,
+  ModelInputType,
+  ModelOutputType,
   PromptMessage,
 } from '../../plugins/copilot/providers';
 import {
@@ -14,49 +18,135 @@ import { sleep } from '../utils/utils';
 
 export class MockCopilotProvider extends OpenAIProvider {
   override readonly models = [
-    'test',
-    'gpt-4o',
-    'gpt-4o-2024-08-06',
-    'gpt-4.1',
-    'gpt-4.1-2025-04-14',
-    'gpt-4.1-mini',
-    'fast-sdxl/image-to-image',
-    'lcm-sd15-i2i',
-    'clarity-upscaler',
-    'imageutils/rembg',
-    'gemini-2.5-pro-preview-03-25',
+    {
+      id: 'test',
+      capabilities: [
+        {
+          input: [ModelInputType.Text],
+          output: [ModelOutputType.Text],
+          defaultForOutputType: true,
+        },
+      ],
+    },
+    {
+      id: 'test-image',
+      capabilities: [
+        {
+          input: [ModelInputType.Text],
+          output: [ModelOutputType.Image],
+          defaultForOutputType: true,
+        },
+      ],
+    },
+    {
+      id: 'gpt-4o',
+      capabilities: [
+        {
+          input: [ModelInputType.Text, ModelInputType.Image],
+          output: [ModelOutputType.Text],
+        },
+      ],
+    },
+    {
+      id: 'gpt-4o-2024-08-06',
+      capabilities: [
+        {
+          input: [ModelInputType.Text, ModelInputType.Image],
+          output: [ModelOutputType.Text],
+        },
+      ],
+    },
+    {
+      id: 'gpt-4.1',
+      capabilities: [
+        {
+          input: [ModelInputType.Text, ModelInputType.Image],
+          output: [ModelOutputType.Text],
+        },
+      ],
+    },
+    {
+      id: 'gpt-4.1-2025-04-14',
+      capabilities: [
+        {
+          input: [ModelInputType.Text, ModelInputType.Image],
+          output: [ModelOutputType.Text],
+        },
+      ],
+    },
+    {
+      id: 'gpt-4.1-mini',
+      capabilities: [
+        {
+          input: [ModelInputType.Text, ModelInputType.Image],
+          output: [ModelOutputType.Text],
+        },
+      ],
+    },
+    {
+      id: 'lcm-sd15-i2i',
+      capabilities: [
+        {
+          input: [ModelInputType.Image],
+          output: [ModelOutputType.Image],
+        },
+      ],
+    },
+    {
+      id: 'clarity-upscaler',
+      capabilities: [
+        {
+          input: [ModelInputType.Image],
+          output: [ModelOutputType.Image],
+        },
+      ],
+    },
+    {
+      id: 'imageutils/rembg',
+      capabilities: [
+        {
+          input: [ModelInputType.Image],
+          output: [ModelOutputType.Image],
+        },
+      ],
+    },
+    {
+      id: 'gemini-2.5-pro-preview-03-25',
+      capabilities: [
+        {
+          input: [ModelInputType.Text, ModelInputType.Image],
+          output: [ModelOutputType.Text, ModelOutputType.Structured],
+        },
+      ],
+    },
   ];
 
-  override readonly capabilities = [
-    CopilotCapability.TextToText,
-    CopilotCapability.TextToEmbedding,
-    CopilotCapability.TextToImage,
-    CopilotCapability.ImageToImage,
-    CopilotCapability.ImageToText,
-  ];
-
-  // ====== text to text ======
-
-  override async generateText(
+  override async text(
+    cond: ModelConditions,
     messages: PromptMessage[],
-    model: string = 'test',
     options: CopilotChatOptions = {}
   ): Promise<string> {
-    await this.checkParams({ messages, model, options });
+    const fullCond = {
+      ...cond,
+      outputType: ModelOutputType.Text,
+    };
+    await this.checkParams({ messages, cond: fullCond, options });
     // make some time gap for history test case
     await sleep(100);
     return 'generate text to text';
   }
 
-  override async *generateTextStream(
+  override async *streamText(
+    cond: ModelConditions,
     messages: PromptMessage[],
-    model: string = 'gpt-4.1-mini',
     options: CopilotChatOptions = {}
   ): AsyncIterable<string> {
-    await this.checkParams({ messages, model, options });
+    const fullCond = { ...cond, outputType: ModelOutputType.Text };
+    await this.checkParams({ messages, cond: fullCond, options });
 
     // make some time gap for history test case
     await sleep(100);
+
     const result = 'generate text to text stream';
     for (const message of result) {
       yield message;
@@ -66,52 +156,60 @@ export class MockCopilotProvider extends OpenAIProvider {
     }
   }
 
+  override async structure(
+    cond: ModelConditions,
+    messages: PromptMessage[],
+    options: CopilotStructuredOptions = {}
+  ): Promise<string> {
+    const fullCond = { ...cond, outputType: ModelOutputType.Structured };
+    await this.checkParams({ messages, cond: fullCond, options });
+
+    // make some time gap for history test case
+    await sleep(100);
+    return 'generate text to text';
+  }
+
+  override async *streamImages(
+    cond: ModelConditions,
+    messages: PromptMessage[],
+    options: CopilotImageOptions = {}
+  ) {
+    const fullCond = { ...cond, outputType: ModelOutputType.Image };
+    await this.checkParams({ messages, cond: fullCond, options });
+
+    // make some time gap for history test case
+    await sleep(100);
+
+    const { content: prompt } = [...messages].pop() || {};
+    if (!prompt) throw new Error('Prompt is required');
+
+    const imageUrls = [
+      `https://example.com/${cond.modelId || 'test'}.jpg`,
+      prompt,
+    ];
+
+    for (const imageUrl of imageUrls) {
+      yield imageUrl;
+      if (options.signal?.aborted) {
+        break;
+      }
+    }
+    return;
+  }
+
   // ====== text to embedding ======
 
-  override async generateEmbedding(
+  override async embedding(
+    cond: ModelConditions,
     messages: string | string[],
-    model: string,
     options: CopilotEmbeddingOptions = { dimensions: DEFAULT_DIMENSIONS }
   ): Promise<number[][]> {
     messages = Array.isArray(messages) ? messages : [messages];
-    await this.checkParams({ embeddings: messages, model, options });
+    const fullCond = { ...cond, outputType: ModelOutputType.Embedding };
+    await this.checkParams({ embeddings: messages, cond: fullCond, options });
 
     // make some time gap for history test case
     await sleep(100);
     return [Array.from(randomBytes(options.dimensions)).map(v => v % 128)];
   }
-
-  // ====== text to image ======
-  override async generateImages(
-    messages: PromptMessage[],
-    model: string = 'test',
-    _options: {
-      signal?: AbortSignal;
-      user?: string;
-    } = {}
-  ): Promise<Array<string>> {
-    const { content: prompt } = messages[0] || {};
-    if (!prompt) {
-      throw new Error('Prompt is required');
-    }
-
-    // make some time gap for history test case
-    await sleep(100);
-    // just let test case can easily verify the final prompt
-    return [`https://example.com/${model}.jpg`, prompt];
-  }
-
-  override async *generateImagesStream(
-    messages: PromptMessage[],
-    model: string = 'dall-e-3',
-    options: {
-      signal?: AbortSignal;
-      user?: string;
-    } = {}
-  ): AsyncIterable<string> {
-    const ret = await this.generateImages(messages, model, options);
-    for (const url of ret) {
-      yield url;
-    }
-  }
 }