feat(server): switch i2i to gpt (#12238)

fix AI-14
fix AI-17
fix AI-39
fix AI-112

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Expanded and reorganized prompt options for text and image actions, adding new prompts for image generation, style conversions, upscaling, background removal, and sticker creation.
  - Enhanced image editing capabilities with direct support for image attachments in prompts.

- **Improvements**
  - Updated prompt names and descriptions to be more user-friendly and descriptive.
  - Simplified and clarified prompt selection and image processing workflows with improved default behaviors.
  - Better organization of prompts through clear grouping and categorization.

- **Bug Fixes**
  - Improved validation and handling of image attachments during editing requests.

- **Refactor**
  - Internal code restructuring of prompts and provider logic for clarity and maintainability without affecting user workflows.
  - Refined message handling and content merging logic to ensure consistent prompt processing.
  - Adjusted image attachment rendering logic for improved display consistency.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/plugins/copilot/providers/openai.ts

@@ -13,6 +13,7 @@ import {
   streamText,
   ToolSet,
 } from 'ai';
+import { z } from 'zod';
 
 import {
   CopilotPromptInvalid,
@@ -40,6 +41,20 @@ export type OpenAIConfig = {
   baseUrl?: string;
 };
 
+const ImageResponseSchema = z.union([
+  z.object({
+    data: z.array(z.object({ b64_json: z.string() })),
+  }),
+  z.object({
+    error: z.object({
+      message: z.string(),
+      type: z.string().nullish(),
+      param: z.any().nullish(),
+      code: z.union([z.string(), z.number()]).nullish(),
+    }),
+  }),
+]);
+
 export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
   readonly type = CopilotProviderType.OpenAI;
 
@@ -389,6 +404,63 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
     }
   }
 
+  // ====== text to image ======
+  private async *generateImageWithAttachments(
+    model: string,
+    prompt: string,
+    attachments: NonNullable<PromptMessage['attachments']>
+  ): AsyncGenerator<string> {
+    const form = new FormData();
+    form.set('model', model);
+    form.set('prompt', prompt);
+    form.set('output_format', 'webp');
+
+    for (const [idx, entry] of attachments.entries()) {
+      const url = typeof entry === 'string' ? entry : entry.attachment;
+      const resp = await fetch(url);
+      if (resp.ok) {
+        const type = resp.headers.get('content-type');
+        if (type && type.startsWith('image/')) {
+          const buffer = new Uint8Array(await resp.arrayBuffer());
+          const file = new File([buffer], `${idx}.png`, { type });
+          form.append('image[]', file);
+        }
+      }
+    }
+
+    if (!form.getAll('image[]').length) {
+      throw new CopilotPromptInvalid(
+        'No valid image attachments found. Please attach images.'
+      );
+    }
+
+    const url = `${this.config.baseUrl || 'https://api.openai.com'}/v1/images/edits`;
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${this.config.apiKey}` },
+      body: form,
+    });
+
+    if (!res.ok) {
+      throw new Error(`OpenAI API error ${res.status}: ${await res.text()}`);
+    }
+
+    const json = await res.json();
+    const imageResponse = ImageResponseSchema.safeParse(json);
+    if (imageResponse.success) {
+      const data = imageResponse.data;
+      if ('error' in data) {
+        throw new Error(data.error.message);
+      } else {
+        for (const image of data.data) {
+          yield `data:image/webp;base64,${image.b64_json}`;
+        }
+      }
+    } else {
+      throw new Error(imageResponse.error.message);
+    }
+  }
+
   override async *streamImages(
     cond: ModelConditions,
     messages: PromptMessage[],
@@ -402,30 +474,33 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
       .counter('generate_images_stream_calls')
       .add(1, { model: model.id });
 
-    const { content: prompt } = [...messages].pop() || {};
+    const { content: prompt, attachments } = [...messages].pop() || {};
     if (!prompt) throw new CopilotPromptInvalid('Prompt is required');
 
     try {
-      const modelInstance = this.#instance.image(model.id);
-
-      const result = await generateImage({
-        model: modelInstance,
-        prompt,
-        providerOptions: {
-          openai: {
-            quality: options.quality || null,
+      if (attachments && attachments.length > 0) {
+        yield* this.generateImageWithAttachments(model.id, prompt, attachments);
+      } else {
+        const modelInstance = this.#instance.image(model.id);
+        const result = await generateImage({
+          model: modelInstance,
+          prompt,
+          providerOptions: {
+            openai: {
+              quality: options.quality || null,
+            },
           },
-        },
-      });
+        });
 
-      const imageUrls = result.images.map(
-        image => `data:image/png;base64,${image.base64}`
-      );
+        const imageUrls = result.images.map(
+          image => `data:image/png;base64,${image.base64}`
+        );
 
-      for (const imageUrl of imageUrls) {
-        yield imageUrl;
-        if (options.signal?.aborted) {
-          break;
+        for (const imageUrl of imageUrls) {
+          yield imageUrl;
+          if (options.signal?.aborted) {
+            break;
+          }
         }
       }
       return;

packages/backend/server/src/plugins/copilot/providers/utils.ts

@@ -39,7 +39,7 @@ const FORMAT_INFER_MAP: Record<string, string> = {
   flv: 'video/flv',
 };
 
-async function inferMimeType(url: string) {
+export async function inferMimeType(url: string) {
   if (url.startsWith('data:')) {
     return url.split(';')[0].split(':')[1];
   }