fix(server): empty mimetype attachments fallback (#11869)

packages/backend/server/src/plugins/copilot/providers/fal.ts

@@ -138,7 +138,15 @@ export class FalProvider
     );
     return {
       model_name: options.modelName || undefined,
-      image_url: attachments?.[0],
+      image_url: attachments
+        ?.map(v =>
+          typeof v === 'string'
+            ? v
+            : v.mimeType.startsWith('image/')
+              ? v.attachment
+              : undefined
+        )
+        .filter(v => !!v)[0],
       prompt: content.trim(),
       loras: lora.length ? lora : undefined,
       controlnets: controlnets.length ? controlnets : undefined,

packages/backend/server/src/plugins/copilot/providers/types.ts

@@ -52,7 +52,15 @@ export const ChatMessageRole = Object.values(AiPromptRole) as [
 
 export const PureMessageSchema = z.object({
   content: z.string(),
-  attachments: z.array(z.string()).optional().nullable(),
+  attachments: z
+    .array(
+      z.union([
+        z.string(),
+        z.object({ attachment: z.string(), mimeType: z.string() }),
+      ])
+    )
+    .optional()
+    .nullable(),
   params: z.record(z.any()).optional().nullable(),
 });
 

packages/backend/server/src/plugins/copilot/providers/utils.ts

@@ -35,15 +35,26 @@ const FORMAT_INFER_MAP: Record<string, string> = {
   flv: 'video/flv',
 };
 
-function inferMimeType(url: string) {
+async function inferMimeType(url: string) {
   if (url.startsWith('data:')) {
     return url.split(';')[0].split(':')[1];
   }
-  const extension = url.split('.').pop();
+  const pathname = new URL(url).pathname;
+  const extension = pathname.split('.').pop();
   if (extension) {
-    return FORMAT_INFER_MAP[extension];
+    const ext = FORMAT_INFER_MAP[extension];
+    if (ext) {
+      return ext;
+    }
+    const mimeType = await fetch(url, {
+      method: 'HEAD',
+      redirect: 'follow',
+    }).then(res => res.headers.get('Content-Type'));
+    if (mimeType) {
+      return mimeType;
+    }
   }
-  return undefined;
+  return 'application/octet-stream';
 }
 
 export async function chatToGPTMessage(
@@ -66,19 +77,24 @@ export async function chatToGPTMessage(
         contents.push({ type: 'text', text: content });
       }
 
-      for (const url of attachments) {
-        if (SIMPLE_IMAGE_URL_REGEX.test(url)) {
-          const mimeType =
-            typeof mimetype === 'string' ? mimetype : inferMimeType(url);
-          if (mimeType) {
-            if (mimeType.startsWith('image/')) {
-              contents.push({ type: 'image', image: url, mimeType });
-            } else {
-              const data = url.startsWith('data:')
-                ? await fetch(url).then(r => r.arrayBuffer())
-                : new URL(url);
-              contents.push({ type: 'file' as const, data, mimeType });
-            }
+      for (let attachment of attachments) {
+        let mimeType: string;
+        if (typeof attachment === 'string') {
+          mimeType =
+            typeof mimetype === 'string'
+              ? mimetype
+              : await inferMimeType(attachment);
+        } else {
+          ({ attachment, mimeType } = attachment);
+        }
+        if (SIMPLE_IMAGE_URL_REGEX.test(attachment)) {
+          if (mimeType.startsWith('image/')) {
+            contents.push({ type: 'image', image: attachment, mimeType });
+          } else {
+            const data = attachment.startsWith('data:')
+              ? await fetch(attachment).then(r => r.arrayBuffer())
+              : new URL(attachment);
+            contents.push({ type: 'file' as const, data, mimeType });
           }
         }
       }

packages/backend/server/src/plugins/copilot/resolver.ts

@@ -34,6 +34,7 @@ import { Admin } from '../../core/common';
 import { AccessController } from '../../core/permission';
 import { UserType } from '../../core/user';
 import { PromptService } from './prompt';
+import { PromptMessage } from './providers';
 import { ChatSessionService } from './session';
 import { CopilotStorage } from './storage';
 import {
@@ -113,7 +114,7 @@ class CreateChatMessageInput implements Omit<SubmittedMessage, 'content'> {
   @Field(() => String, { nullable: true })
   content!: string | undefined;
 
-  @Field(() => [String], { nullable: true })
+  @Field(() => [String], { nullable: true, deprecationReason: 'use blobs' })
   attachments!: string[] | undefined;
 
   @Field(() => [GraphQLUpload], { nullable: true })
@@ -527,8 +528,8 @@ export class CopilotResolver {
       throw new BadRequestException('Session not found');
     }
 
+    const attachments: PromptMessage['attachments'] = options.attachments || [];
     if (options.blobs) {
-      options.attachments = options.attachments || [];
       const { workspaceId } = session.config;
 
       const blobs = await Promise.all(options.blobs);
@@ -539,18 +540,18 @@ export class CopilotResolver {
         const filename = createHash('sha256')
           .update(uploaded.buffer)
           .digest('base64url');
-        const link = await this.storage.put(
+        const attachment = await this.storage.put(
           user.id,
           workspaceId,
           filename,
           uploaded.buffer
         );
-        options.attachments.push(link);
+        attachments.push({ attachment, mimeType: blob.mimetype });
       }
     }
 
     try {
-      return await this.chatSession.createMessage(options);
+      return await this.chatSession.createMessage({ ...options, attachments });
     } catch (e: any) {
       throw new CopilotFailedToCreateMessage(e.message);
     }

packages/backend/server/src/plugins/copilot/session.ts

@@ -166,7 +166,11 @@ export class ChatSession implements AsyncDisposable {
         firstMessage.attachments || [],
       ]
         .flat()
-        .filter(v => !!v?.trim());
+        .filter(v =>
+          typeof v === 'string'
+            ? !!v.trim()
+            : v && v.attachment.trim() && v.mimeType
+        );
 
       return finished;
     }
@@ -553,7 +557,12 @@ export class ChatSessionService {
                     action: prompt.action || null,
                     tokens: tokenCost,
                     createdAt,
-                    messages: preload.concat(ret.data),
+                    messages: preload.concat(ret.data).map(m => ({
+                      ...m,
+                      attachments: m.attachments
+                        ?.map(a => (typeof a === 'string' ? a : a.attachment))
+                        .filter(a => !!a),
+                    })),
                   };
                 } else {
                   this.logger.error(