fix(server): empty mimetype attachments fallback (#11869)

packages/backend/server/src/__tests__/copilot.e2e.ts

@@ -385,6 +385,45 @@ test('should create message correctly', async t => {
     t.truthy(messageId, 'should be able to create message with valid session');
   }
 
+  {
+    // with attachment url
+    {
+      const { id } = await createWorkspace(app);
+      const sessionId = await createCopilotSession(
+        app,
+        id,
+        randomUUID(),
+        promptName
+      );
+      const messageId = await createCopilotMessage(app, sessionId, undefined, [
+        'http://example.com/cat.jpg',
+      ]);
+      t.truthy(messageId, 'should be able to create message with url link');
+    }
+
+    // with attachment
+    {
+      const { id } = await createWorkspace(app);
+      const sessionId = await createCopilotSession(
+        app,
+        id,
+        randomUUID(),
+        promptName
+      );
+      const smallestPng =
+        'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII';
+      const pngData = await fetch(smallestPng).then(res => res.arrayBuffer());
+      const messageId = await createCopilotMessage(
+        app,
+        sessionId,
+        undefined,
+        undefined,
+        [new File([new Uint8Array(pngData)], '1.png', { type: 'image/png' })]
+      );
+      t.truthy(messageId, 'should be able to create message with blobs');
+    }
+  }
+
   {
     await t.throwsAsync(
       createCopilotMessage(app, randomUUID()),

packages/backend/server/src/__tests__/utils/copilot.ts

@@ -492,19 +492,53 @@ export async function createCopilotMessage(
   sessionId: string,
   content?: string,
   attachments?: string[],
-  blobs?: ArrayBuffer[],
+  blobs?: File[],
   params?: Record<string, string>
 ): Promise<string> {
-  const res = await app.gql(
-    `
-    mutation createCopilotMessage($options: CreateChatMessageInput!) {
-      createCopilotMessage(options: $options)
+  let resp = app
+    .POST('/graphql')
+    .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
+    .field(
+      'operations',
+      JSON.stringify({
+        query: `
+          mutation createCopilotMessage($options: CreateChatMessageInput!) {
+            createCopilotMessage(options: $options)
+          }
+        `,
+        variables: {
+          options: { sessionId, content, attachments, blobs: [], params },
+        },
+      })
+    )
+    .field(
+      'map',
+      JSON.stringify(
+        Array.from<any>({ length: blobs?.length ?? 0 }).reduce(
+          (acc, _, idx) => {
+            acc[idx.toString()] = [`variables.options.blobs.${idx}`];
+            return acc;
+          },
+          {}
+        )
+      )
+    );
+  if (blobs && blobs.length) {
+    for (const [idx, file] of blobs.entries()) {
+      resp = resp.attach(
+        idx.toString(),
+        Buffer.from(await file.arrayBuffer()),
+        {
+          filename: file.name || `file${idx}`,
+          contentType: file.type || 'application/octet-stream',
+        }
+      );
     }
-  `,
-    { options: { sessionId, content, attachments, blobs, params } }
-  );
+  }
 
-  return res.createCopilotMessage;
+  const res = await resp.expect(200);
+
+  return res.body.data.createCopilotMessage;
 }
 
 export async function chatWithText(

packages/backend/server/src/plugins/copilot/providers/fal.ts

@@ -138,7 +138,15 @@ export class FalProvider
     );
     return {
       model_name: options.modelName || undefined,
-      image_url: attachments?.[0],
+      image_url: attachments
+        ?.map(v =>
+          typeof v === 'string'
+            ? v
+            : v.mimeType.startsWith('image/')
+              ? v.attachment
+              : undefined
+        )
+        .filter(v => !!v)[0],
       prompt: content.trim(),
       loras: lora.length ? lora : undefined,
       controlnets: controlnets.length ? controlnets : undefined,

packages/backend/server/src/plugins/copilot/providers/types.ts

@@ -52,7 +52,15 @@ export const ChatMessageRole = Object.values(AiPromptRole) as [
 
 export const PureMessageSchema = z.object({
   content: z.string(),
-  attachments: z.array(z.string()).optional().nullable(),
+  attachments: z
+    .array(
+      z.union([
+        z.string(),
+        z.object({ attachment: z.string(), mimeType: z.string() }),
+      ])
+    )
+    .optional()
+    .nullable(),
   params: z.record(z.any()).optional().nullable(),
 });
 

packages/backend/server/src/plugins/copilot/providers/utils.ts

@@ -35,15 +35,26 @@ const FORMAT_INFER_MAP: Record<string, string> = {
   flv: 'video/flv',
 };
 
-function inferMimeType(url: string) {
+async function inferMimeType(url: string) {
   if (url.startsWith('data:')) {
     return url.split(';')[0].split(':')[1];
   }
-  const extension = url.split('.').pop();
+  const pathname = new URL(url).pathname;
+  const extension = pathname.split('.').pop();
   if (extension) {
-    return FORMAT_INFER_MAP[extension];
+    const ext = FORMAT_INFER_MAP[extension];
+    if (ext) {
+      return ext;
+    }
+    const mimeType = await fetch(url, {
+      method: 'HEAD',
+      redirect: 'follow',
+    }).then(res => res.headers.get('Content-Type'));
+    if (mimeType) {
+      return mimeType;
+    }
   }
-  return undefined;
+  return 'application/octet-stream';
 }
 
 export async function chatToGPTMessage(
@@ -66,19 +77,24 @@ export async function chatToGPTMessage(
         contents.push({ type: 'text', text: content });
       }
 
-      for (const url of attachments) {
-        if (SIMPLE_IMAGE_URL_REGEX.test(url)) {
-          const mimeType =
-            typeof mimetype === 'string' ? mimetype : inferMimeType(url);
-          if (mimeType) {
-            if (mimeType.startsWith('image/')) {
-              contents.push({ type: 'image', image: url, mimeType });
-            } else {
-              const data = url.startsWith('data:')
-                ? await fetch(url).then(r => r.arrayBuffer())
-                : new URL(url);
-              contents.push({ type: 'file' as const, data, mimeType });
-            }
+      for (let attachment of attachments) {
+        let mimeType: string;
+        if (typeof attachment === 'string') {
+          mimeType =
+            typeof mimetype === 'string'
+              ? mimetype
+              : await inferMimeType(attachment);
+        } else {
+          ({ attachment, mimeType } = attachment);
+        }
+        if (SIMPLE_IMAGE_URL_REGEX.test(attachment)) {
+          if (mimeType.startsWith('image/')) {
+            contents.push({ type: 'image', image: attachment, mimeType });
+          } else {
+            const data = attachment.startsWith('data:')
+              ? await fetch(attachment).then(r => r.arrayBuffer())
+              : new URL(attachment);
+            contents.push({ type: 'file' as const, data, mimeType });
           }
         }
       }

packages/backend/server/src/plugins/copilot/resolver.ts

@@ -34,6 +34,7 @@ import { Admin } from '../../core/common';
 import { AccessController } from '../../core/permission';
 import { UserType } from '../../core/user';
 import { PromptService } from './prompt';
+import { PromptMessage } from './providers';
 import { ChatSessionService } from './session';
 import { CopilotStorage } from './storage';
 import {
@@ -113,7 +114,7 @@ class CreateChatMessageInput implements Omit<SubmittedMessage, 'content'> {
   @Field(() => String, { nullable: true })
   content!: string | undefined;
 
-  @Field(() => [String], { nullable: true })
+  @Field(() => [String], { nullable: true, deprecationReason: 'use blobs' })
   attachments!: string[] | undefined;
 
   @Field(() => [GraphQLUpload], { nullable: true })
@@ -527,8 +528,8 @@ export class CopilotResolver {
       throw new BadRequestException('Session not found');
     }
 
+    const attachments: PromptMessage['attachments'] = options.attachments || [];
     if (options.blobs) {
-      options.attachments = options.attachments || [];
       const { workspaceId } = session.config;
 
       const blobs = await Promise.all(options.blobs);
@@ -539,18 +540,18 @@ export class CopilotResolver {
         const filename = createHash('sha256')
           .update(uploaded.buffer)
           .digest('base64url');
-        const link = await this.storage.put(
+        const attachment = await this.storage.put(
           user.id,
           workspaceId,
           filename,
           uploaded.buffer
         );
-        options.attachments.push(link);
+        attachments.push({ attachment, mimeType: blob.mimetype });
       }
     }
 
     try {
-      return await this.chatSession.createMessage(options);
+      return await this.chatSession.createMessage({ ...options, attachments });
     } catch (e: any) {
       throw new CopilotFailedToCreateMessage(e.message);
     }

packages/backend/server/src/plugins/copilot/session.ts

@@ -166,7 +166,11 @@ export class ChatSession implements AsyncDisposable {
         firstMessage.attachments || [],
       ]
         .flat()
-        .filter(v => !!v?.trim());
+        .filter(v =>
+          typeof v === 'string'
+            ? !!v.trim()
+            : v && v.attachment.trim() && v.mimeType
+        );
 
       return finished;
     }
@@ -553,7 +557,12 @@ export class ChatSessionService {
                     action: prompt.action || null,
                     tokens: tokenCost,
                     createdAt,
-                    messages: preload.concat(ret.data),
+                    messages: preload.concat(ret.data).map(m => ({
+                      ...m,
+                      attachments: m.attachments
+                        ?.map(a => (typeof a === 'string' ? a : a.attachment))
+                        .filter(a => !!a),
+                    })),
                   };
                 } else {
                   this.logger.error(