diff --git a/packages/backend/server/src/__tests__/copilot.e2e.ts b/packages/backend/server/src/__tests__/copilot.e2e.ts index a340b88efd..04d507eee5 100644 --- a/packages/backend/server/src/__tests__/copilot.e2e.ts +++ b/packages/backend/server/src/__tests__/copilot.e2e.ts @@ -385,6 +385,45 @@ test('should create message correctly', async t => { t.truthy(messageId, 'should be able to create message with valid session'); } + { + // with attachment url + { + const { id } = await createWorkspace(app); + const sessionId = await createCopilotSession( + app, + id, + randomUUID(), + promptName + ); + const messageId = await createCopilotMessage(app, sessionId, undefined, [ + 'http://example.com/cat.jpg', + ]); + t.truthy(messageId, 'should be able to create message with url link'); + } + + // with attachment + { + const { id } = await createWorkspace(app); + const sessionId = await createCopilotSession( + app, + id, + randomUUID(), + promptName + ); + const smallestPng = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII'; + const pngData = await fetch(smallestPng).then(res => res.arrayBuffer()); + const messageId = await createCopilotMessage( + app, + sessionId, + undefined, + undefined, + [new File([new Uint8Array(pngData)], '1.png', { type: 'image/png' })] + ); + t.truthy(messageId, 'should be able to create message with blobs'); + } + } + { await t.throwsAsync( createCopilotMessage(app, randomUUID()), diff --git a/packages/backend/server/src/__tests__/utils/copilot.ts b/packages/backend/server/src/__tests__/utils/copilot.ts index 4d12a1939b..8ddfc631a2 100644 --- a/packages/backend/server/src/__tests__/utils/copilot.ts +++ b/packages/backend/server/src/__tests__/utils/copilot.ts @@ -490,19 +490,53 @@ export async function createCopilotMessage( sessionId: string, content?: string, attachments?: string[], - blobs?: ArrayBuffer[], + blobs?: File[], params?: Record ): Promise { - const res = await app.gql( - ` - mutation createCopilotMessage($options: CreateChatMessageInput!) { - createCopilotMessage(options: $options) + let resp = app + .POST('/graphql') + .set({ 'x-request-id': 'test', 'x-operation-name': 'test' }) + .field( + 'operations', + JSON.stringify({ + query: ` + mutation createCopilotMessage($options: CreateChatMessageInput!) { + createCopilotMessage(options: $options) + } + `, + variables: { + options: { sessionId, content, attachments, blobs: [], params }, + }, + }) + ) + .field( + 'map', + JSON.stringify( + Array.from({ length: blobs?.length ?? 0 }).reduce( + (acc, _, idx) => { + acc[idx.toString()] = [`variables.options.blobs.${idx}`]; + return acc; + }, + {} + ) + ) + ); + if (blobs && blobs.length) { + for (const [idx, file] of blobs.entries()) { + resp = resp.attach( + idx.toString(), + Buffer.from(await file.arrayBuffer()), + { + filename: file.name || `file${idx}`, + contentType: file.type || 'application/octet-stream', + } + ); } - `, - { options: { sessionId, content, attachments, blobs, params } } - ); + } - return res.createCopilotMessage; + const res = await resp.expect(200); + + return res.body.data.createCopilotMessage; } export async function chatWithText( diff --git a/packages/backend/server/src/plugins/copilot/providers/fal.ts b/packages/backend/server/src/plugins/copilot/providers/fal.ts index 238c3f9a19..6fe50e98e6 100644 --- a/packages/backend/server/src/plugins/copilot/providers/fal.ts +++ b/packages/backend/server/src/plugins/copilot/providers/fal.ts @@ -138,7 +138,15 @@ export class FalProvider ); return { model_name: options.modelName || undefined, - image_url: attachments?.[0], + image_url: attachments + ?.map(v => + typeof v === 'string' + ? v + : v.mimeType.startsWith('image/') + ? v.attachment + : undefined + ) + .filter(v => !!v)[0], prompt: content.trim(), loras: lora.length ? lora : undefined, controlnets: controlnets.length ? controlnets : undefined, diff --git a/packages/backend/server/src/plugins/copilot/providers/types.ts b/packages/backend/server/src/plugins/copilot/providers/types.ts index 786e1413f4..08fae71e8f 100644 --- a/packages/backend/server/src/plugins/copilot/providers/types.ts +++ b/packages/backend/server/src/plugins/copilot/providers/types.ts @@ -51,7 +51,15 @@ export const ChatMessageRole = Object.values(AiPromptRole) as [ export const PureMessageSchema = z.object({ content: z.string(), - attachments: z.array(z.string()).optional().nullable(), + attachments: z + .array( + z.union([ + z.string(), + z.object({ attachment: z.string(), mimeType: z.string() }), + ]) + ) + .optional() + .nullable(), params: z.record(z.any()).optional().nullable(), }); diff --git a/packages/backend/server/src/plugins/copilot/providers/utils.ts b/packages/backend/server/src/plugins/copilot/providers/utils.ts index 7d12bac50f..4e96fb6d54 100644 --- a/packages/backend/server/src/plugins/copilot/providers/utils.ts +++ b/packages/backend/server/src/plugins/copilot/providers/utils.ts @@ -35,15 +35,26 @@ const FORMAT_INFER_MAP: Record = { flv: 'video/flv', }; -function inferMimeType(url: string) { +async function inferMimeType(url: string) { if (url.startsWith('data:')) { return url.split(';')[0].split(':')[1]; } - const extension = url.split('.').pop(); + const pathname = new URL(url).pathname; + const extension = pathname.split('.').pop(); if (extension) { - return FORMAT_INFER_MAP[extension]; + const ext = FORMAT_INFER_MAP[extension]; + if (ext) { + return ext; + } + const mimeType = await fetch(url, { + method: 'HEAD', + redirect: 'follow', + }).then(res => res.headers.get('Content-Type')); + if (mimeType) { + return mimeType; + } } - return undefined; + return 'application/octet-stream'; } export async function chatToGPTMessage( @@ -66,19 +77,24 @@ export async function chatToGPTMessage( contents.push({ type: 'text', text: content }); } - for (const url of attachments) { - if (SIMPLE_IMAGE_URL_REGEX.test(url)) { - const mimeType = - typeof mimetype === 'string' ? mimetype : inferMimeType(url); - if (mimeType) { - if (mimeType.startsWith('image/')) { - contents.push({ type: 'image', image: url, mimeType }); - } else { - const data = url.startsWith('data:') - ? await fetch(url).then(r => r.arrayBuffer()) - : new URL(url); - contents.push({ type: 'file' as const, data, mimeType }); - } + for (let attachment of attachments) { + let mimeType: string; + if (typeof attachment === 'string') { + mimeType = + typeof mimetype === 'string' + ? mimetype + : await inferMimeType(attachment); + } else { + ({ attachment, mimeType } = attachment); + } + if (SIMPLE_IMAGE_URL_REGEX.test(attachment)) { + if (mimeType.startsWith('image/')) { + contents.push({ type: 'image', image: attachment, mimeType }); + } else { + const data = attachment.startsWith('data:') + ? await fetch(attachment).then(r => r.arrayBuffer()) + : new URL(attachment); + contents.push({ type: 'file' as const, data, mimeType }); } } } diff --git a/packages/backend/server/src/plugins/copilot/resolver.ts b/packages/backend/server/src/plugins/copilot/resolver.ts index 345aefaa65..1540024517 100644 --- a/packages/backend/server/src/plugins/copilot/resolver.ts +++ b/packages/backend/server/src/plugins/copilot/resolver.ts @@ -34,6 +34,7 @@ import { Admin } from '../../core/common'; import { AccessController } from '../../core/permission'; import { UserType } from '../../core/user'; import { PromptService } from './prompt'; +import { PromptMessage } from './providers'; import { ChatSessionService } from './session'; import { CopilotStorage } from './storage'; import { @@ -113,7 +114,7 @@ class CreateChatMessageInput implements Omit { @Field(() => String, { nullable: true }) content!: string | undefined; - @Field(() => [String], { nullable: true }) + @Field(() => [String], { nullable: true, deprecationReason: 'use blobs' }) attachments!: string[] | undefined; @Field(() => [GraphQLUpload], { nullable: true }) @@ -527,8 +528,8 @@ export class CopilotResolver { throw new BadRequestException('Session not found'); } + const attachments: PromptMessage['attachments'] = options.attachments || []; if (options.blobs) { - options.attachments = options.attachments || []; const { workspaceId } = session.config; const blobs = await Promise.all(options.blobs); @@ -539,18 +540,18 @@ export class CopilotResolver { const filename = createHash('sha256') .update(uploaded.buffer) .digest('base64url'); - const link = await this.storage.put( + const attachment = await this.storage.put( user.id, workspaceId, filename, uploaded.buffer ); - options.attachments.push(link); + attachments.push({ attachment, mimeType: blob.mimetype }); } } try { - return await this.chatSession.createMessage(options); + return await this.chatSession.createMessage({ ...options, attachments }); } catch (e: any) { throw new CopilotFailedToCreateMessage(e.message); } diff --git a/packages/backend/server/src/plugins/copilot/session.ts b/packages/backend/server/src/plugins/copilot/session.ts index 60ec974b03..ca71a41b12 100644 --- a/packages/backend/server/src/plugins/copilot/session.ts +++ b/packages/backend/server/src/plugins/copilot/session.ts @@ -166,7 +166,11 @@ export class ChatSession implements AsyncDisposable { firstMessage.attachments || [], ] .flat() - .filter(v => !!v?.trim()); + .filter(v => + typeof v === 'string' + ? !!v.trim() + : v && v.attachment.trim() && v.mimeType + ); return finished; } @@ -553,7 +557,12 @@ export class ChatSessionService { action: prompt.action || null, tokens: tokenCost, createdAt, - messages: preload.concat(ret.data), + messages: preload.concat(ret.data).map(m => ({ + ...m, + attachments: m.attachments + ?.map(a => (typeof a === 'string' ? a : a.attachment)) + .filter(a => !!a), + })), }; } else { this.logger.error(