feat: adopt createMessage upload api (#6596)

packages/backend/server/src/plugins/copilot/resolver.ts

@@ -64,7 +64,7 @@ class CreateChatMessageInput implements Omit<SubmittedMessage, 'content'> {
   attachments!: string[] | undefined;
 
   @Field(() => [GraphQLUpload], { nullable: true })
-  blobs!: FileUpload[] | undefined;
+  blobs!: Promise<FileUpload>[] | undefined;
 
   @Field(() => GraphQLJSON, { nullable: true })
   params!: Record<string, string> | undefined;
@@ -277,7 +277,7 @@ export class CopilotResolver {
       options.attachments = options.attachments || [];
       const { workspaceId } = session.config;
 
-      for (const blob of options.blobs) {
+      for (const blob of await Promise.all(options.blobs)) {
         const uploaded = await this.storage.handleUpload(user.id, blob);
         const link = await this.storage.put(
           user.id,
@@ -286,6 +286,7 @@ export class CopilotResolver {
           uploaded.buffer
         );
         options.attachments.push(link);
+        delete options.blobs;
       }
     }
 

packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts

@@ -263,7 +263,7 @@ export function setupAIProvider() {
     return toImage({
       ...options,
       promptName,
-      forceToImage: true,
+      forceCreate: true,
     });
   });
 

packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts

@@ -1,4 +1,5 @@
 import { toTextStream } from '@blocksuite/presets';
+import { partition } from 'lodash-es';
 
 import { CopilotClient } from './copilot-client';
 import type { PromptKey } from './prompt';
@@ -7,19 +8,14 @@ const TIMEOUT = 50000;
 
 const client = new CopilotClient();
 
-function readBlobAsURL(blob: Blob | File) {
-  return new Promise<string>((resolve, reject) => {
-    const reader = new FileReader();
-    reader.onload = e => {
-      if (typeof e.target?.result === 'string') {
-        resolve(e.target.result);
-      } else {
-        reject();
-      }
-    };
-    reader.onerror = reject;
-    reader.readAsDataURL(blob);
-  });
+async function calculateBlobHash(blob: Blob) {
+  const buffer = await blob.arrayBuffer();
+  const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
+  const hashArray = Array.from(new Uint8Array(hashBuffer));
+  return hashArray
+    .map(byte => byte.toString(16).padStart(2, '0'))
+    .join('')
+    .slice(0, 32);
 }
 
 export type TextToTextOptions = {
@@ -32,7 +28,7 @@ export type TextToTextOptions = {
   params?: Record<string, string>;
   timeout?: number;
   stream?: boolean;
-  forceToImage?: boolean; // force to image
+  forceCreate?: boolean; // force to create a message
 };
 
 export function createChatSession({
@@ -57,7 +53,7 @@ async function createSessionMessage({
   sessionId: providedSessionId,
   attachments,
   params,
-  forceToImage,
+  forceCreate,
 }: TextToTextOptions) {
   if (!promptName && !providedSessionId) {
     throw new Error('promptName or sessionId is required');
@@ -70,28 +66,27 @@ async function createSessionMessage({
       promptName: promptName as string,
     }));
 
-  if (forceToImage || hasAttachments) {
-    const options = {
+  if (forceCreate || hasAttachments) {
+    const options: Parameters<CopilotClient['createMessage']>[0] = {
       sessionId,
       content,
       params,
-    } as {
-      sessionId: string;
-      content?: string;
-      params?: Record<string, string>;
-      attachments?: string[];
     };
     if (hasAttachments) {
-      const normalizedAttachments = await Promise.all(
-        attachments.map(async attachment => {
-          if (typeof attachment === 'string') {
-            return attachment;
+      const [stringAttachments, blobs] = partition(
+        attachments,
+        attachment => typeof attachment === 'string'
+      ) as [string[], (Blob | File)[]];
+      options.attachments = stringAttachments;
+      options.blobs = await Promise.all(
+        blobs.map(async blob => {
+          if (blob instanceof File) {
+            return blob;
+          } else {
+            return new File([blob], await calculateBlobHash(blob));
           }
-          const url = await readBlobAsURL(attachment);
-          return url;
         })
       );
-      options.attachments = normalizedAttachments;
     }
     const messageId = await client.createMessage(options);
     return {
@@ -180,7 +175,7 @@ export function toImage({
   content,
   attachments,
   params,
-  forceToImage,
+  forceCreate,
   timeout = TIMEOUT,
 }: TextToTextOptions) {
   return {
@@ -192,7 +187,7 @@ export function toImage({
         content,
         attachments,
         params,
-        forceToImage,
+        forceCreate,
       });
 
       const eventSource = client.imagesStream(

packages/frontend/graphql/export-gql-plugin.cjs

@@ -23,7 +23,7 @@ function getExportedName(def) {
  * @type {import('@graphql-codegen/plugin-helpers').CodegenPlugin}
  */
 module.exports = {
-  plugin: (_schema, documents, { output }) => {
+  plugin: (schema, documents, { output }) => {
     const nameLocationMap = new Map();
     const locationSourceMap = new Map(
       documents
@@ -133,12 +133,24 @@ module.exports = {
               const { variableDefinitions } = def;
               if (variableDefinitions) {
                 return variableDefinitions.some(variableDefinition => {
-                  if (
-                    variableDefinition?.type?.type?.name?.value === 'Upload'
-                  ) {
-                    return true;
-                  }
-                  return false;
+                  const varType = variableDefinition?.type?.type?.name?.value;
+                  const checkContainFile = type => {
+                    if (schema.getType(type)?.name === 'Upload') return true;
+                    const typeDef = schema.getType(type);
+                    const fields = typeDef.getFields?.();
+                    if (!fields || !fields) return false;
+                    for (let field of Object.values(fields)) {
+                      let type = field.type;
+                      while (type.ofType) {
+                        type = type.ofType;
+                      }
+                      if (type.name === 'Upload') {
+                        return true;
+                      }
+                    }
+                    return false;
+                  };
+                  return varType ? checkContainFile(varType) : false;
                 });
               } else {
                 return false;

packages/frontend/graphql/src/fetcher.ts

@@ -115,17 +115,26 @@ export function transformToForm(body: RequestBody) {
   if (body.operationName) {
     gqlBody.name = body.operationName;
   }
-  const map: Record<string, [string]> = {};
+  const map: Record<string, string[]> = {};
   const files: File[] = [];
   if (body.variables) {
     let i = 0;
-    Object.entries(body.variables).forEach(([key, value]) => {
+    const buildMap = (key: string, value: any) => {
       if (value instanceof File) {
-        map['0'] = [`variables.${key}`];
+        map['' + i] = [key];
         files[i] = value;
         i++;
+      } else if (Array.isArray(value)) {
+        value.forEach((v, index) => {
+          buildMap(`${key}.${index}`, v);
+        });
+      } else if (isObject(value)) {
+        Object.entries(value).forEach(([k, v]) => {
+          buildMap(`${key}.${k}`, v);
+        });
       }
-    });
+    };
+    buildMap('variables', body.variables);
   }
 
   form.set('operations', JSON.stringify(gqlBody));

packages/frontend/graphql/src/graphql/index.ts

@@ -109,7 +109,7 @@ export const createCopilotMessageMutation = {
   id: 'createCopilotMessageMutation' as const,
   operationName: 'createCopilotMessage',
   definitionName: 'createCopilotMessage',
-  containsFile: false,
+  containsFile: true,
   query: `
 mutation createCopilotMessage($options: CreateChatMessageInput!) {
   createCopilotMessage(options: $options)