test(server): improve server copilot test (#11596)

packages/backend/server/src/__tests__/copilot-provider.spec.ts

@@ -1,6 +1,5 @@
 import type { ExecutionContext, TestFn } from 'ava';
 import ava from 'ava';
-import { z } from 'zod';
 
 import { ServerFeature, ServerService } from '../core';
 import { AuthService } from '../core/auth';
@@ -8,6 +7,7 @@ import { QuotaModule } from '../core/quota';
 import { CopilotModule } from '../plugins/copilot';
 import { prompts, PromptService } from '../plugins/copilot/prompt';
 import { CopilotProviderFactory } from '../plugins/copilot/providers';
+import { TranscriptionResponseSchema } from '../plugins/copilot/transcript/types';
 import {
   CopilotChatTextExecutor,
   CopilotWorkflowService,
@@ -315,6 +315,7 @@ const actions = [
     type: 'text' as const,
   },
   {
+    name: 'Should transcribe short audio',
     promptName: ['Transcript audio'],
     messages: [
       {
@@ -323,23 +324,58 @@ const actions = [
         attachments: [
           'https://cdn.affine.pro/copilot-test/MP9qDGuYgnY+ILoEAmHpp3h9Npuw2403EAYMEA.mp3',
         ],
+        params: {
+          schema: TranscriptionResponseSchema,
+        },
       },
     ],
     verifier: (t: ExecutionContext<Tester>, result: string) => {
-      // cleanup json markdown wrap
-      const cleaned = result
-        .replace(/```[\w\s]+\n/g, '')
-        .replace(/\n```/g, '')
-        .trim();
       t.notThrows(() => {
-        z.object({
-          a: z.string(),
-          s: z.number(),
-          e: z.number(),
-          t: z.string(),
-        })
-          .array()
-          .parse(JSON.parse(cleaned));
+        TranscriptionResponseSchema.parse(JSON.parse(result));
+      });
+    },
+    type: 'text' as const,
+  },
+  {
+    name: 'Should transcribe middle audio',
+    promptName: ['Transcript audio'],
+    messages: [
+      {
+        role: 'user' as const,
+        content: '',
+        attachments: [
+          'https://cdn.affine.pro/copilot-test/2ed05eo1KvZ2tWB_BAjFo67EAPZZY-w4LylUAw.m4a',
+        ],
+        params: {
+          schema: TranscriptionResponseSchema,
+        },
+      },
+    ],
+    verifier: (t: ExecutionContext<Tester>, result: string) => {
+      t.notThrows(() => {
+        TranscriptionResponseSchema.parse(JSON.parse(result));
+      });
+    },
+    type: 'text' as const,
+  },
+  {
+    name: 'Should transcribe long audio',
+    promptName: ['Transcript audio'],
+    messages: [
+      {
+        role: 'user' as const,
+        content: '',
+        attachments: [
+          'https://cdn.affine.pro/copilot-test/nC9-e7P85PPI2rU29QWwf8slBNRMy92teLIIMw.opus',
+        ],
+        params: {
+          schema: TranscriptionResponseSchema,
+        },
+      },
+    ],
+    verifier: (t: ExecutionContext<Tester>, result: string) => {
+      t.notThrows(() => {
+        TranscriptionResponseSchema.parse(JSON.parse(result));
       });
     },
     type: 'text' as const,
@@ -516,7 +552,8 @@ for (const { name, promptName, messages, verifier, type } of actions) {
                 ),
                 ...messages,
               ],
-              prompt.model
+              prompt.model,
+              Object.assign({}, prompt.config)
             );
             t.truthy(result, 'should return result');
             verifier?.(t, result);

packages/backend/server/src/plugins/copilot/providers/openai.ts

@@ -180,7 +180,6 @@ export class OpenAIProvider
     options: CopilotChatOptions = {}
   ): Promise<string> {
     await this.checkParams({ messages, model, options });
-    console.log('messages', messages);
 
     try {
       metrics.ai.counter('chat_text_calls').add(1, { model });
@@ -215,7 +214,6 @@ export class OpenAIProvider
 
       return text.trim();
     } catch (e: any) {
-      console.log('error', e);
       metrics.ai.counter('chat_text_errors').add(1, { model });
       throw this.handleError(e, model, options);
     }

packages/backend/server/src/plugins/copilot/providers/utils.ts

@@ -14,6 +14,12 @@ const SIMPLE_IMAGE_URL_REGEX = /^(https?:\/\/|data:image\/)/;
 const FORMAT_INFER_MAP: Record<string, string> = {
   pdf: 'application/pdf',
   mp3: 'audio/mpeg',
+  opus: 'audio/opus',
+  ogg: 'audio/ogg',
+  aac: 'audio/aac',
+  m4a: 'audio/aac',
+  flac: 'audio/flac',
+  ogv: 'video/ogg',
   wav: 'audio/wav',
   png: 'image/png',
   jpeg: 'image/jpeg',
@@ -57,10 +63,7 @@ export async function chatToGPTMessage(
     if (Array.isArray(attachments)) {
       const contents: (TextPart | ImagePart | FilePart)[] = [];
       if (content.length) {
-        contents.push({
-          type: 'text',
-          text: content,
-        });
+        contents.push({ type: 'text', text: content });
       }
 
       for (const url of attachments) {
@@ -69,20 +72,12 @@ export async function chatToGPTMessage(
             typeof mimetype === 'string' ? mimetype : inferMimeType(url);
           if (mimeType) {
             if (mimeType.startsWith('image/')) {
-              contents.push({
-                type: 'image',
-                image: url,
-                mimeType,
-              });
+              contents.push({ type: 'image', image: url, mimeType });
             } else {
               const data = url.startsWith('data:')
                 ? await fetch(url).then(r => r.arrayBuffer())
                 : new URL(url);
-              contents.push({
-                type: 'file' as const,
-                data,
-                mimeType,
-              });
+              contents.push({ type: 'file' as const, data, mimeType });
             }
           }
         }