feat(server): support sliced audio (#11562)

fix AF-2479

packages/backend/server/src/__tests__/__snapshots__/copilot.e2e.ts.md

@@ -88,3 +88,47 @@ Generated by [AVA](https://avajs.dev).
         ],
       },
     ]
+
+> should submit audio transcription job
+
+    [
+      {
+        status: 'running',
+      },
+    ]
+
+> should claim audio transcription job
+
+    [
+      {
+        status: 'claimed',
+        summary: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]',
+        title: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]',
+        transcription: [
+          {
+            end: '00:00:45',
+            speaker: 'A',
+            start: '00:00:30',
+            transcription: 'Hello, everyone.',
+          },
+          {
+            end: '00:01:10',
+            speaker: 'B',
+            start: '00:00:46',
+            transcription: 'Hi, thank you for joining the meeting today.',
+          },
+          {
+            end: '00:10:45',
+            speaker: 'A',
+            start: '00:10:30',
+            transcription: 'Hello, everyone.',
+          },
+          {
+            end: '00:11:10',
+            speaker: 'B',
+            start: '00:10:46',
+            transcription: 'Hi, thank you for joining the meeting today.',
+          },
+        ],
+      },
+    ]

packages/backend/server/src/__tests__/copilot.e2e.ts

@@ -883,34 +883,67 @@ test('should be able to transcript', async t => {
     '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]'
   );
 
-  const job = await submitAudioTranscription(
-    app,
-    workspaceId,
-    'blobId',
-    'test.mp3',
-    Buffer.from([1, 1])
-  );
-  t.snapshot(
-    cleanObject([job], ['id']),
-    'should submit audio transcription job'
-  );
-  t.truthy(job.id, 'should have job id');
-
-  // wait for processing
   {
-    let { status } = (await audioTranscription(app, workspaceId, job.id)) || {};
+    const job = await submitAudioTranscription(app, workspaceId, '1', '1.mp3', [
+      Buffer.from([1, 1]),
+    ]);
+    t.snapshot(
+      cleanObject([job], ['id']),
+      'should submit audio transcription job'
+    );
+    t.truthy(job.id, 'should have job id');
 
-    while (status !== 'finished') {
-      await new Promise(resolve => setTimeout(resolve, 1000));
-      ({ status } = (await audioTranscription(app, workspaceId, job.id)) || {});
+    // wait for processing
+    {
+      let { status } =
+        (await audioTranscription(app, workspaceId, job.id)) || {};
+
+      while (status !== 'finished') {
+        await new Promise(resolve => setTimeout(resolve, 1000));
+        ({ status } =
+          (await audioTranscription(app, workspaceId, job.id)) || {});
+      }
+    }
+
+    {
+      const result = await claimAudioTranscription(app, job.id);
+      t.snapshot(
+        cleanObject([result], ['id']),
+        'should claim audio transcription job'
+      );
     }
   }
 
   {
-    const result = await claimAudioTranscription(app, job.id);
+    // sliced audio
+    const job = await submitAudioTranscription(app, workspaceId, '2', '2.mp3', [
+      Buffer.from([1, 1]),
+      Buffer.from([1, 2]),
+    ]);
     t.snapshot(
-      cleanObject([result], ['id']),
-      'should claim audio transcription job'
+      cleanObject([job], ['id']),
+      'should submit audio transcription job'
     );
+    t.truthy(job.id, 'should have job id');
+
+    // wait for processing
+    {
+      let { status } =
+        (await audioTranscription(app, workspaceId, job.id)) || {};
+
+      while (status !== 'finished') {
+        await new Promise(resolve => setTimeout(resolve, 1000));
+        ({ status } =
+          (await audioTranscription(app, workspaceId, job.id)) || {});
+      }
+    }
+
+    {
+      const result = await claimAudioTranscription(app, job.id);
+      t.snapshot(
+        cleanObject([result], ['id']),
+        'should claim audio transcription job'
+      );
+    }
   }
 });

packages/backend/server/src/__tests__/utils/copilot.ts

@@ -335,17 +335,17 @@ export async function submitAudioTranscription(
   workspaceId: string,
   blobId: string,
   fileName: string,
-  content: Buffer
+  content: Buffer[]
 ): Promise<{ id: string; status: string }> {
-  const res = await app
+  let resp = app
     .POST('/graphql')
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .field(
       'operations',
       JSON.stringify({
         query: `
-          mutation submitAudioTranscription($blob: Upload!, $blobId: String!, $workspaceId: String!) {
-            submitAudioTranscription(blob: $blob, blobId: $blobId, workspaceId: $workspaceId) {
+          mutation submitAudioTranscription($blob: Upload, $blobs: [Upload!], $blobId: String!, $workspaceId: String!) {
+            submitAudioTranscription(blob: $blob, blobs: $blobs, blobId: $blobId, workspaceId: $workspaceId) {
               id
               status
             }
@@ -353,17 +353,29 @@ export async function submitAudioTranscription(
         `,
         variables: {
           blob: null,
+          blobs: [],
           blobId,
           workspaceId,
         },
       })
     )
-    .field('map', JSON.stringify({ '0': ['variables.blob'] }))
-    .attach('0', content, {
+    .field(
+      'map',
+      JSON.stringify(
+        Array.from<any>({ length: content.length }).reduce((acc, _, idx) => {
+          acc[idx.toString()] = [`variables.blobs.${idx}`];
+          return acc;
+        }, {})
+      )
+    );
+  for (const [idx, buffer] of content.entries()) {
+    resp = resp.attach(idx.toString(), buffer, {
       filename: fileName,
       contentType: 'application/octet-stream',
-    })
-    .expect(200);
+    });
+  }
+
+  const res = await resp.expect(200);
 
   return res.body.data.submitAudioTranscription;
 }

packages/backend/server/src/base/error/def.ts

@@ -709,6 +709,10 @@ export const USER_FRIENDLY_ERRORS = {
     type: 'bad_request',
     message: () => `Transcription job not found.`,
   },
+  copilot_transcription_audio_not_provided: {
+    type: 'bad_request',
+    message: () => `Audio not provided.`,
+  },
 
   // Quota & Limit errors
   blob_quota_exceeded: {

packages/backend/server/src/base/error/errors.gen.ts

@@ -771,6 +771,12 @@ export class CopilotTranscriptionJobNotFound extends UserFriendlyError {
   }
 }
 
+export class CopilotTranscriptionAudioNotProvided extends UserFriendlyError {
+  constructor(message?: string) {
+    super('bad_request', 'copilot_transcription_audio_not_provided', message);
+  }
+}
+
 export class BlobQuotaExceeded extends UserFriendlyError {
   constructor(message?: string) {
     super('quota_exceeded', 'blob_quota_exceeded', message);
@@ -1027,6 +1033,7 @@ export enum ErrorNames {
   COPILOT_EMBEDDING_UNAVAILABLE,
   COPILOT_TRANSCRIPTION_JOB_EXISTS,
   COPILOT_TRANSCRIPTION_JOB_NOT_FOUND,
+  COPILOT_TRANSCRIPTION_AUDIO_NOT_PROVIDED,
   BLOB_QUOTA_EXCEEDED,
   STORAGE_QUOTA_EXCEEDED,
   MEMBER_QUOTA_EXCEEDED,

packages/backend/server/src/plugins/copilot/transcript/resolver.ts

@@ -14,6 +14,7 @@ import { AiJobStatus } from '@prisma/client';
 import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';
 
 import {
+  CopilotTranscriptionAudioNotProvided,
   CopilotTranscriptionJobNotFound,
   type FileUpload,
 } from '../../../base';
@@ -100,20 +101,27 @@ export class CopilotTranscriptionResolver {
     @CurrentUser() user: CurrentUser,
     @Args('workspaceId') workspaceId: string,
     @Args('blobId') blobId: string,
-    @Args({ name: 'blob', type: () => GraphQLUpload })
-    blob: FileUpload
+    @Args({ name: 'blob', type: () => GraphQLUpload, nullable: true })
+    blob: FileUpload | null,
+    @Args({ name: 'blobs', type: () => [GraphQLUpload], nullable: true })
+    blobs: FileUpload[] | null
   ): Promise<TranscriptionResultType | null> {
     await this.ac
       .user(user.id)
       .workspace(workspaceId)
       .allowLocal()
       .assert('Workspace.Copilot');
+    // merge blobs
+    const allBlobs = blob ? [blob, ...(blobs || [])].filter(v => !!v) : blobs;
+    if (!allBlobs || allBlobs.length === 0) {
+      throw new CopilotTranscriptionAudioNotProvided();
+    }
 
     const jobResult = await this.service.submitTranscriptionJob(
       user.id,
       workspaceId,
       blobId,
-      blob
+      await Promise.all(allBlobs)
     );
 
     return this.handleJobResult(jobResult);
@@ -136,14 +144,13 @@ export class CopilotTranscriptionResolver {
       workspaceId,
       jobId
     );
-    if (!job || !job.url || !job.mimeType) {
+    if (!job || !job.infos) {
       throw new CopilotTranscriptionJobNotFound();
     }
 
     const jobResult = await this.service.executeTranscriptionJob(
       job.id,
-      job.url,
-      job.mimeType
+      job.infos
     );
 
     return this.handleJobResult(jobResult);

packages/backend/server/src/plugins/copilot/transcript/service.ts

@@ -23,6 +23,7 @@ import {
 } from '../providers';
 import { CopilotStorage } from '../storage';
 import {
+  AudioBlobInfos,
   TranscriptionPayload,
   TranscriptionResponseSchema,
   TranscriptPayloadSchema,
@@ -32,8 +33,7 @@ import { readStream } from './utils';
 export type TranscriptionJob = {
   id: string;
   status: AiJobStatus;
-  url?: string;
-  mimeType?: string;
+  infos?: AudioBlobInfos;
   transcription?: TranscriptionPayload;
 };
 
@@ -52,7 +52,7 @@ export class CopilotTranscriptionService {
     userId: string,
     workspaceId: string,
     blobId: string,
-    blob: FileUpload
+    blobs: FileUpload[]
   ): Promise<TranscriptionJob> {
     if (await this.models.copilotJob.has(userId, workspaceId, blobId)) {
       throw new CopilotTranscriptionJobExists();
@@ -65,21 +65,24 @@ export class CopilotTranscriptionService {
       type: AiJobType.transcription,
     });
 
-    const buffer = await readStream(blob.createReadStream());
-    const url = await this.storage.put(userId, workspaceId, blobId, buffer);
+    const infos: AudioBlobInfos = [];
+    for (const blob of blobs) {
+      const buffer = await readStream(blob.createReadStream());
+      const url = await this.storage.put(userId, workspaceId, blobId, buffer);
+      infos.push({ url, mimeType: blob.mimetype });
+    }
 
-    return await this.executeTranscriptionJob(jobId, url, blob.mimetype);
+    return await this.executeTranscriptionJob(jobId, infos);
   }
 
   async executeTranscriptionJob(
     jobId: string,
-    url: string,
-    mimeType: string
+    infos: AudioBlobInfos
   ): Promise<TranscriptionJob> {
     const status = AiJobStatus.running;
     const success = await this.models.copilotJob.update(jobId, {
       status,
-      payload: { url, mimeType },
+      payload: { infos },
     });
 
     if (!success) {
@@ -88,8 +91,7 @@ export class CopilotTranscriptionService {
 
     await this.job.add('copilot.transcript.submit', {
       jobId,
-      url,
-      mimeType,
+      infos,
     });
 
     return { id: jobId, status };
@@ -132,8 +134,13 @@ export class CopilotTranscriptionService {
 
     const payload = TranscriptPayloadSchema.safeParse(job.payload);
     if (payload.success) {
-      ret.url = payload.data.url || undefined;
-      ret.mimeType = payload.data.mimeType || undefined;
+      let { url, mimeType, infos } = payload.data;
+      infos = infos || [];
+      if (url && mimeType) {
+        infos.push({ url, mimeType });
+      }
+
+      ret.infos = this.mergeInfos(infos, url, mimeType);
       if (job.status === AiJobStatus.claimed) {
         ret.transcription = payload.data;
       }
@@ -173,7 +180,24 @@ export class CopilotTranscriptionService {
     );
   }
 
-  private convertTime(time: number) {
+  // TODO(@darkskygit): remove after old server down
+  private mergeInfos(
+    infos?: AudioBlobInfos | null,
+    url?: string | null,
+    mimeType?: string | null
+  ) {
+    if (url && mimeType) {
+      if (infos) {
+        infos.push({ url, mimeType });
+      } else {
+        infos = [{ url, mimeType }];
+      }
+    }
+    return infos || [];
+  }
+
+  private convertTime(time: number, offset = 0) {
+    time = time + offset;
     const minutes = Math.floor(time / 60);
     const seconds = Math.floor(time % 60);
     const hours = Math.floor(minutes / 60);
@@ -186,29 +210,38 @@ export class CopilotTranscriptionService {
   @OnJob('copilot.transcript.submit')
   async transcriptAudio({
     jobId,
+    infos,
+    // @deprecated
     url,
     mimeType,
   }: Jobs['copilot.transcript.submit']) {
     try {
-      const result = await this.chatWithPrompt(
-        'Transcript audio',
-        {
-          attachments: [url],
-          params: { mimetype: mimeType },
-        },
-        TranscriptionResponseSchema
-      );
+      const blobInfos = this.mergeInfos(infos, url, mimeType);
+      const transcriptions = [];
+      for (const [idx, { url, mimeType }] of blobInfos.entries()) {
+        const result = await this.chatWithPrompt(
+          'Transcript audio',
+          {
+            attachments: [url],
+            params: { mimetype: mimeType },
+          },
+          TranscriptionResponseSchema
+        );
+
+        const offset = idx * 10 * 60;
+        const transcription = TranscriptionResponseSchema.parse(
+          JSON.parse(result)
+        ).map(t => ({
+          speaker: t.a,
+          start: this.convertTime(t.s, offset),
+          end: this.convertTime(t.e, offset),
+          transcription: t.t,
+        }));
+        transcriptions.push(transcription);
+      }
 
-      const transcription = TranscriptionResponseSchema.parse(
-        JSON.parse(result)
-      ).map(t => ({
-        speaker: t.a,
-        start: this.convertTime(t.s),
-        end: this.convertTime(t.e),
-        transcription: t.t,
-      }));
       await this.models.copilotJob.update(jobId, {
-        payload: { transcription },
+        payload: { transcription: transcriptions.flat() },
       });
 
       await this.job.add('copilot.transcript.summary.submit', {

packages/backend/server/src/plugins/copilot/transcript/types.ts

@@ -20,9 +20,17 @@ const TranscriptionItemSchema = z.object({
 
 export const TranscriptionSchema = z.array(TranscriptionItemSchema);
 
+export const AudioBlobInfosSchema = z
+  .object({
+    url: z.string(),
+    mimeType: z.string(),
+  })
+  .array();
+
 export const TranscriptPayloadSchema = z.object({
   url: z.string().nullable().optional(),
   mimeType: z.string().nullable().optional(),
+  infos: AudioBlobInfosSchema.nullable().optional(),
   title: z.string().nullable().optional(),
   summary: z.string().nullable().optional(),
   transcription: TranscriptionSchema.nullable().optional(),
@@ -32,6 +40,8 @@ export type TranscriptionItem = z.infer<typeof TranscriptionItemSchema>;
 export type Transcription = z.infer<typeof TranscriptionSchema>;
 export type TranscriptionPayload = z.infer<typeof TranscriptPayloadSchema>;
 
+export type AudioBlobInfos = z.infer<typeof AudioBlobInfosSchema>;
+
 declare global {
   interface Events {
     'workspace.file.transcript.finished': {
@@ -44,8 +54,11 @@ declare global {
   interface Jobs {
     'copilot.transcript.submit': {
       jobId: string;
-      url: string;
-      mimeType: string;
+      infos?: AudioBlobInfos;
+      /// @deprecated use `infos` instead
+      url?: string;
+      /// @deprecated use `infos` instead
+      mimeType?: string;
     };
     'copilot.transcript.summary.submit': {
       jobId: string;

packages/backend/server/src/schema.gql

@@ -435,6 +435,7 @@ enum ErrorNames {
   COPILOT_QUOTA_EXCEEDED
   COPILOT_SESSION_DELETED
   COPILOT_SESSION_NOT_FOUND
+  COPILOT_TRANSCRIPTION_AUDIO_NOT_PROVIDED
   COPILOT_TRANSCRIPTION_JOB_EXISTS
   COPILOT_TRANSCRIPTION_JOB_NOT_FOUND
   CUSTOMER_PORTAL_CREATE_FAILED
@@ -1004,7 +1005,7 @@ type Mutation {
   sendVerifyChangeEmail(callbackUrl: String!, email: String!, token: String!): Boolean!
   sendVerifyEmail(callbackUrl: String!): Boolean!
   setBlob(blob: Upload!, workspaceId: String!): String!
-  submitAudioTranscription(blob: Upload!, blobId: String!, workspaceId: String!): TranscriptionResultType
+  submitAudioTranscription(blob: Upload, blobId: String!, blobs: [Upload!], workspaceId: String!): TranscriptionResultType
 
   """update app configuration"""
   updateAppConfig(updates: [UpdateAppConfigInput!]!): JSONObject!

packages/common/graphql/src/graphql/copilot-jobs-transcription-add.gql

@@ -1,5 +1,5 @@
-mutation submitAudioTranscription($workspaceId: String!, $blobId: String!, $blob: Upload!) {
-  submitAudioTranscription(blob: $blob, blobId: $blobId, workspaceId: $workspaceId) {
+mutation submitAudioTranscription($workspaceId: String!, $blobId: String!, $blob: Upload, $blobs: [Upload!]) {
+  submitAudioTranscription(blob: $blob, blobs: $blobs, blobId: $blobId, workspaceId: $workspaceId) {
     id
     status
   }

packages/common/graphql/src/graphql/index.ts

@@ -601,9 +601,10 @@ export const getCopilotHistoriesQuery = {
 export const submitAudioTranscriptionMutation = {
   id: 'submitAudioTranscriptionMutation' as const,
   op: 'submitAudioTranscription',
-  query: `mutation submitAudioTranscription($workspaceId: String!, $blobId: String!, $blob: Upload!) {
+  query: `mutation submitAudioTranscription($workspaceId: String!, $blobId: String!, $blob: Upload, $blobs: [Upload!]) {
   submitAudioTranscription(
     blob: $blob
+    blobs: $blobs
     blobId: $blobId
     workspaceId: $workspaceId
   ) {
@@ -611,7 +612,6 @@ export const submitAudioTranscriptionMutation = {
     status
   }
 }`,
-  file: true,
 };
 
 export const claimAudioTranscriptionMutation = {

packages/common/graphql/src/schema.ts

@@ -580,6 +580,7 @@ export enum ErrorNames {
   COPILOT_QUOTA_EXCEEDED = 'COPILOT_QUOTA_EXCEEDED',
   COPILOT_SESSION_DELETED = 'COPILOT_SESSION_DELETED',
   COPILOT_SESSION_NOT_FOUND = 'COPILOT_SESSION_NOT_FOUND',
+  COPILOT_TRANSCRIPTION_AUDIO_NOT_PROVIDED = 'COPILOT_TRANSCRIPTION_AUDIO_NOT_PROVIDED',
   COPILOT_TRANSCRIPTION_JOB_EXISTS = 'COPILOT_TRANSCRIPTION_JOB_EXISTS',
   COPILOT_TRANSCRIPTION_JOB_NOT_FOUND = 'COPILOT_TRANSCRIPTION_JOB_NOT_FOUND',
   CUSTOMER_PORTAL_CREATE_FAILED = 'CUSTOMER_PORTAL_CREATE_FAILED',
@@ -1432,8 +1433,9 @@ export interface MutationSetBlobArgs {
 }
 
 export interface MutationSubmitAudioTranscriptionArgs {
-  blob: Scalars['Upload']['input'];
+  blob?: InputMaybe<Scalars['Upload']['input']>;
   blobId: Scalars['String']['input'];
+  blobs?: InputMaybe<Array<Scalars['Upload']['input']>>;
   workspaceId: Scalars['String']['input'];
 }
 
@@ -2991,7 +2993,10 @@ export type GetCopilotHistoriesQuery = {
 export type SubmitAudioTranscriptionMutationVariables = Exact<{
   workspaceId: Scalars['String']['input'];
   blobId: Scalars['String']['input'];
-  blob: Scalars['Upload']['input'];
+  blob?: InputMaybe<Scalars['Upload']['input']>;
+  blobs?: InputMaybe<
+    Array<Scalars['Upload']['input']> | Scalars['Upload']['input']
+  >;
 }>;
 
 export type SubmitAudioTranscriptionMutation = {

packages/frontend/i18n/src/i18n.gen.ts

@@ -8127,6 +8127,10 @@ export function useAFFiNEI18N(): {
       * `Transcription job not found.`
       */
     ["error.COPILOT_TRANSCRIPTION_JOB_NOT_FOUND"](): string;
+    /**
+      * `Audio not provided.`
+      */
+    ["error.COPILOT_TRANSCRIPTION_AUDIO_NOT_PROVIDED"](): string;
     /**
       * `You have exceeded your blob size quota.`
       */

packages/frontend/i18n/src/resources/en.json

@@ -2010,6 +2010,7 @@
   "error.COPILOT_EMBEDDING_UNAVAILABLE": "Embedding feature not available, you may need to install pgvector extension to your database",
   "error.COPILOT_TRANSCRIPTION_JOB_EXISTS": "Transcription job already exists",
   "error.COPILOT_TRANSCRIPTION_JOB_NOT_FOUND": "Transcription job not found.",
+  "error.COPILOT_TRANSCRIPTION_AUDIO_NOT_PROVIDED": "Audio not provided.",
   "error.BLOB_QUOTA_EXCEEDED": "You have exceeded your blob size quota.",
   "error.STORAGE_QUOTA_EXCEEDED": "You have exceeded your storage quota.",
   "error.MEMBER_QUOTA_EXCEEDED": "You have exceeded your workspace member quota.",