feat(server): update trascript endpoint (#11196)

packages/backend/server/src/plugins/copilot/transcript/resolver.ts

@@ -44,16 +44,24 @@ class TranscriptionResultType implements TranscriptionPayload {
   @Field(() => ID)
   id!: string;
 
-  @Field(() => [TranscriptionItemType], { nullable: true })
-  transcription!: TranscriptionItemType[] | null;
+  @Field(() => String, { nullable: true })
+  title!: string | null;
 
   @Field(() => String, { nullable: true })
   summary!: string | null;
 
+  @Field(() => [TranscriptionItemType], { nullable: true })
+  transcription!: TranscriptionItemType[] | null;
+
   @Field(() => AiJobStatus)
   status!: AiJobStatus;
 }
 
+const FinishedStatus: Set<AiJobStatus> = new Set([
+  AiJobStatus.finished,
+  AiJobStatus.claimed,
+]);
+
 @Injectable()
 @Resolver(() => CopilotType)
 export class CopilotTranscriptionResolver {
@@ -67,12 +75,19 @@ export class CopilotTranscriptionResolver {
   ): TranscriptionResultType | null {
     if (job) {
       const { transcription: ret, status } = job;
-      return {
+      const finalJob: TranscriptionResultType = {
         id: job.id,
-        transcription: ret?.transcription || null,
-        summary: ret?.summary || null,
         status,
+        title: null,
+        summary: null,
+        transcription: null,
       };
+      if (FinishedStatus.has(finalJob.status)) {
+        finalJob.title = ret?.title || null;
+        finalJob.summary = ret?.summary || null;
+        finalJob.transcription = ret?.transcription || null;
+      }
+      return finalJob;
     }
     return null;
   }
@@ -110,14 +125,20 @@ export class CopilotTranscriptionResolver {
     return this.handleJobResult(job);
   }
 
-  @ResolveField(() => [TranscriptionResultType], {})
+  @ResolveField(() => TranscriptionResultType, {
+    nullable: true,
+  })
   async audioTranscription(
     @Parent() copilot: CopilotType,
     @CurrentUser() user: CurrentUser,
     @Args('jobId', { nullable: true })
-    jobId: string
+    jobId?: string,
+    @Args('blobId', { nullable: true })
+    blobId?: string
   ): Promise<TranscriptionResultType | null> {
     if (!copilot.workspaceId) return null;
+    if (!jobId && !blobId) return null;
+
     await this.ac
       .user(user.id)
       .workspace(copilot.workspaceId)
@@ -127,7 +148,8 @@ export class CopilotTranscriptionResolver {
     const job = await this.service.queryTranscriptionJob(
       user.id,
       copilot.workspaceId,
-      jobId
+      jobId,
+      blobId
     );
     return this.handleJobResult(job);
   }

packages/backend/server/src/plugins/copilot/transcript/service.ts

@@ -47,7 +47,7 @@ export class CopilotTranscriptionService {
     blobId: string,
     blob: FileUpload
   ): Promise<TranscriptionJob> {
-    if (await this.models.copilotJob.has(workspaceId, blobId)) {
+    if (await this.models.copilotJob.has(userId, workspaceId, blobId)) {
       throw new CopilotTranscriptionJobExists();
     }
 
@@ -97,12 +97,14 @@ export class CopilotTranscriptionService {
   async queryTranscriptionJob(
     userId: string,
     workspaceId: string,
-    jobId: string
+    jobId?: string,
+    blobId?: string
   ) {
     const job = await this.models.copilotJob.getWithUser(
       userId,
       workspaceId,
       jobId,
+      blobId,
       AiJobType.transcription
     );
 
@@ -170,10 +172,12 @@ export class CopilotTranscriptionService {
     const transcription = TranscriptionSchema.parse(
       JSON.parse(this.cleanupResponse(result))
     );
-    await this.models.copilotJob.update(jobId, { payload: { transcription } });
+    await this.models.copilotJob.update(jobId, {
+      payload: { transcription },
+    });
 
     await this.job.add(
-      'copilot.summary.submit',
+      'copilot.transcriptSummary.submit',
       {
         jobId,
       },
@@ -182,8 +186,8 @@ export class CopilotTranscriptionService {
     );
   }
 
-  @OnJob('copilot.summary.submit')
-  async summaryTranscription({ jobId }: Jobs['copilot.summary.submit']) {
+  @OnJob('copilot.transcriptSummary.submit')
+  async transcriptSummary({ jobId }: Jobs['copilot.transcriptSummary.submit']) {
     const payload = await this.models.copilotJob.getPayload(
       jobId,
       TranscriptPayloadSchema
@@ -196,7 +200,41 @@ export class CopilotTranscriptionService {
       const result = await this.chatWithPrompt('Summary', { content });
 
       payload.summary = this.cleanupResponse(result);
-      await this.models.copilotJob.update(jobId, { payload });
+      await this.models.copilotJob.update(jobId, {
+        payload,
+      });
+
+      await this.job.add(
+        'copilot.transcriptTitle.submit',
+        { jobId },
+        // retry 3 times
+        { removeOnFail: 3 }
+      );
+    } else {
+      await this.models.copilotJob.update(jobId, {
+        status: AiJobStatus.failed,
+      });
+    }
+  }
+
+  @OnJob('copilot.transcriptTitle.submit')
+  async transcriptTitle({ jobId }: Jobs['copilot.transcriptTitle.submit']) {
+    const payload = await this.models.copilotJob.getPayload(
+      jobId,
+      TranscriptPayloadSchema
+    );
+    if (payload.transcription && payload.summary) {
+      const content = payload.transcription
+        .map(t => t.transcription)
+        .join('\n');
+
+      const result = await this.chatWithPrompt('Summary as title', { content });
+
+      payload.title = this.cleanupResponse(result);
+      await this.models.copilotJob.update(jobId, {
+        payload,
+        status: AiJobStatus.finished,
+      });
     } else {
       await this.models.copilotJob.update(jobId, {
         status: AiJobStatus.failed,

packages/backend/server/src/plugins/copilot/transcript/types.ts

@@ -12,8 +12,9 @@ const TranscriptionItemSchema = z.object({
 export const TranscriptionSchema = z.array(TranscriptionItemSchema);
 
 export const TranscriptPayloadSchema = z.object({
-  transcription: TranscriptionSchema.nullable().optional(),
+  title: z.string().nullable().optional(),
   summary: z.string().nullable().optional(),
+  transcription: TranscriptionSchema.nullable().optional(),
 });
 
 export type TranscriptionItem = z.infer<typeof TranscriptionItemSchema>;
@@ -27,7 +28,10 @@ declare global {
       url: string;
       mimeType: string;
     };
-    'copilot.summary.submit': {
+    'copilot.transcriptSummary.submit': {
+      jobId: string;
+    };
+    'copilot.transcriptTitle.submit': {
       jobId: string;
     };
   }