feat(server): audio transcription (#10733)

2026-02-17 06:16:59 +08:00 · 2025-03-20 07:12:27 +00:00
parent bd5d930490
commit c16ae2d5b4
19 changed files with 712 additions and 58 deletions
--- a/packages/backend/server/package.json
+++ b/packages/backend/server/package.json
@@ -14,7 +14,7 @@
    "test": "ava --concurrency 1 --serial",
    "test:copilot": "ava \"src/__tests__/**/copilot-*.spec.ts\"",
    "test:coverage": "c8 ava --concurrency 1 --serial",
-    "test:copilot:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.spec.ts\"",
+    "test:copilot:coverage": "c8 ava --timeout=5m \"src/__tests__/copilot-*.spec.ts\"",
    "e2e": "cross-env TEST_MODE=e2e ava",
    "e2e:coverage": "cross-env TEST_MODE=e2e c8 ava",
    "data-migration": "cross-env NODE_ENV=development r ./src/data/index.ts",
--- a/packages/backend/server/src/tests/models/copilot-job.spec.ts
+++ b/packages/backend/server/src/tests/models/copilot-job.spec.ts
@@ -1,4 +1,10 @@
-import { AiJobStatus, AiJobType, PrismaClient } from '@prisma/client';
+import {
+  AiJobStatus,
+  AiJobType,
+  PrismaClient,
+  User,
+  Workspace,
+} from '@prisma/client';
 import ava, { TestFn } from 'ava';

 import { Config } from '../../base';
@@ -28,8 +34,15 @@ test.before(async t => {
  t.context.module = module;
 });

+let user: User;
+let workspace: Workspace;
+
 test.beforeEach(async t => {
  await t.context.module.initTestingDB();
+  user = await t.context.user.create({
+    email: 'test@affine.pro',
+  });
+  workspace = await t.context.workspace.create(user.id);
 });

 test.after(async t => {
@@ -37,11 +50,6 @@ test.after(async t => {
 });

 test('should create a copilot job', async t => {
-  const user = await t.context.user.create({
-    email: 'test@affine.pro',
-  });
-  const workspace = await t.context.workspace.create(user.id);
-
  const data = {
    workspaceId: workspace.id,
    blobId: 'blob-id',
@@ -71,10 +79,6 @@ test('should get null for non-exist job', async t => {
 });

 test('should update job', async t => {
-  const user = await t.context.user.create({
-    email: 'test@affine.pro',
-  });
-  const workspace = await t.context.workspace.create(user.id);
  const { id: jobId } = await t.context.copilotJob.create({
    workspaceId: workspace.id,
    blobId: 'blob-id',
@@ -97,10 +101,6 @@ test('should update job', async t => {
 });

 test('should claim job', async t => {
-  const user = await t.context.user.create({
-    email: 'test@affine.pro',
-  });
-  const workspace = await t.context.workspace.create(user.id);
  const { id: jobId } = await t.context.copilotJob.create({
    workspaceId: workspace.id,
    blobId: 'blob-id',
--- a/packages/backend/server/src/base/error/def.ts
+++ b/packages/backend/server/src/base/error/def.ts
@@ -697,6 +697,10 @@ export const USER_FRIENDLY_ERRORS = {
    type: 'action_forbidden',
    message: `Embedding feature not available, you may need to install pgvector extension to your database`,
  },
+  copilot_transcription_job_exists: {
+    type: 'bad_request',
+    message: () => 'Transcription job already exists',
+  },

  // Quota & Limit errors
  blob_quota_exceeded: {
--- a/packages/backend/server/src/base/error/errors.gen.ts
+++ b/packages/backend/server/src/base/error/errors.gen.ts
@@ -753,6 +753,12 @@ export class CopilotEmbeddingUnavailable extends UserFriendlyError {
  }
 }

+export class CopilotTranscriptionJobExists extends UserFriendlyError {
+  constructor(message?: string) {
+    super('bad_request', 'copilot_transcription_job_exists', message);
+  }
+}
+
 export class BlobQuotaExceeded extends UserFriendlyError {
  constructor(message?: string) {
    super('quota_exceeded', 'blob_quota_exceeded', message);
@@ -1000,6 +1006,7 @@ export enum ErrorNames {
  COPILOT_FAILED_TO_MODIFY_CONTEXT,
  COPILOT_FAILED_TO_MATCH_CONTEXT,
  COPILOT_EMBEDDING_UNAVAILABLE,
+  COPILOT_TRANSCRIPTION_JOB_EXISTS,
  BLOB_QUOTA_EXCEEDED,
  STORAGE_QUOTA_EXCEEDED,
  MEMBER_QUOTA_EXCEEDED,
--- a/packages/backend/server/src/plugins/copilot/index.ts
+++ b/packages/backend/server/src/plugins/copilot/index.ts
@@ -31,6 +31,10 @@ import {
 } from './resolver';
 import { ChatSessionService } from './session';
 import { CopilotStorage } from './storage';
+import {
+  CopilotTranscriptionResolver,
+  CopilotTranscriptionService,
+} from './transcript';
 import { CopilotWorkflowExecutors, CopilotWorkflowService } from './workflow';

 registerCopilotProvider(FalProvider);
@@ -58,6 +62,9 @@ registerCopilotProvider(PerplexityProvider);
    CopilotContextResolver,
    CopilotContextService,
    CopilotContextDocJob,
+    // transcription
+    CopilotTranscriptionService,
+    CopilotTranscriptionResolver,
  ],
  controllers: [CopilotController],
  contributesTo: ServerFeature.Copilot,
--- a/packages/backend/server/src/plugins/copilot/providers/google.ts
+++ b/packages/backend/server/src/plugins/copilot/providers/google.ts
@@ -101,53 +101,52 @@ export class GoogleProvider implements CopilotTextToTextProvider {
    return undefined;
  }

-  protected chatToGPTMessage(
+  protected async chatToGPTMessage(
    messages: PromptMessage[]
-  ): [string | undefined, ChatMessage[]] {
+  ): Promise<[string | undefined, ChatMessage[]]> {
    let system =
      messages[0]?.role === 'system' ? messages.shift()?.content : undefined;

    // filter redundant fields
-    const msgs = messages
-      .filter(m => m.role !== 'system')
-      .map(({ role, content, attachments, params }) => {
-        content = content.trim();
-        role = role as 'user' | 'assistant';
-        const mimetype = params?.mimetype;
-        if (Array.isArray(attachments)) {
-          const contents: (TextPart | FilePart)[] = [];
-          if (content.length) {
-            contents.push({
-              type: 'text',
-              text: content,
-            });
-          }
-          contents.push(
-            ...attachments
-              .map(url => {
-                if (SIMPLE_IMAGE_URL_REGEX.test(url)) {
-                  const mimeType =
-                    typeof mimetype === 'string'
-                      ? mimetype
-                      : this.inferMimeType(url);
-                  if (mimeType) {
-                    const data = url.startsWith('data:') ? url : new URL(url);
-                    return {
-                      type: 'file' as const,
-                      data,
-                      mimeType,
-                    };
-                  }
-                }
-                return undefined;
-              })
-              .filter(c => !!c)
-          );
-          return { role, content: contents } as ChatMessage;
-        } else {
-          return { role, content } as ChatMessage;
+    const msgs: ChatMessage[] = [];
+    for (let { role, content, attachments, params } of messages.filter(
+      m => m.role !== 'system'
+    )) {
+      content = content.trim();
+      role = role as 'user' | 'assistant';
+      const mimetype = params?.mimetype;
+      if (Array.isArray(attachments)) {
+        const contents: (TextPart | FilePart)[] = [];
+        if (content.length) {
+          contents.push({
+            type: 'text',
+            text: content,
+          });
        }
-      });
+
+        for (const url of attachments) {
+          if (SIMPLE_IMAGE_URL_REGEX.test(url)) {
+            const mimeType =
+              typeof mimetype === 'string' ? mimetype : this.inferMimeType(url);
+            if (mimeType) {
+              const data = url.startsWith('data:')
+                ? await fetch(url).then(r => r.arrayBuffer())
+                : new URL(url);
+              contents.push({
+                type: 'file' as const,
+                data,
+                mimeType,
+              });
+            }
+          }
+        }
+
+        msgs.push({ role, content: contents } as ChatMessage);
+      } else {
+        msgs.push({ role, content });
+      }
+    }
+
    return [system, msgs];
  }

@@ -237,7 +236,7 @@ export class GoogleProvider implements CopilotTextToTextProvider {
    try {
      metrics.ai.counter('chat_text_calls').add(1, { model });

-      const [system, msgs] = this.chatToGPTMessage(messages);
+      const [system, msgs] = await this.chatToGPTMessage(messages);

      const { text } = await generateText({
        model: this.instance(model, {
@@ -266,7 +265,7 @@ export class GoogleProvider implements CopilotTextToTextProvider {

    try {
      metrics.ai.counter('chat_text_stream_calls').add(1, { model });
-      const [system, msgs] = this.chatToGPTMessage(messages);
+      const [system, msgs] = await this.chatToGPTMessage(messages);

      const { textStream } = streamText({
        model: this.instance(model),
--- a/packages/backend/server/src/plugins/copilot/transcript/index.ts
+++ b/packages/backend/server/src/plugins/copilot/transcript/index.ts
@@ -0,0 +1,2 @@
+export { CopilotTranscriptionResolver } from './resolver';
+export { CopilotTranscriptionService } from './service';
--- a/packages/backend/server/src/plugins/copilot/transcript/resolver.ts
+++ b/packages/backend/server/src/plugins/copilot/transcript/resolver.ts
@@ -0,0 +1,134 @@
+import { Injectable } from '@nestjs/common';
+import {
+  Args,
+  Field,
+  ID,
+  Mutation,
+  ObjectType,
+  Parent,
+  registerEnumType,
+  ResolveField,
+  Resolver,
+} from '@nestjs/graphql';
+import { AiJobStatus } from '@prisma/client';
+import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';
+
+import type { FileUpload } from '../../../base';
+import { CurrentUser } from '../../../core/auth';
+import { AccessController } from '../../../core/permission';
+import { CopilotType } from '../resolver';
+import { CopilotTranscriptionService, TranscriptionJob } from './service';
+import type { TranscriptionItem, TranscriptionPayload } from './types';
+
+registerEnumType(AiJobStatus, {
+  name: 'AiJobStatus',
+});
+
+@ObjectType()
+class TranscriptionItemType implements TranscriptionItem {
+  @Field(() => String)
+  speaker!: string;
+
+  @Field(() => String)
+  start!: string;
+
+  @Field(() => String)
+  end!: string;
+
+  @Field(() => String)
+  transcription!: string;
+}
+
+@ObjectType()
+class TranscriptionResultType implements TranscriptionPayload {
+  @Field(() => ID)
+  id!: string;
+
+  @Field(() => [TranscriptionItemType], { nullable: true })
+  transcription!: TranscriptionItemType[] | null;
+
+  @Field(() => String, { nullable: true })
+  summary!: string | null;
+
+  @Field(() => AiJobStatus)
+  status!: AiJobStatus;
+}
+
+@Injectable()
+@Resolver(() => CopilotType)
+export class CopilotTranscriptionResolver {
+  constructor(
+    private readonly ac: AccessController,
+    private readonly service: CopilotTranscriptionService
+  ) {}
+
+  private handleJobResult(
+    job: TranscriptionJob | null
+  ): TranscriptionResultType | null {
+    if (job) {
+      const { transcription: ret, status } = job;
+      return {
+        id: job.id,
+        transcription: ret?.transcription || null,
+        summary: ret?.summary || null,
+        status,
+      };
+    }
+    return null;
+  }
+
+  @Mutation(() => TranscriptionResultType, { nullable: true })
+  async submitAudioTranscription(
+    @CurrentUser() user: CurrentUser,
+    @Args('workspaceId') workspaceId: string,
+    @Args('blobId') blobId: string,
+    @Args({ name: 'blob', type: () => GraphQLUpload })
+    blob: FileUpload
+  ): Promise<TranscriptionResultType | null> {
+    await this.ac
+      .user(user.id)
+      .workspace(workspaceId)
+      .allowLocal()
+      .assert('Workspace.Copilot');
+
+    const job = await this.service.submitTranscriptionJob(
+      user.id,
+      workspaceId,
+      blobId,
+      blob
+    );
+
+    return this.handleJobResult(job);
+  }
+
+  @Mutation(() => TranscriptionResultType, { nullable: true })
+  async claimAudioTranscription(
+    @CurrentUser() user: CurrentUser,
+    @Args('jobId') jobId: string
+  ): Promise<TranscriptionResultType | null> {
+    const job = await this.service.claimTranscriptionJob(user.id, jobId);
+    return this.handleJobResult(job);
+  }
+
+  @ResolveField(() => [TranscriptionResultType], {})
+  async audioTranscription(
+    @Parent() copilot: CopilotType,
+    @CurrentUser() user: CurrentUser,
+    @Args('jobId', { nullable: true })
+    jobId: string
+  ): Promise<TranscriptionResultType | null> {
+    if (!copilot.workspaceId) return null;
+    await this.ac
+      .user(user.id)
+      .workspace(copilot.workspaceId)
+      .allowLocal()
+      .assert('Workspace.Copilot');
+
+    const job = await this.service.queryTranscriptionJob(
+      user.id,
+      copilot.workspaceId,
+      jobId
+    );
+    return this.handleJobResult(job);
+  }
+}
--- a/packages/backend/server/src/plugins/copilot/transcript/service.ts
+++ b/packages/backend/server/src/plugins/copilot/transcript/service.ts
@@ -0,0 +1,206 @@
+import { Injectable } from '@nestjs/common';
+import { AiJobStatus, AiJobType } from '@prisma/client';
+
+import {
+  CopilotPromptNotFound,
+  CopilotTranscriptionJobExists,
+  type FileUpload,
+  JobQueue,
+  NoCopilotProviderAvailable,
+  OnJob,
+} from '../../../base';
+import { Models } from '../../../models';
+import { PromptService } from '../prompt';
+import { CopilotProviderService } from '../providers';
+import { CopilotStorage } from '../storage';
+import {
+  CopilotCapability,
+  CopilotTextProvider,
+  PromptMessage,
+} from '../types';
+import {
+  TranscriptionPayload,
+  TranscriptionSchema,
+  TranscriptPayloadSchema,
+} from './types';
+import { readStream } from './utils';
+
+export type TranscriptionJob = {
+  id: string;
+  status: AiJobStatus;
+  transcription?: TranscriptionPayload;
+};
+
+@Injectable()
+export class CopilotTranscriptionService {
+  constructor(
+    private readonly models: Models,
+    private readonly job: JobQueue,
+    private readonly storage: CopilotStorage,
+    private readonly prompt: PromptService,
+    private readonly provider: CopilotProviderService
+  ) {}
+
+  async submitTranscriptionJob(
+    userId: string,
+    workspaceId: string,
+    blobId: string,
+    blob: FileUpload
+  ): Promise<TranscriptionJob> {
+    if (await this.models.copilotJob.has(workspaceId, blobId)) {
+      throw new CopilotTranscriptionJobExists();
+    }
+
+    const { id: jobId, status } = await this.models.copilotJob.create({
+      workspaceId,
+      blobId,
+      createdBy: userId,
+      type: AiJobType.transcription,
+    });
+
+    const buffer = await readStream(blob.createReadStream());
+    const url = await this.storage.put(userId, workspaceId, blobId, buffer);
+
+    await this.models.copilotJob.update(jobId, {
+      status: AiJobStatus.running,
+    });
+
+    await this.job.add(
+      'copilot.transcript.submit',
+      {
+        jobId,
+        url,
+        mimeType: blob.mimetype,
+      },
+      // retry 3 times
+      { removeOnFail: 3 }
+    );
+
+    return { id: jobId, status };
+  }
+
+  async claimTranscriptionJob(
+    userId: string,
+    jobId: string
+  ): Promise<TranscriptionJob | null> {
+    const status = await this.models.copilotJob.claim(jobId, userId);
+    if (status === AiJobStatus.claimed) {
+      const transcription = await this.models.copilotJob.getPayload(
+        jobId,
+        TranscriptPayloadSchema
+      );
+      return { id: jobId, transcription, status };
+    }
+    return null;
+  }
+
+  async queryTranscriptionJob(
+    userId: string,
+    workspaceId: string,
+    jobId: string
+  ) {
+    const job = await this.models.copilotJob.getWithUser(
+      userId,
+      workspaceId,
+      jobId,
+      AiJobType.transcription
+    );
+
+    if (!job) {
+      return null;
+    }
+
+    const ret: TranscriptionJob = { id: job.id, status: job.status };
+
+    const payload = TranscriptPayloadSchema.safeParse(job.payload);
+    if (payload.success) {
+      ret.transcription = payload.data;
+    }
+
+    return ret;
+  }
+
+  private async getProvider(model: string): Promise<CopilotTextProvider> {
+    let provider = await this.provider.getProviderByCapability(
+      CopilotCapability.TextToText,
+      model
+    );
+
+    if (!provider) {
+      throw new NoCopilotProviderAvailable();
+    }
+
+    return provider;
+  }
+
+  private async chatWithPrompt(
+    promptName: string,
+    message: Partial<PromptMessage>
+  ): Promise<string> {
+    const prompt = await this.prompt.get(promptName);
+    if (!prompt) {
+      throw new CopilotPromptNotFound({ name: promptName });
+    }
+
+    const provider = await this.getProvider(prompt.model);
+    return provider.generateText(
+      [...prompt.finish({}), { role: 'user', content: '', ...message }],
+      prompt.model
+    );
+  }
+
+  private cleanupResponse(response: string): string {
+    return response
+      .replace(/```[\w\s]+\n/g, '')
+      .replace(/\n```/g, '')
+      .trim();
+  }
+
+  @OnJob('copilot.transcript.submit')
+  async transcriptAudio({
+    jobId,
+    url,
+    mimeType,
+  }: Jobs['copilot.transcript.submit']) {
+    const result = await this.chatWithPrompt('Transcript audio', {
+      attachments: [url],
+      params: { mimetype: mimeType },
+    });
+
+    const transcription = TranscriptionSchema.parse(
+      JSON.parse(this.cleanupResponse(result))
+    );
+    await this.models.copilotJob.update(jobId, { payload: { transcription } });
+
+    await this.job.add(
+      'copilot.summary.submit',
+      {
+        jobId,
+      },
+      // retry 3 times
+      { removeOnFail: 3 }
+    );
+  }
+
+  @OnJob('copilot.summary.submit')
+  async summaryTranscription({ jobId }: Jobs['copilot.summary.submit']) {
+    const payload = await this.models.copilotJob.getPayload(
+      jobId,
+      TranscriptPayloadSchema
+    );
+    if (payload.transcription) {
+      const content = payload.transcription
+        .map(t => t.transcription)
+        .join('\n');
+
+      const result = await this.chatWithPrompt('Summary', { content });
+
+      payload.summary = this.cleanupResponse(result);
+      await this.models.copilotJob.update(jobId, { payload });
+    } else {
+      await this.models.copilotJob.update(jobId, {
+        status: AiJobStatus.failed,
+      });
+    }
+  }
+}
--- a/packages/backend/server/src/plugins/copilot/transcript/types.ts
+++ b/packages/backend/server/src/plugins/copilot/transcript/types.ts
@@ -0,0 +1,36 @@
+import { z } from 'zod';
+
+import { OneMB } from '../../../base';
+
+const TranscriptionItemSchema = z.object({
+  speaker: z.string(),
+  start: z.string(),
+  end: z.string(),
+  transcription: z.string(),
+});
+
+export const TranscriptionSchema = z.array(TranscriptionItemSchema);
+
+export const TranscriptPayloadSchema = z.object({
+  transcription: TranscriptionSchema.nullable().optional(),
+  summary: z.string().nullable().optional(),
+});
+
+export type TranscriptionItem = z.infer<typeof TranscriptionItemSchema>;
+export type Transcription = z.infer<typeof TranscriptionSchema>;
+export type TranscriptionPayload = z.infer<typeof TranscriptPayloadSchema>;
+
+declare global {
+  interface Jobs {
+    'copilot.transcript.submit': {
+      jobId: string;
+      url: string;
+      mimeType: string;
+    };
+    'copilot.summary.submit': {
+      jobId: string;
+    };
+  }
+}
+
+export const MAX_TRANSCRIPTION_SIZE = 50 * OneMB;
--- a/packages/backend/server/src/plugins/copilot/transcript/utils.ts
+++ b/packages/backend/server/src/plugins/copilot/transcript/utils.ts
@@ -0,0 +1,11 @@
+import { Readable } from 'node:stream';
+
+import { readBufferWithLimit } from '../../../base';
+import { MAX_TRANSCRIPTION_SIZE } from './types';
+
+export function readStream(
+  readable: Readable,
+  maxSize = MAX_TRANSCRIPTION_SIZE
+): Promise<Buffer> {
+  return readBufferWithLimit(readable, maxSize);
+}
--- a/packages/backend/server/src/schema.gql
+++ b/packages/backend/server/src/schema.gql
@@ -18,6 +18,14 @@ input AddRemoveContextCategoryInput {
  type: ContextCategories!
 }

+enum AiJobStatus {
+  claimed
+  failed
+  finished
+  pending
+  running
+}
+
 type AlreadyInSpaceDataType {
  spaceId: String!
 }
@@ -72,6 +80,8 @@ type ContextWorkspaceEmbeddingStatus {
 }

 type Copilot {
+  audioTranscription(jobId: String): [TranscriptionResultType!]!
+
  """Get the context list of a session"""
  contexts(contextId: String, sessionId: String): [CopilotContext!]!
  histories(docId: String, options: QueryChatHistoriesInput): [CopilotHistories!]!
@@ -409,6 +419,7 @@ enum ErrorNames {
  COPILOT_QUOTA_EXCEEDED
  COPILOT_SESSION_DELETED
  COPILOT_SESSION_NOT_FOUND
+  COPILOT_TRANSCRIPTION_JOB_EXISTS
  CUSTOMER_PORTAL_CREATE_FAILED
  DOC_ACTION_DENIED
  DOC_DEFAULT_ROLE_CAN_NOT_BE_OWNER
@@ -842,6 +853,7 @@ type Mutation {
  cancelSubscription(idempotencyKey: String @deprecated(reason: "use header `Idempotency-Key`"), plan: SubscriptionPlan = Pro, workspaceId: String): SubscriptionType!
  changeEmail(email: String!, token: String!): UserType!
  changePassword(newPassword: String!, token: String!, userId: String): Boolean!
+  claimAudioTranscription(jobId: String!): TranscriptionResultType

  """Cleanup sessions"""
  cleanupCopilotSession(options: DeleteSessionInput!): [String!]!
@@ -934,6 +946,7 @@ type Mutation {
  sendVerifyChangeEmail(callbackUrl: String!, email: String!, token: String!): Boolean!
  sendVerifyEmail(callbackUrl: String!): Boolean!
  setBlob(blob: Upload!, workspaceId: String!): String!
+  submitAudioTranscription(blob: Upload!, blobId: String!, workspaceId: String!): TranscriptionResultType

  """Update a copilot prompt"""
  updateCopilotPrompt(messages: [CopilotPromptMessageInput!]!, name: String!): CopilotPromptType!
@@ -1386,6 +1399,20 @@ enum SubscriptionVariant {
  Onetime
 }

+type TranscriptionItemType {
+  end: String!
+  speaker: String!
+  start: String!
+  transcription: String!
+}
+
+type TranscriptionResultType {
+  id: ID!
+  status: AiJobStatus!
+  summary: String
+  transcription: [TranscriptionItemType!]
+}
+
 union UnionNotificationBodyType = InvitationAcceptedNotificationBodyType | InvitationBlockedNotificationBodyType | InvitationNotificationBodyType | MentionNotificationBodyType

 type UnknownOauthProviderDataType {
--- a/packages/common/graphql/src/graphql/copilot-jobs-transcription-add.gql
+++ b/packages/common/graphql/src/graphql/copilot-jobs-transcription-add.gql
@@ -0,0 +1,6 @@
+mutation submitAudioTranscription($workspaceId: String!, $blobId: String!, $blob: Upload!) {
+  submitAudioTranscription(blob: $blob, blobId: $blobId, workspaceId: $workspaceId) {
+    id
+    status
+  }
+}
--- a/packages/common/graphql/src/graphql/copilot-jobs-transcription-claim.gql
+++ b/packages/common/graphql/src/graphql/copilot-jobs-transcription-claim.gql
@@ -0,0 +1,13 @@
+mutation claimAudioTranscription($jobId: String!) {
+  claimAudioTranscription(jobId: $jobId) {
+    id
+    status
+    transcription {
+      speaker
+      start
+      end
+      transcription
+    }
+    summary
+  }
+}
--- a/packages/common/graphql/src/graphql/copilot-jobs-transcription-list.gql
+++ b/packages/common/graphql/src/graphql/copilot-jobs-transcription-list.gql
@@ -0,0 +1,20 @@
+query getAudioTranscription(
+  $workspaceId: String!
+  $jobId: String!
+) {
+  currentUser {
+    copilot(workspaceId: $workspaceId) {
+      audioTranscription(jobId: $jobId) {
+        id
+        status
+        transcription {
+          speaker
+          start
+          end
+          transcription
+        }
+        summary
+      }
+    }
+  }
+}
--- a/packages/common/graphql/src/graphql/index.ts
+++ b/packages/common/graphql/src/graphql/index.ts
@@ -349,6 +349,62 @@ export const getCopilotHistoriesQuery = {
 }`,
 };

+export const submitAudioTranscriptionMutation = {
+  id: 'submitAudioTranscriptionMutation' as const,
+  op: 'submitAudioTranscription',
+  query: `mutation submitAudioTranscription($workspaceId: String!, $blobId: String!, $blob: Upload!) {
+  submitAudioTranscription(
+    blob: $blob
+    blobId: $blobId
+    workspaceId: $workspaceId
+  ) {
+    id
+    status
+  }
+}`,
+  file: true,
+};
+
+export const claimAudioTranscriptionMutation = {
+  id: 'claimAudioTranscriptionMutation' as const,
+  op: 'claimAudioTranscription',
+  query: `mutation claimAudioTranscription($jobId: String!) {
+  claimAudioTranscription(jobId: $jobId) {
+    id
+    status
+    transcription {
+      speaker
+      start
+      end
+      transcription
+    }
+    summary
+  }
+}`,
+};
+
+export const getAudioTranscriptionQuery = {
+  id: 'getAudioTranscriptionQuery' as const,
+  op: 'getAudioTranscription',
+  query: `query getAudioTranscription($workspaceId: String!, $jobId: String!) {
+  currentUser {
+    copilot(workspaceId: $workspaceId) {
+      audioTranscription(jobId: $jobId) {
+        id
+        status
+        transcription {
+          speaker
+          start
+          end
+          transcription
+        }
+        summary
+      }
+    }
+  }
+}`,
+};
+
 export const createCopilotMessageMutation = {
  id: 'createCopilotMessageMutation' as const,
  op: 'createCopilotMessage',
--- a/packages/common/graphql/src/schema.ts
+++ b/packages/common/graphql/src/schema.ts
@@ -53,6 +53,14 @@ export interface AddRemoveContextCategoryInput {
  type: ContextCategories;
 }

+export enum AiJobStatus {
+  claimed = 'claimed',
+  failed = 'failed',
+  finished = 'finished',
+  pending = 'pending',
+  running = 'running',
+}
+
 export interface AlreadyInSpaceDataType {
  __typename?: 'AlreadyInSpaceDataType';
  spaceId: Scalars['String']['output'];
@@ -114,6 +122,7 @@ export interface ContextWorkspaceEmbeddingStatus {

 export interface Copilot {
  __typename?: 'Copilot';
+  audioTranscription: Array<TranscriptionResultType>;
  /** Get the context list of a session */
  contexts: Array<CopilotContext>;
  histories: Array<CopilotHistories>;
@@ -129,6 +138,10 @@ export interface Copilot {
  workspaceId: Maybe<Scalars['ID']['output']>;
 }

+export interface CopilotAudioTranscriptionArgs {
+  jobId?: InputMaybe<Scalars['String']['input']>;
+}
+
 export interface CopilotContextsArgs {
  contextId?: InputMaybe<Scalars['String']['input']>;
  sessionId?: InputMaybe<Scalars['String']['input']>;
@@ -550,6 +563,7 @@ export enum ErrorNames {
  COPILOT_QUOTA_EXCEEDED = 'COPILOT_QUOTA_EXCEEDED',
  COPILOT_SESSION_DELETED = 'COPILOT_SESSION_DELETED',
  COPILOT_SESSION_NOT_FOUND = 'COPILOT_SESSION_NOT_FOUND',
+  COPILOT_TRANSCRIPTION_JOB_EXISTS = 'COPILOT_TRANSCRIPTION_JOB_EXISTS',
  CUSTOMER_PORTAL_CREATE_FAILED = 'CUSTOMER_PORTAL_CREATE_FAILED',
  DOC_ACTION_DENIED = 'DOC_ACTION_DENIED',
  DOC_DEFAULT_ROLE_CAN_NOT_BE_OWNER = 'DOC_DEFAULT_ROLE_CAN_NOT_BE_OWNER',
@@ -977,6 +991,7 @@ export interface Mutation {
  cancelSubscription: SubscriptionType;
  changeEmail: UserType;
  changePassword: Scalars['Boolean']['output'];
+  claimAudioTranscription: Maybe<TranscriptionResultType>;
  /** Cleanup sessions */
  cleanupCopilotSession: Array<Scalars['String']['output']>;
  /** Create change password url */
@@ -1050,6 +1065,7 @@ export interface Mutation {
  sendVerifyChangeEmail: Scalars['Boolean']['output'];
  sendVerifyEmail: Scalars['Boolean']['output'];
  setBlob: Scalars['String']['output'];
+  submitAudioTranscription: Maybe<TranscriptionResultType>;
  /** Update a copilot prompt */
  updateCopilotPrompt: CopilotPromptType;
  /** Update a chat session */
@@ -1130,6 +1146,10 @@ export interface MutationChangePasswordArgs {
  userId?: InputMaybe<Scalars['String']['input']>;
 }

+export interface MutationClaimAudioTranscriptionArgs {
+  jobId: Scalars['String']['input'];
+}
+
 export interface MutationCleanupCopilotSessionArgs {
  options: DeleteSessionInput;
 }
@@ -1352,6 +1372,12 @@ export interface MutationSetBlobArgs {
  workspaceId: Scalars['String']['input'];
 }

+export interface MutationSubmitAudioTranscriptionArgs {
+  blob: Scalars['Upload']['input'];
+  blobId: Scalars['String']['input'];
+  workspaceId: Scalars['String']['input'];
+}
+
 export interface MutationUpdateCopilotPromptArgs {
  messages: Array<CopilotPromptMessageInput>;
  name: Scalars['String']['input'];
@@ -1878,6 +1904,22 @@ export enum SubscriptionVariant {
  Onetime = 'Onetime',
 }

+export interface TranscriptionItemType {
+  __typename?: 'TranscriptionItemType';
+  end: Scalars['String']['output'];
+  speaker: Scalars['String']['output'];
+  start: Scalars['String']['output'];
+  transcription: Scalars['String']['output'];
+}
+
+export interface TranscriptionResultType {
+  __typename?: 'TranscriptionResultType';
+  id: Scalars['ID']['output'];
+  status: AiJobStatus;
+  summary: Maybe<Scalars['String']['output']>;
+  transcription: Maybe<Array<TranscriptionItemType>>;
+}
+
 export type UnionNotificationBodyType =
  | InvitationAcceptedNotificationBodyType
  | InvitationBlockedNotificationBodyType
@@ -2651,6 +2693,70 @@ export type GetCopilotHistoriesQuery = {
  } | null;
 };

+export type SubmitAudioTranscriptionMutationVariables = Exact<{
+  workspaceId: Scalars['String']['input'];
+  blobId: Scalars['String']['input'];
+  blob: Scalars['Upload']['input'];
+}>;
+
+export type SubmitAudioTranscriptionMutation = {
+  __typename?: 'Mutation';
+  submitAudioTranscription: {
+    __typename?: 'TranscriptionResultType';
+    id: string;
+    status: AiJobStatus;
+  } | null;
+};
+
+export type ClaimAudioTranscriptionMutationVariables = Exact<{
+  jobId: Scalars['String']['input'];
+}>;
+
+export type ClaimAudioTranscriptionMutation = {
+  __typename?: 'Mutation';
+  claimAudioTranscription: {
+    __typename?: 'TranscriptionResultType';
+    id: string;
+    status: AiJobStatus;
+    summary: string | null;
+    transcription: Array<{
+      __typename?: 'TranscriptionItemType';
+      speaker: string;
+      start: string;
+      end: string;
+      transcription: string;
+    }> | null;
+  } | null;
+};
+
+export type GetAudioTranscriptionQueryVariables = Exact<{
+  workspaceId: Scalars['String']['input'];
+  jobId: Scalars['String']['input'];
+}>;
+
+export type GetAudioTranscriptionQuery = {
+  __typename?: 'Query';
+  currentUser: {
+    __typename?: 'UserType';
+    copilot: {
+      __typename?: 'Copilot';
+      audioTranscription: Array<{
+        __typename?: 'TranscriptionResultType';
+        id: string;
+        status: AiJobStatus;
+        summary: string | null;
+        transcription: Array<{
+          __typename?: 'TranscriptionItemType';
+          speaker: string;
+          start: string;
+          end: string;
+          transcription: string;
+        }> | null;
+      }>;
+    };
+  } | null;
+};
+
 export type CreateCopilotMessageMutationVariables = Exact<{
  options: CreateChatMessageInput;
 }>;
@@ -4153,6 +4259,11 @@ export type Queries =
      variables: GetCopilotHistoriesQueryVariables;
      response: GetCopilotHistoriesQuery;
    }
+  | {
+      name: 'getAudioTranscriptionQuery';
+      variables: GetAudioTranscriptionQueryVariables;
+      response: GetAudioTranscriptionQuery;
+    }
  | {
      name: 'getPromptsQuery';
      variables: GetPromptsQueryVariables;
@@ -4460,6 +4571,16 @@ export type Mutations =
      variables: QueueWorkspaceEmbeddingMutationVariables;
      response: QueueWorkspaceEmbeddingMutation;
    }
+  | {
+      name: 'submitAudioTranscriptionMutation';
+      variables: SubmitAudioTranscriptionMutationVariables;
+      response: SubmitAudioTranscriptionMutation;
+    }
+  | {
+      name: 'claimAudioTranscriptionMutation';
+      variables: ClaimAudioTranscriptionMutationVariables;
+      response: ClaimAudioTranscriptionMutation;
+    }
  | {
      name: 'createCopilotMessageMutation';
      variables: CreateCopilotMessageMutationVariables;
--- a/packages/frontend/i18n/src/i18n.gen.ts
+++ b/packages/frontend/i18n/src/i18n.gen.ts
@@ -7779,6 +7779,10 @@ export function useAFFiNEI18N(): {
      * `Embedding feature not available, you may need to install pgvector extension to your database`
      */
    ["error.COPILOT_EMBEDDING_UNAVAILABLE"](): string;
+    /**
+      * `Transcription job already exists`
+      */
+    ["error.COPILOT_TRANSCRIPTION_JOB_EXISTS"](): string;
    /**
      * `You have exceeded your blob size quota.`
      */
--- a/packages/frontend/i18n/src/resources/en.json
+++ b/packages/frontend/i18n/src/resources/en.json
@@ -1914,6 +1914,7 @@
  "error.COPILOT_FAILED_TO_MODIFY_CONTEXT": "Failed to modify context {{contextId}}: {{message}}",
  "error.COPILOT_FAILED_TO_MATCH_CONTEXT": "Failed to match context {{contextId}} with \"%7B%7Bcontent%7D%7D\": {{message}}",
  "error.COPILOT_EMBEDDING_UNAVAILABLE": "Embedding feature not available, you may need to install pgvector extension to your database",
+  "error.COPILOT_TRANSCRIPTION_JOB_EXISTS": "Transcription job already exists",
  "error.BLOB_QUOTA_EXCEEDED": "You have exceeded your blob size quota.",
  "error.STORAGE_QUOTA_EXCEEDED": "You have exceeded your storage quota.",
  "error.MEMBER_QUOTA_EXCEEDED": "You have exceeded your workspace member quota.",