mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-12 07:36:42 +08:00
feat(server): audio transcription (#10733)
This commit is contained in:
darkskygit
@@ -0,0 +1,2 @@
|
||||
export { CopilotTranscriptionResolver } from './resolver';
|
||||
export { CopilotTranscriptionService } from './service';
|
||||
@@ -0,0 +1,134 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
Args,
|
||||
Field,
|
||||
ID,
|
||||
Mutation,
|
||||
ObjectType,
|
||||
Parent,
|
||||
registerEnumType,
|
||||
ResolveField,
|
||||
Resolver,
|
||||
} from '@nestjs/graphql';
|
||||
import { AiJobStatus } from '@prisma/client';
|
||||
import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';
|
||||
|
||||
import type { FileUpload } from '../../../base';
|
||||
import { CurrentUser } from '../../../core/auth';
|
||||
import { AccessController } from '../../../core/permission';
|
||||
import { CopilotType } from '../resolver';
|
||||
import { CopilotTranscriptionService, TranscriptionJob } from './service';
|
||||
import type { TranscriptionItem, TranscriptionPayload } from './types';
|
||||
|
||||
registerEnumType(AiJobStatus, {
|
||||
name: 'AiJobStatus',
|
||||
});
|
||||
|
||||
@ObjectType()
|
||||
class TranscriptionItemType implements TranscriptionItem {
|
||||
@Field(() => String)
|
||||
speaker!: string;
|
||||
|
||||
@Field(() => String)
|
||||
start!: string;
|
||||
|
||||
@Field(() => String)
|
||||
end!: string;
|
||||
|
||||
@Field(() => String)
|
||||
transcription!: string;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class TranscriptionResultType implements TranscriptionPayload {
|
||||
@Field(() => ID)
|
||||
id!: string;
|
||||
|
||||
@Field(() => [TranscriptionItemType], { nullable: true })
|
||||
transcription!: TranscriptionItemType[] | null;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
summary!: string | null;
|
||||
|
||||
@Field(() => AiJobStatus)
|
||||
status!: AiJobStatus;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
@Resolver(() => CopilotType)
|
||||
export class CopilotTranscriptionResolver {
|
||||
constructor(
|
||||
private readonly ac: AccessController,
|
||||
private readonly service: CopilotTranscriptionService
|
||||
) {}
|
||||
|
||||
private handleJobResult(
|
||||
job: TranscriptionJob | null
|
||||
): TranscriptionResultType | null {
|
||||
if (job) {
|
||||
const { transcription: ret, status } = job;
|
||||
return {
|
||||
id: job.id,
|
||||
transcription: ret?.transcription || null,
|
||||
summary: ret?.summary || null,
|
||||
status,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Mutation(() => TranscriptionResultType, { nullable: true })
|
||||
async submitAudioTranscription(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('workspaceId') workspaceId: string,
|
||||
@Args('blobId') blobId: string,
|
||||
@Args({ name: 'blob', type: () => GraphQLUpload })
|
||||
blob: FileUpload
|
||||
): Promise<TranscriptionResultType | null> {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspaceId)
|
||||
.allowLocal()
|
||||
.assert('Workspace.Copilot');
|
||||
|
||||
const job = await this.service.submitTranscriptionJob(
|
||||
user.id,
|
||||
workspaceId,
|
||||
blobId,
|
||||
blob
|
||||
);
|
||||
|
||||
return this.handleJobResult(job);
|
||||
}
|
||||
|
||||
@Mutation(() => TranscriptionResultType, { nullable: true })
|
||||
async claimAudioTranscription(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('jobId') jobId: string
|
||||
): Promise<TranscriptionResultType | null> {
|
||||
const job = await this.service.claimTranscriptionJob(user.id, jobId);
|
||||
return this.handleJobResult(job);
|
||||
}
|
||||
|
||||
@ResolveField(() => [TranscriptionResultType], {})
|
||||
async audioTranscription(
|
||||
@Parent() copilot: CopilotType,
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('jobId', { nullable: true })
|
||||
jobId: string
|
||||
): Promise<TranscriptionResultType | null> {
|
||||
if (!copilot.workspaceId) return null;
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(copilot.workspaceId)
|
||||
.allowLocal()
|
||||
.assert('Workspace.Copilot');
|
||||
|
||||
const job = await this.service.queryTranscriptionJob(
|
||||
user.id,
|
||||
copilot.workspaceId,
|
||||
jobId
|
||||
);
|
||||
return this.handleJobResult(job);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { AiJobStatus, AiJobType } from '@prisma/client';
|
||||
|
||||
import {
|
||||
CopilotPromptNotFound,
|
||||
CopilotTranscriptionJobExists,
|
||||
type FileUpload,
|
||||
JobQueue,
|
||||
NoCopilotProviderAvailable,
|
||||
OnJob,
|
||||
} from '../../../base';
|
||||
import { Models } from '../../../models';
|
||||
import { PromptService } from '../prompt';
|
||||
import { CopilotProviderService } from '../providers';
|
||||
import { CopilotStorage } from '../storage';
|
||||
import {
|
||||
CopilotCapability,
|
||||
CopilotTextProvider,
|
||||
PromptMessage,
|
||||
} from '../types';
|
||||
import {
|
||||
TranscriptionPayload,
|
||||
TranscriptionSchema,
|
||||
TranscriptPayloadSchema,
|
||||
} from './types';
|
||||
import { readStream } from './utils';
|
||||
|
||||
export type TranscriptionJob = {
|
||||
id: string;
|
||||
status: AiJobStatus;
|
||||
transcription?: TranscriptionPayload;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class CopilotTranscriptionService {
|
||||
constructor(
|
||||
private readonly models: Models,
|
||||
private readonly job: JobQueue,
|
||||
private readonly storage: CopilotStorage,
|
||||
private readonly prompt: PromptService,
|
||||
private readonly provider: CopilotProviderService
|
||||
) {}
|
||||
|
||||
async submitTranscriptionJob(
|
||||
userId: string,
|
||||
workspaceId: string,
|
||||
blobId: string,
|
||||
blob: FileUpload
|
||||
): Promise<TranscriptionJob> {
|
||||
if (await this.models.copilotJob.has(workspaceId, blobId)) {
|
||||
throw new CopilotTranscriptionJobExists();
|
||||
}
|
||||
|
||||
const { id: jobId, status } = await this.models.copilotJob.create({
|
||||
workspaceId,
|
||||
blobId,
|
||||
createdBy: userId,
|
||||
type: AiJobType.transcription,
|
||||
});
|
||||
|
||||
const buffer = await readStream(blob.createReadStream());
|
||||
const url = await this.storage.put(userId, workspaceId, blobId, buffer);
|
||||
|
||||
await this.models.copilotJob.update(jobId, {
|
||||
status: AiJobStatus.running,
|
||||
});
|
||||
|
||||
await this.job.add(
|
||||
'copilot.transcript.submit',
|
||||
{
|
||||
jobId,
|
||||
url,
|
||||
mimeType: blob.mimetype,
|
||||
},
|
||||
// retry 3 times
|
||||
{ removeOnFail: 3 }
|
||||
);
|
||||
|
||||
return { id: jobId, status };
|
||||
}
|
||||
|
||||
async claimTranscriptionJob(
|
||||
userId: string,
|
||||
jobId: string
|
||||
): Promise<TranscriptionJob | null> {
|
||||
const status = await this.models.copilotJob.claim(jobId, userId);
|
||||
if (status === AiJobStatus.claimed) {
|
||||
const transcription = await this.models.copilotJob.getPayload(
|
||||
jobId,
|
||||
TranscriptPayloadSchema
|
||||
);
|
||||
return { id: jobId, transcription, status };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async queryTranscriptionJob(
|
||||
userId: string,
|
||||
workspaceId: string,
|
||||
jobId: string
|
||||
) {
|
||||
const job = await this.models.copilotJob.getWithUser(
|
||||
userId,
|
||||
workspaceId,
|
||||
jobId,
|
||||
AiJobType.transcription
|
||||
);
|
||||
|
||||
if (!job) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const ret: TranscriptionJob = { id: job.id, status: job.status };
|
||||
|
||||
const payload = TranscriptPayloadSchema.safeParse(job.payload);
|
||||
if (payload.success) {
|
||||
ret.transcription = payload.data;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
private async getProvider(model: string): Promise<CopilotTextProvider> {
|
||||
let provider = await this.provider.getProviderByCapability(
|
||||
CopilotCapability.TextToText,
|
||||
model
|
||||
);
|
||||
|
||||
if (!provider) {
|
||||
throw new NoCopilotProviderAvailable();
|
||||
}
|
||||
|
||||
return provider;
|
||||
}
|
||||
|
||||
private async chatWithPrompt(
|
||||
promptName: string,
|
||||
message: Partial<PromptMessage>
|
||||
): Promise<string> {
|
||||
const prompt = await this.prompt.get(promptName);
|
||||
if (!prompt) {
|
||||
throw new CopilotPromptNotFound({ name: promptName });
|
||||
}
|
||||
|
||||
const provider = await this.getProvider(prompt.model);
|
||||
return provider.generateText(
|
||||
[...prompt.finish({}), { role: 'user', content: '', ...message }],
|
||||
prompt.model
|
||||
);
|
||||
}
|
||||
|
||||
private cleanupResponse(response: string): string {
|
||||
return response
|
||||
.replace(/```[\w\s]+\n/g, '')
|
||||
.replace(/\n```/g, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
@OnJob('copilot.transcript.submit')
|
||||
async transcriptAudio({
|
||||
jobId,
|
||||
url,
|
||||
mimeType,
|
||||
}: Jobs['copilot.transcript.submit']) {
|
||||
const result = await this.chatWithPrompt('Transcript audio', {
|
||||
attachments: [url],
|
||||
params: { mimetype: mimeType },
|
||||
});
|
||||
|
||||
const transcription = TranscriptionSchema.parse(
|
||||
JSON.parse(this.cleanupResponse(result))
|
||||
);
|
||||
await this.models.copilotJob.update(jobId, { payload: { transcription } });
|
||||
|
||||
await this.job.add(
|
||||
'copilot.summary.submit',
|
||||
{
|
||||
jobId,
|
||||
},
|
||||
// retry 3 times
|
||||
{ removeOnFail: 3 }
|
||||
);
|
||||
}
|
||||
|
||||
@OnJob('copilot.summary.submit')
|
||||
async summaryTranscription({ jobId }: Jobs['copilot.summary.submit']) {
|
||||
const payload = await this.models.copilotJob.getPayload(
|
||||
jobId,
|
||||
TranscriptPayloadSchema
|
||||
);
|
||||
if (payload.transcription) {
|
||||
const content = payload.transcription
|
||||
.map(t => t.transcription)
|
||||
.join('\n');
|
||||
|
||||
const result = await this.chatWithPrompt('Summary', { content });
|
||||
|
||||
payload.summary = this.cleanupResponse(result);
|
||||
await this.models.copilotJob.update(jobId, { payload });
|
||||
} else {
|
||||
await this.models.copilotJob.update(jobId, {
|
||||
status: AiJobStatus.failed,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
import { OneMB } from '../../../base';
|
||||
|
||||
const TranscriptionItemSchema = z.object({
|
||||
speaker: z.string(),
|
||||
start: z.string(),
|
||||
end: z.string(),
|
||||
transcription: z.string(),
|
||||
});
|
||||
|
||||
export const TranscriptionSchema = z.array(TranscriptionItemSchema);
|
||||
|
||||
export const TranscriptPayloadSchema = z.object({
|
||||
transcription: TranscriptionSchema.nullable().optional(),
|
||||
summary: z.string().nullable().optional(),
|
||||
});
|
||||
|
||||
export type TranscriptionItem = z.infer<typeof TranscriptionItemSchema>;
|
||||
export type Transcription = z.infer<typeof TranscriptionSchema>;
|
||||
export type TranscriptionPayload = z.infer<typeof TranscriptPayloadSchema>;
|
||||
|
||||
declare global {
|
||||
interface Jobs {
|
||||
'copilot.transcript.submit': {
|
||||
jobId: string;
|
||||
url: string;
|
||||
mimeType: string;
|
||||
};
|
||||
'copilot.summary.submit': {
|
||||
jobId: string;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const MAX_TRANSCRIPTION_SIZE = 50 * OneMB;
|
||||
@@ -0,0 +1,11 @@
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import { readBufferWithLimit } from '../../../base';
|
||||
import { MAX_TRANSCRIPTION_SIZE } from './types';
|
||||
|
||||
export function readStream(
|
||||
readable: Readable,
|
||||
maxSize = MAX_TRANSCRIPTION_SIZE
|
||||
): Promise<Buffer> {
|
||||
return readBufferWithLimit(readable, maxSize);
|
||||
}
|
||||
Reference in New Issue
Block a user