Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-18 10:36:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): audio transcription (#10733)

Browse Source
This commit is contained in:
darkskygit
2025-03-20 07:12:27 +00:00
parent bd5d930490
commit c16ae2d5b4
19 changed files with 712 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/server/src/plugins/copilot/providers/google.ts
+42 -43
View File
@@ -101,53 +101,52 @@ export class GoogleProvider implements CopilotTextToTextProvider {
return undefined;
}
protected chatToGPTMessage(
protected async chatToGPTMessage(
messages: PromptMessage[]
): [string | undefined, ChatMessage[]] {
): Promise<[string | undefined, ChatMessage[]]> {
let system =
messages[0]?.role === 'system' ? messages.shift()?.content : undefined;
// filter redundant fields
const msgs = messages
.filter(m => m.role !== 'system')
.map(({ role, content, attachments, params }) => {
content = content.trim();
role = role as 'user' | 'assistant';
const mimetype = params?.mimetype;
if (Array.isArray(attachments)) {
const contents: (TextPart | FilePart)[] = [];
if (content.length) {
contents.push({
type: 'text',
text: content,
});
}
contents.push(
...attachments
.map(url => {
if (SIMPLE_IMAGE_URL_REGEX.test(url)) {
const mimeType =
typeof mimetype === 'string'
? mimetype
: this.inferMimeType(url);
if (mimeType) {
const data = url.startsWith('data:') ? url : new URL(url);
return {
type: 'file' as const,
data,
mimeType,
};
}
}
return undefined;
})
.filter(c => !!c)
);
return { role, content: contents } as ChatMessage;
} else {
return { role, content } as ChatMessage;
const msgs: ChatMessage[] = [];
for (let { role, content, attachments, params } of messages.filter(
m => m.role !== 'system'
)) {
content = content.trim();
role = role as 'user' | 'assistant';
const mimetype = params?.mimetype;
if (Array.isArray(attachments)) {
const contents: (TextPart | FilePart)[] = [];
if (content.length) {
contents.push({
type: 'text',
text: content,
});
}
});
for (const url of attachments) {
if (SIMPLE_IMAGE_URL_REGEX.test(url)) {
const mimeType =
typeof mimetype === 'string' ? mimetype : this.inferMimeType(url);
if (mimeType) {
const data = url.startsWith('data:')
? await fetch(url).then(r => r.arrayBuffer())
: new URL(url);
contents.push({
type: 'file' as const,
data,
mimeType,
});
}
}
}
msgs.push({ role, content: contents } as ChatMessage);
} else {
msgs.push({ role, content });
}
}
return [system, msgs];
}
@@ -237,7 +236,7 @@ export class GoogleProvider implements CopilotTextToTextProvider {
try {
metrics.ai.counter('chat_text_calls').add(1, { model });
const [system, msgs] = this.chatToGPTMessage(messages);
const [system, msgs] = await this.chatToGPTMessage(messages);
const { text } = await generateText({
model: this.instance(model, {
@@ -266,7 +265,7 @@ export class GoogleProvider implements CopilotTextToTextProvider {
try {
metrics.ai.counter('chat_text_stream_calls').add(1, { model });
const [system, msgs] = this.chatToGPTMessage(messages);
const [system, msgs] = await this.chatToGPTMessage(messages);
const { textStream } = streamText({
model: this.instance(model),