diff --git a/Cargo.lock b/Cargo.lock index 66880f7d20..530ee2bfe9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2963,7 +2963,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows-link 0.2.1", + "windows-link 0.1.3", "windows-result 0.4.1", ] @@ -4756,9 +4756,9 @@ dependencies = [ [[package]] name = "llm_adapter" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df35c253da1563c33f0733ef3a46da3194299807cbd7e03c48dfa12a89a72f" +checksum = "c17ef06aac72e31a4fae7937998029267c4d923a0671bf1a4062c96a1ab5fa70" dependencies = [ "base64", "jsonschema", diff --git a/packages/backend/native/src/llm/assets/prompts/built-in.json b/packages/backend/native/src/llm/assets/prompts/built-in.json index 97f4ad5dd6..9cb8e1e30c 100644 --- a/packages/backend/native/src/llm/assets/prompts/built-in.json +++ b/packages/backend/native/src/llm/assets/prompts/built-in.json @@ -3,8 +3,8 @@ "name": "Transcript audio", "action": "Transcript audio", "managedRoute": { - "targets": ["gemini-3.5-flash-lite"], - "premiumTargets": ["gemini-3.6-flash"] + "targets": ["gemini-3.7-flash"], + "premiumTargets": ["gemini-3.7-flash"] }, "config": { "requireContent": false, @@ -17,8 +17,8 @@ "name": "Transcript audio structured", "action": "Transcript audio structured", "managedRoute": { - "targets": ["gemini-3.5-flash-lite"], - "premiumTargets": ["gemini-3.6-flash"] + "targets": ["gemini-3.7-flash"], + "premiumTargets": ["gemini-3.7-flash"] }, "config": { "requireContent": false, @@ -39,7 +39,7 @@ { "name": "Generate a caption", "action": "Generate a caption", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "config": { "requireContent": false, "requireAttachment": true @@ -129,7 +129,7 @@ { "name": "Explain this image", "action": "Explain this image", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "config": { "requireContent": false, "requireAttachment": true @@ -148,7 +148,7 @@ { "name": "Explain this code", "action": "Explain this code", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "messages": [ { "role": "system", @@ -163,7 +163,7 @@ { "name": "Translate to", "action": "Translate", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "params": { "language": { "default": "English", @@ -256,7 +256,7 @@ { "name": "Write a twitter about this", "action": "Write a twitter about this", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "messages": [ { "role": "system", @@ -271,7 +271,7 @@ { "name": "Write a poem about this", "action": "Write a poem about this", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "messages": [ { "role": "system", @@ -301,7 +301,7 @@ { "name": "Write outline", "action": "Write outline", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "messages": [ { "role": "system", @@ -316,7 +316,7 @@ { "name": "Change tone to", "action": "Change tone", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "params": { "tone": { "default": "professional", @@ -337,7 +337,7 @@ { "name": "Brainstorm ideas about this", "action": "Brainstorm ideas about this", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "messages": [ { "role": "system", @@ -386,7 +386,7 @@ { "name": "Improve writing for it", "action": "Improve writing for it", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "messages": [ { "role": "system", @@ -416,7 +416,7 @@ { "name": "Fix spelling for it", "action": "Fix spelling for it", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "messages": [ { "role": "system", @@ -476,7 +476,7 @@ { "name": "Create headings", "action": "Create headings", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "messages": [ { "role": "system", @@ -521,7 +521,7 @@ { "name": "Make it longer", "action": "Make it longer", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "messages": [ { "role": "system", @@ -536,7 +536,7 @@ { "name": "Make it shorter", "action": "Make it shorter", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "messages": [ { "role": "system", @@ -551,7 +551,7 @@ { "name": "Continue writing", "action": "Continue writing", - "managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, + "managedRoute": { "targets": ["gemini-3.7-flash"] }, "messages": [ { "role": "system", @@ -711,8 +711,8 @@ }, { "id": "gemini", - "modelId": "gemini-3.6-flash", - "displayName": "Gemini 3.6 Flash", + "modelId": "gemini-3.7-flash", + "displayName": "Gemini 3.7 Flash", "minimumTier": "premium" }, { diff --git a/packages/backend/native/src/llm/core/model_registry.rs b/packages/backend/native/src/llm/core/model_registry.rs index 1c9bf5b20d..df2381e29b 100644 --- a/packages/backend/native/src/llm/core/model_registry.rs +++ b/packages/backend/native/src/llm/core/model_registry.rs @@ -81,12 +81,12 @@ mod tests { let response = llm_resolve_model_registry_variant(ModelRegistryResolveRequest { backend_kind: Some("gemini_api".to_string()), - model_id: "gemini-3.6-flash".to_string(), + model_id: "gemini-3.7-flash".to_string(), }) .unwrap(); assert_eq!(response.matched_by.as_deref(), Some("raw_model_id")); - assert_eq!(response.variant.unwrap().raw_model_id, "gemini-3.6-flash"); + assert_eq!(response.variant.unwrap().raw_model_id, "gemini-3.7-flash"); } #[test] @@ -301,7 +301,7 @@ mod tests { attachment_kinds: None, attachment_source_kinds: None, has_remote_attachments: None, - model_id: Some("gemini-3.6-flash".to_string()), + model_id: Some("gemini-3.7-flash".to_string()), output_type: Some("image".to_string()), }, }) diff --git a/packages/backend/native/src/llm/core/request_builder/mod.rs b/packages/backend/native/src/llm/core/request_builder/mod.rs index b2baa0156e..e499306ebd 100644 --- a/packages/backend/native/src/llm/core/request_builder/mod.rs +++ b/packages/backend/native/src/llm/core/request_builder/mod.rs @@ -210,7 +210,7 @@ mod tests { fn should_materialize_structured_request_with_response_contract() { let response = llm_build_canonical_structured_request( serde_json::from_value::(json!({ - "model": "gemini-3.6-flash", + "model": "gemini-3.7-flash", "messages": [ { "role": "user", "content": "hello" } ], @@ -226,7 +226,7 @@ mod tests { assert_eq!( response, json!({ - "model": "gemini-3.6-flash", + "model": "gemini-3.7-flash", "messages": [ { "role": "user", diff --git a/packages/backend/native/src/llm/prompt_catalog.rs b/packages/backend/native/src/llm/prompt_catalog.rs index ef16f4654e..b486b91675 100644 --- a/packages/backend/native/src/llm/prompt_catalog.rs +++ b/packages/backend/native/src/llm/prompt_catalog.rs @@ -597,13 +597,13 @@ mod tests { ); let transcript = built_in_prompt("Transcript audio structured").expect("transcript prompt"); - assert_eq!(transcript.managed_targets, ["gemini-3.5-flash-lite"]); + assert_eq!(transcript.managed_targets, ["gemini-3.7-flash"]); assert_eq!( transcript .managed_premium_targets .as_deref() .map(|targets| targets.iter().map(String::as_str).collect::>()), - Some(vec!["gemini-3.6-flash"]) + Some(vec!["gemini-3.7-flash"]) ); } } diff --git a/packages/backend/native/src/llm/route/catalog.rs b/packages/backend/native/src/llm/route/catalog.rs index 79dd7b75b9..f1f831ee73 100644 --- a/packages/backend/native/src/llm/route/catalog.rs +++ b/packages/backend/native/src/llm/route/catalog.rs @@ -262,7 +262,7 @@ mod tests { CopilotManagedTier::Premium ) .unwrap(), - ["gemini-3.6-flash"] + ["gemini-3.7-flash"] ); assert_eq!( managed_targets(&slot("index.embedding").unwrap(), None, CopilotManagedTier::Standard).unwrap(), diff --git a/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs b/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs index 05d6b977ec..bb3c06b42d 100644 --- a/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs +++ b/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs @@ -326,7 +326,7 @@ mod tests { id: "vertex".to_string(), provider: "geminiVertex".to_string(), enabled: true, - models: vec!["gemini-3.6-flash".to_string()], + models: vec!["gemini-3.7-flash".to_string()], config: json!({ "project": "affine-us", "location": location }), } } diff --git a/packages/backend/server/src/__tests__/copilot/transcript-contract.spec.ts b/packages/backend/server/src/__tests__/copilot/transcript-contract.spec.ts index 8f2cfaca93..e77a4456d3 100644 --- a/packages/backend/server/src/__tests__/copilot/transcript-contract.spec.ts +++ b/packages/backend/server/src/__tests__/copilot/transcript-contract.spec.ts @@ -294,7 +294,7 @@ test('retryTask reuses failed task and queues a new action attempt', async t => } as never, {} as never, { - resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.5-flash-lite'), + resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.7-flash'), } as never, {} as never, {} as never, @@ -303,7 +303,7 @@ test('retryTask reuses failed task and queues a new action attempt', async t => const result = await service.retryTask('user-1', 'workspace-1', 'task-1'); - t.is(result.status, AiJobStatus.running); + t.is(result?.status, AiJobStatus.running); t.like(queuedJobs[0] as Record, { name: 'copilot.transcript.task.submit', }); @@ -351,9 +351,7 @@ for (const status of ['ready', 'settled']) { } as never, {} as never, { - resolveTranscriptionModel: Sinon.stub().resolves( - 'gemini-3.5-flash-lite' - ), + resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.7-flash'), } as never, {} as never, {} as never, @@ -403,7 +401,8 @@ test('transcriptTask runs native transcript recipe through action bridge when av ], infos: [ { - url: 'data:image/png;base64,YXVkaW8=', + key: 'blob-1-0', + url: 'https://affine.fail/api/copilot/blob/user-1/workspace-1/blob-1-0', mimeType: 'audio/opus', index: 0, }, @@ -428,7 +427,11 @@ test('transcriptTask runs native transcript recipe through action bridge when av }, } as never, {} as never, - {} as never, + { + presignGet: Sinon.stub().resolves( + 'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test' + ), + } as never, {} as never, createTranscriptPromptService() as never, createSuccessfulTranscriptBridge('run-bridge', bridgeInputs) as never @@ -447,6 +450,20 @@ test('transcriptTask runs native transcript recipe through action bridge when av slot: 'transcript.audio', builtInRouteId: 'Transcript audio structured', }); + t.deepEqual( + ( + bridgeInputs[0] as { + nativeInput: { input: { infos: unknown[] } }; + } + ).nativeInput.input.infos, + [ + { + url: 'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test', + mimeType: 'audio/opus', + index: 0, + }, + ] + ); const messages = ( bridgeInputs[0] as { step: { @@ -459,7 +476,10 @@ test('transcriptTask runs native transcript recipe through action bridge when av infos: [{ mimeType: 'audio/opus', index: 0 }], }); t.deepEqual(messages.at(-1)?.attachments, [ - { attachment: 'data:image/png;base64,YXVkaW8=', mimeType: 'audio/opus' }, + { + attachment: 'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test', + mimeType: 'audio/opus', + }, ]); t.like(complete.firstCall.args[1], { status: 'ready', @@ -471,6 +491,7 @@ test('transcriptTask runs native transcript recipe through action bridge when av complete.firstCall.args[1].protectedResult.normalizedTranscript, '00:00:05 A: Kickoff' ); + t.deepEqual(complete.firstCall.args[1].protectedResult.infos, payload.infos); }); test('transcriptTask fails task when native action bridge reports an error event', async t => { diff --git a/packages/backend/server/src/data/__tests__/migrations.spec.ts b/packages/backend/server/src/data/__tests__/migrations.spec.ts index d491e9d74a..7d77fb11b5 100644 --- a/packages/backend/server/src/data/__tests__/migrations.spec.ts +++ b/packages/backend/server/src/data/__tests__/migrations.spec.ts @@ -5,6 +5,7 @@ import ava, { TestFn } from 'ava'; import { createTestingModule, type TestingModule } from '../../__tests__/utils'; import { Models } from '../../models'; import { BackfillPermissionProjection1765500000000 } from '../migrations/1765500000000-backfill-permission-projection'; +import { BackfillTranscriptStorageKeys1786805802350 } from '../migrations/1786805802350-backfill-transcript-storage-keys'; interface Context { module: TestingModule; @@ -81,3 +82,44 @@ test('permission backfill repairs ownerless workspaces before runtime state proj { role: 'owner' } ); }); + +test('transcript backfill adds stable keys without removing compatibility URLs', async t => { + const payload = { + sourceAudio: { blobId: 'blob-1' }, + infos: [ + { + url: 'https://affine.example/api/copilot/blob/user-1/workspace-1/blob-1', + mimeType: 'audio/m4a', + }, + { + url: 'https://example.com/external.m4a', + mimeType: 'audio/m4a', + }, + ], + }; + await t.context.db.aiTranscriptTask.create({ + data: { + userId: 'user-1', + workspaceId: 'workspace-1', + blobId: 'blob-1', + status: 'failed', + recipeId: 'transcript.audio', + recipeVersion: 'v1', + inputSnapshot: payload, + protectedResult: payload, + }, + }); + + await BackfillTranscriptStorageKeys1786805802350.up(t.context.db); + await BackfillTranscriptStorageKeys1786805802350.up(t.context.db); + + const task = await t.context.db.aiTranscriptTask.findFirstOrThrow({ + where: { blobId: 'blob-1' }, + }); + const expected = { + ...payload, + infos: [{ ...payload.infos[0], key: 'blob-1' }, payload.infos[1]], + }; + t.deepEqual(task.inputSnapshot, expected); + t.deepEqual(task.protectedResult, expected); +}); diff --git a/packages/backend/server/src/data/migrations/1786805802350-backfill-transcript-storage-keys.ts b/packages/backend/server/src/data/migrations/1786805802350-backfill-transcript-storage-keys.ts new file mode 100644 index 0000000000..bf35af52ca --- /dev/null +++ b/packages/backend/server/src/data/migrations/1786805802350-backfill-transcript-storage-keys.ts @@ -0,0 +1,96 @@ +import { type Prisma, PrismaClient } from '@prisma/client'; + +const BATCH_SIZE = 100; + +function isRecord(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +function backfillTranscriptStorageKeys( + payload: Prisma.JsonValue | null, + userId: string, + workspaceId: string +): Prisma.InputJsonValue | undefined { + if (!isRecord(payload) || !Array.isArray(payload.infos)) { + return; + } + + const prefix = `/api/copilot/blob/${encodeURIComponent(userId)}/${encodeURIComponent(workspaceId)}/`; + let changed = false; + const infos = payload.infos.map(info => { + if (!isRecord(info) || info.key || typeof info.url !== 'string') { + return info; + } + + try { + const url = new URL(info.url); + if (!url.pathname.startsWith(prefix)) { + return info; + } + const key = decodeURIComponent(url.pathname.slice(prefix.length)); + if (!key) { + return info; + } + changed = true; + return { ...info, key }; + } catch { + return info; + } + }); + + return changed ? ({ ...payload, infos } as Prisma.InputJsonValue) : undefined; +} + +export class BackfillTranscriptStorageKeys1786805802350 { + static async up(db: PrismaClient) { + let cursor: string | undefined; + + while (true) { + const tasks = await db.aiTranscriptTask.findMany({ + select: { + id: true, + userId: true, + workspaceId: true, + inputSnapshot: true, + protectedResult: true, + }, + orderBy: { id: 'asc' }, + take: BATCH_SIZE, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + }); + if (!tasks.length) { + return; + } + + const updates = tasks.flatMap(task => { + const inputSnapshot = backfillTranscriptStorageKeys( + task.inputSnapshot, + task.userId, + task.workspaceId + ); + const protectedResult = backfillTranscriptStorageKeys( + task.protectedResult, + task.userId, + task.workspaceId + ); + if (!inputSnapshot && !protectedResult) { + return []; + } + return db.aiTranscriptTask.update({ + where: { id: task.id }, + data: { + ...(inputSnapshot ? { inputSnapshot } : {}), + ...(protectedResult ? { protectedResult } : {}), + }, + }); + }); + + if (updates.length) { + await db.$transaction(updates); + } + cursor = tasks.at(-1)?.id; + } + } + + static async down(_db: PrismaClient) {} +} diff --git a/packages/backend/server/src/data/migrations/index.ts b/packages/backend/server/src/data/migrations/index.ts index 390b9fc149..f5fc4e3dbe 100644 --- a/packages/backend/server/src/data/migrations/index.ts +++ b/packages/backend/server/src/data/migrations/index.ts @@ -6,3 +6,4 @@ export * from './1751966744168-correct-session-update-time'; export * from './1763800000000-rebuild-manticore-mixed-script-indexes'; export * from './1765500000000-backfill-permission-projection'; export * from './1765600000000-backfill-entitlement-projection'; +export * from './1786805802350-backfill-transcript-storage-keys'; diff --git a/packages/backend/server/src/plugins/copilot/storage.ts b/packages/backend/server/src/plugins/copilot/storage.ts index 27189654f7..e7f3b58918 100644 --- a/packages/backend/server/src/plugins/copilot/storage.ts +++ b/packages/backend/server/src/plugins/copilot/storage.ts @@ -59,14 +59,31 @@ export class CopilotStorage { ): Promise { const name = `${userId}/${workspaceId}/${key}`; if (signedUrl) { - const presigned = await this.rt.presignGet('copilot', name); - if (presigned) { - return { redirectUrl: presigned.url }; - } + const redirectUrl = await this.presignGet(userId, workspaceId, key); + if (redirectUrl) return { redirectUrl }; } return this.rt.getObject('copilot', name); } + async presignGet(userId: string, workspaceId: string, key: string) { + return ( + await this.rt.presignGet('copilot', `${userId}/${workspaceId}/${key}`) + )?.url; + } + + keyFromUrl(userId: string, workspaceId: string, url: string) { + try { + const parsed = new URL(url); + const prefix = `/api/copilot/blob/${encodeURIComponent(userId)}/${encodeURIComponent(workspaceId)}/`; + if (parsed.pathname.startsWith(prefix)) { + return decodeURIComponent(parsed.pathname.slice(prefix.length)); + } + } catch { + return; + } + return undefined; + } + @CallMetric('ai', 'blob_delete') async delete(userId: string, workspaceId: string, key: string) { await this.rt.deleteObject('copilot', `${userId}/${workspaceId}/${key}`); diff --git a/packages/backend/server/src/plugins/copilot/transcript/schema.ts b/packages/backend/server/src/plugins/copilot/transcript/schema.ts index 6bb8b21198..a1faeca1d2 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/schema.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/schema.ts @@ -18,7 +18,8 @@ export const LegacyTranscriptionSchema = z.array( ); export const AudioBlobInfoSchema = z.object({ - url: z.string(), + key: z.string().min(1).optional(), + url: z.string().min(1), mimeType: z.string(), index: z.number().int().nullable().optional(), }); diff --git a/packages/backend/server/src/plugins/copilot/transcript/service.ts b/packages/backend/server/src/plugins/copilot/transcript/service.ts index 2a6557cd59..4998ef2840 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/service.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/service.ts @@ -46,10 +46,6 @@ export class CopilotTranscriptionService { private readonly realtime: RealtimePublisher ) {} - private parseTaskPayload(payload: unknown): TranscriptionPayloadV2 { - return TranscriptPayloadSchema.parse(payload); - } - private buildTaskPublicMeta(payload: TranscriptionPayloadV2) { return { sourceAudio: payload.sourceAudio, @@ -68,13 +64,10 @@ export class CopilotTranscriptionService { const infos: AudioBlobInfos = []; for (const [idx, blob] of blobs.entries()) { const buffer = await readStream(blob.createReadStream()); - const url = await this.storage.put( - userId, - workspaceId, - `${blobId}-${idx}`, - buffer - ); + const key = `${blobId}-${idx}`; + const url = await this.storage.put(userId, workspaceId, key, buffer); infos.push({ + key, url, mimeType: sniffMime(buffer, blob.mimetype) || blob.mimetype, index: idx, @@ -103,6 +96,47 @@ export class CopilotTranscriptionService { } satisfies TranscriptionPayloadV2; } + private async resolveAttachmentUrl( + userId: string, + workspaceId: string, + info: AudioBlobInfos[number] + ) { + if (info.url.startsWith('data:')) { + return info.url; + } + + const key = + info.key ?? this.storage.keyFromUrl(userId, workspaceId, info.url); + if (!key) { + throw new Error('Transcript attachment cannot be resolved'); + } + + const signedUrl = await this.storage.presignGet(userId, workspaceId, key); + if (!signedUrl) { + throw new Error('Transcript attachment signing is not configured'); + } + return signedUrl; + } + + private async materializePayload( + userId: string, + workspaceId: string, + payload: TranscriptionPayloadV2 + ) { + return { + ...payload, + infos: payload.infos + ? await Promise.all( + payload.infos.map(async info => ({ + url: await this.resolveAttachmentUrl(userId, workspaceId, info), + mimeType: info.mimeType, + index: info.index, + })) + ) + : payload.infos, + } satisfies TranscriptionPayloadV2; + } + private async buildTranscriptActionMessages(payload: TranscriptionPayloadV2) { const prompt = await this.prompts.get(TRANSCRIPT_PROMPT_REF); if (!prompt) { @@ -118,13 +152,12 @@ export class CopilotTranscriptionService { index: info.index ?? null, })) ?? null, }; - const attachments = - payload.infos?.map(info => ({ - role: 'user' as const, - content: `Audio attachment ${info.index ?? 0}`, - attachments: [{ attachment: info.url, mimeType: info.mimeType }], - params: { mimetype: info.mimeType }, - })) ?? []; + const attachments = (payload.infos ?? []).map(info => ({ + role: 'user' as const, + content: `Audio attachment ${info.index ?? 0}`, + attachments: [{ attachment: info.url, mimeType: info.mimeType }], + params: { mimetype: info.mimeType }, + })); return [ ...this.prompts.finish(prompt, { content: JSON.stringify(metadata), @@ -205,7 +238,7 @@ export class CopilotTranscriptionService { ); } - const payload = this.parseTaskPayload(task.protectedResult); + const payload = TranscriptPayloadSchema.parse(task.protectedResult); await this.runtime.assertRoute( 'transcript.audio', {}, @@ -287,14 +320,19 @@ export class CopilotTranscriptionService { let bridgeFailed = false; let bridgeError = 'transcript native recipe failed'; let finalResult: unknown = null; - const messages = await this.buildTranscriptActionMessages(payload); + const runtimePayload = await this.materializePayload( + task.userId, + task.workspaceId, + payload + ); + const messages = await this.buildTranscriptActionMessages(runtimePayload); for await (const event of this.actionBridge.runStream({ userId: task.userId, workspaceId: task.workspaceId, actionId: TRANSCRIPT_ACTION_ID, actionVersion: TRANSCRIPT_ACTION_VERSION, retryOf: retryOf ?? null, - inputSnapshot: payload, + inputSnapshot: runtimePayload, onRunCreated: async ({ runId }) => { await this.models.copilotTranscriptTask.markRunning(taskId, runId); this.publishTaskChanged( @@ -329,7 +367,10 @@ export class CopilotTranscriptionService { if (bridgeFailed) { throw new Error(bridgeError); } - const parsedResult = TranscriptPayloadSchema.parse(finalResult); + const parsedResult = { + ...TranscriptPayloadSchema.parse(finalResult), + infos: payload.infos, + } satisfies TranscriptionPayloadV2; await this.models.copilotTranscriptTask.complete(taskId, { status: 'ready', actionRunId,