fix(server): transcript key & bump models (#15485)

#### PR Dependency Tree


* **PR #15485** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Upgraded built-in AI actions and chat model selection to Gemini 3.7
Flash.
* Transcript attachments now retain storage keys and support secure
presigned URLs during processing.
* Completed transcript results preserve attachment metadata for reliable
access.

* **Bug Fixes**
* Improved transcript handling across uploads, retries, and native
processing.
  * Added safeguards for invalid or missing attachment URLs.

* **Maintenance**
* Added a migration to backfill storage keys for existing transcript
records without altering their URLs.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-08-16 00:49:27 +08:00
committed by GitHub
parent 7b3a8afc87
commit ff1e3d9c94
14 changed files with 286 additions and 67 deletions
Generated
+3 -3
View File
@@ -2963,7 +2963,7 @@ dependencies = [
"libc",
"log",
"rustversion",
"windows-link 0.2.1",
"windows-link 0.1.3",
"windows-result 0.4.1",
]
@@ -4756,9 +4756,9 @@ dependencies = [
[[package]]
name = "llm_adapter"
version = "0.2.18"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49df35c253da1563c33f0733ef3a46da3194299807cbd7e03c48dfa12a89a72f"
checksum = "c17ef06aac72e31a4fae7937998029267c4d923a0671bf1a4062c96a1ab5fa70"
dependencies = [
"base64",
"jsonschema",
@@ -3,8 +3,8 @@
"name": "Transcript audio",
"action": "Transcript audio",
"managedRoute": {
"targets": ["gemini-3.5-flash-lite"],
"premiumTargets": ["gemini-3.6-flash"]
"targets": ["gemini-3.7-flash"],
"premiumTargets": ["gemini-3.7-flash"]
},
"config": {
"requireContent": false,
@@ -17,8 +17,8 @@
"name": "Transcript audio structured",
"action": "Transcript audio structured",
"managedRoute": {
"targets": ["gemini-3.5-flash-lite"],
"premiumTargets": ["gemini-3.6-flash"]
"targets": ["gemini-3.7-flash"],
"premiumTargets": ["gemini-3.7-flash"]
},
"config": {
"requireContent": false,
@@ -39,7 +39,7 @@
{
"name": "Generate a caption",
"action": "Generate a caption",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"config": {
"requireContent": false,
"requireAttachment": true
@@ -129,7 +129,7 @@
{
"name": "Explain this image",
"action": "Explain this image",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"config": {
"requireContent": false,
"requireAttachment": true
@@ -148,7 +148,7 @@
{
"name": "Explain this code",
"action": "Explain this code",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [
{
"role": "system",
@@ -163,7 +163,7 @@
{
"name": "Translate to",
"action": "Translate",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"params": {
"language": {
"default": "English",
@@ -256,7 +256,7 @@
{
"name": "Write a twitter about this",
"action": "Write a twitter about this",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [
{
"role": "system",
@@ -271,7 +271,7 @@
{
"name": "Write a poem about this",
"action": "Write a poem about this",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [
{
"role": "system",
@@ -301,7 +301,7 @@
{
"name": "Write outline",
"action": "Write outline",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [
{
"role": "system",
@@ -316,7 +316,7 @@
{
"name": "Change tone to",
"action": "Change tone",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"params": {
"tone": {
"default": "professional",
@@ -337,7 +337,7 @@
{
"name": "Brainstorm ideas about this",
"action": "Brainstorm ideas about this",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [
{
"role": "system",
@@ -386,7 +386,7 @@
{
"name": "Improve writing for it",
"action": "Improve writing for it",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [
{
"role": "system",
@@ -416,7 +416,7 @@
{
"name": "Fix spelling for it",
"action": "Fix spelling for it",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [
{
"role": "system",
@@ -476,7 +476,7 @@
{
"name": "Create headings",
"action": "Create headings",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [
{
"role": "system",
@@ -521,7 +521,7 @@
{
"name": "Make it longer",
"action": "Make it longer",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [
{
"role": "system",
@@ -536,7 +536,7 @@
{
"name": "Make it shorter",
"action": "Make it shorter",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [
{
"role": "system",
@@ -551,7 +551,7 @@
{
"name": "Continue writing",
"action": "Continue writing",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
"managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [
{
"role": "system",
@@ -711,8 +711,8 @@
},
{
"id": "gemini",
"modelId": "gemini-3.6-flash",
"displayName": "Gemini 3.6 Flash",
"modelId": "gemini-3.7-flash",
"displayName": "Gemini 3.7 Flash",
"minimumTier": "premium"
},
{
@@ -81,12 +81,12 @@ mod tests {
let response = llm_resolve_model_registry_variant(ModelRegistryResolveRequest {
backend_kind: Some("gemini_api".to_string()),
model_id: "gemini-3.6-flash".to_string(),
model_id: "gemini-3.7-flash".to_string(),
})
.unwrap();
assert_eq!(response.matched_by.as_deref(), Some("raw_model_id"));
assert_eq!(response.variant.unwrap().raw_model_id, "gemini-3.6-flash");
assert_eq!(response.variant.unwrap().raw_model_id, "gemini-3.7-flash");
}
#[test]
@@ -301,7 +301,7 @@ mod tests {
attachment_kinds: None,
attachment_source_kinds: None,
has_remote_attachments: None,
model_id: Some("gemini-3.6-flash".to_string()),
model_id: Some("gemini-3.7-flash".to_string()),
output_type: Some("image".to_string()),
},
})
@@ -210,7 +210,7 @@ mod tests {
fn should_materialize_structured_request_with_response_contract() {
let response = llm_build_canonical_structured_request(
serde_json::from_value::<CanonicalStructuredRequestContract>(json!({
"model": "gemini-3.6-flash",
"model": "gemini-3.7-flash",
"messages": [
{ "role": "user", "content": "hello" }
],
@@ -226,7 +226,7 @@ mod tests {
assert_eq!(
response,
json!({
"model": "gemini-3.6-flash",
"model": "gemini-3.7-flash",
"messages": [
{
"role": "user",
@@ -597,13 +597,13 @@ mod tests {
);
let transcript = built_in_prompt("Transcript audio structured").expect("transcript prompt");
assert_eq!(transcript.managed_targets, ["gemini-3.5-flash-lite"]);
assert_eq!(transcript.managed_targets, ["gemini-3.7-flash"]);
assert_eq!(
transcript
.managed_premium_targets
.as_deref()
.map(|targets| targets.iter().map(String::as_str).collect::<Vec<_>>()),
Some(vec!["gemini-3.6-flash"])
Some(vec!["gemini-3.7-flash"])
);
}
}
@@ -262,7 +262,7 @@ mod tests {
CopilotManagedTier::Premium
)
.unwrap(),
["gemini-3.6-flash"]
["gemini-3.7-flash"]
);
assert_eq!(
managed_targets(&slot("index.embedding").unwrap(), None, CopilotManagedTier::Standard).unwrap(),
@@ -326,7 +326,7 @@ mod tests {
id: "vertex".to_string(),
provider: "geminiVertex".to_string(),
enabled: true,
models: vec!["gemini-3.6-flash".to_string()],
models: vec!["gemini-3.7-flash".to_string()],
config: json!({ "project": "affine-us", "location": location }),
}
}
@@ -294,7 +294,7 @@ test('retryTask reuses failed task and queues a new action attempt', async t =>
} as never,
{} as never,
{
resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.5-flash-lite'),
resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.7-flash'),
} as never,
{} as never,
{} as never,
@@ -303,7 +303,7 @@ test('retryTask reuses failed task and queues a new action attempt', async t =>
const result = await service.retryTask('user-1', 'workspace-1', 'task-1');
t.is(result.status, AiJobStatus.running);
t.is(result?.status, AiJobStatus.running);
t.like(queuedJobs[0] as Record<string, unknown>, {
name: 'copilot.transcript.task.submit',
});
@@ -351,9 +351,7 @@ for (const status of ['ready', 'settled']) {
} as never,
{} as never,
{
resolveTranscriptionModel: Sinon.stub().resolves(
'gemini-3.5-flash-lite'
),
resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.7-flash'),
} as never,
{} as never,
{} as never,
@@ -403,7 +401,8 @@ test('transcriptTask runs native transcript recipe through action bridge when av
],
infos: [
{
url: 'data:image/png;base64,YXVkaW8=',
key: 'blob-1-0',
url: 'https://affine.fail/api/copilot/blob/user-1/workspace-1/blob-1-0',
mimeType: 'audio/opus',
index: 0,
},
@@ -428,7 +427,11 @@ test('transcriptTask runs native transcript recipe through action bridge when av
},
} as never,
{} as never,
{} as never,
{
presignGet: Sinon.stub().resolves(
'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test'
),
} as never,
{} as never,
createTranscriptPromptService() as never,
createSuccessfulTranscriptBridge('run-bridge', bridgeInputs) as never
@@ -447,6 +450,20 @@ test('transcriptTask runs native transcript recipe through action bridge when av
slot: 'transcript.audio',
builtInRouteId: 'Transcript audio structured',
});
t.deepEqual(
(
bridgeInputs[0] as {
nativeInput: { input: { infos: unknown[] } };
}
).nativeInput.input.infos,
[
{
url: 'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test',
mimeType: 'audio/opus',
index: 0,
},
]
);
const messages = (
bridgeInputs[0] as {
step: {
@@ -459,7 +476,10 @@ test('transcriptTask runs native transcript recipe through action bridge when av
infos: [{ mimeType: 'audio/opus', index: 0 }],
});
t.deepEqual(messages.at(-1)?.attachments, [
{ attachment: 'data:image/png;base64,YXVkaW8=', mimeType: 'audio/opus' },
{
attachment: 'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test',
mimeType: 'audio/opus',
},
]);
t.like(complete.firstCall.args[1], {
status: 'ready',
@@ -471,6 +491,7 @@ test('transcriptTask runs native transcript recipe through action bridge when av
complete.firstCall.args[1].protectedResult.normalizedTranscript,
'00:00:05 A: Kickoff'
);
t.deepEqual(complete.firstCall.args[1].protectedResult.infos, payload.infos);
});
test('transcriptTask fails task when native action bridge reports an error event', async t => {
@@ -5,6 +5,7 @@ import ava, { TestFn } from 'ava';
import { createTestingModule, type TestingModule } from '../../__tests__/utils';
import { Models } from '../../models';
import { BackfillPermissionProjection1765500000000 } from '../migrations/1765500000000-backfill-permission-projection';
import { BackfillTranscriptStorageKeys1786805802350 } from '../migrations/1786805802350-backfill-transcript-storage-keys';
interface Context {
module: TestingModule;
@@ -81,3 +82,44 @@ test('permission backfill repairs ownerless workspaces before runtime state proj
{ role: 'owner' }
);
});
test('transcript backfill adds stable keys without removing compatibility URLs', async t => {
const payload = {
sourceAudio: { blobId: 'blob-1' },
infos: [
{
url: 'https://affine.example/api/copilot/blob/user-1/workspace-1/blob-1',
mimeType: 'audio/m4a',
},
{
url: 'https://example.com/external.m4a',
mimeType: 'audio/m4a',
},
],
};
await t.context.db.aiTranscriptTask.create({
data: {
userId: 'user-1',
workspaceId: 'workspace-1',
blobId: 'blob-1',
status: 'failed',
recipeId: 'transcript.audio',
recipeVersion: 'v1',
inputSnapshot: payload,
protectedResult: payload,
},
});
await BackfillTranscriptStorageKeys1786805802350.up(t.context.db);
await BackfillTranscriptStorageKeys1786805802350.up(t.context.db);
const task = await t.context.db.aiTranscriptTask.findFirstOrThrow({
where: { blobId: 'blob-1' },
});
const expected = {
...payload,
infos: [{ ...payload.infos[0], key: 'blob-1' }, payload.infos[1]],
};
t.deepEqual(task.inputSnapshot, expected);
t.deepEqual(task.protectedResult, expected);
});
@@ -0,0 +1,96 @@
import { type Prisma, PrismaClient } from '@prisma/client';
const BATCH_SIZE = 100;
function isRecord(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === 'object' && !Array.isArray(value);
}
function backfillTranscriptStorageKeys(
payload: Prisma.JsonValue | null,
userId: string,
workspaceId: string
): Prisma.InputJsonValue | undefined {
if (!isRecord(payload) || !Array.isArray(payload.infos)) {
return;
}
const prefix = `/api/copilot/blob/${encodeURIComponent(userId)}/${encodeURIComponent(workspaceId)}/`;
let changed = false;
const infos = payload.infos.map(info => {
if (!isRecord(info) || info.key || typeof info.url !== 'string') {
return info;
}
try {
const url = new URL(info.url);
if (!url.pathname.startsWith(prefix)) {
return info;
}
const key = decodeURIComponent(url.pathname.slice(prefix.length));
if (!key) {
return info;
}
changed = true;
return { ...info, key };
} catch {
return info;
}
});
return changed ? ({ ...payload, infos } as Prisma.InputJsonValue) : undefined;
}
export class BackfillTranscriptStorageKeys1786805802350 {
static async up(db: PrismaClient) {
let cursor: string | undefined;
while (true) {
const tasks = await db.aiTranscriptTask.findMany({
select: {
id: true,
userId: true,
workspaceId: true,
inputSnapshot: true,
protectedResult: true,
},
orderBy: { id: 'asc' },
take: BATCH_SIZE,
...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}),
});
if (!tasks.length) {
return;
}
const updates = tasks.flatMap(task => {
const inputSnapshot = backfillTranscriptStorageKeys(
task.inputSnapshot,
task.userId,
task.workspaceId
);
const protectedResult = backfillTranscriptStorageKeys(
task.protectedResult,
task.userId,
task.workspaceId
);
if (!inputSnapshot && !protectedResult) {
return [];
}
return db.aiTranscriptTask.update({
where: { id: task.id },
data: {
...(inputSnapshot ? { inputSnapshot } : {}),
...(protectedResult ? { protectedResult } : {}),
},
});
});
if (updates.length) {
await db.$transaction(updates);
}
cursor = tasks.at(-1)?.id;
}
}
static async down(_db: PrismaClient) {}
}
@@ -6,3 +6,4 @@ export * from './1751966744168-correct-session-update-time';
export * from './1763800000000-rebuild-manticore-mixed-script-indexes';
export * from './1765500000000-backfill-permission-projection';
export * from './1765600000000-backfill-entitlement-projection';
export * from './1786805802350-backfill-transcript-storage-keys';
@@ -59,14 +59,31 @@ export class CopilotStorage {
): Promise<StorageRuntimeGetObjectResult> {
const name = `${userId}/${workspaceId}/${key}`;
if (signedUrl) {
const presigned = await this.rt.presignGet('copilot', name);
if (presigned) {
return { redirectUrl: presigned.url };
}
const redirectUrl = await this.presignGet(userId, workspaceId, key);
if (redirectUrl) return { redirectUrl };
}
return this.rt.getObject('copilot', name);
}
async presignGet(userId: string, workspaceId: string, key: string) {
return (
await this.rt.presignGet('copilot', `${userId}/${workspaceId}/${key}`)
)?.url;
}
keyFromUrl(userId: string, workspaceId: string, url: string) {
try {
const parsed = new URL(url);
const prefix = `/api/copilot/blob/${encodeURIComponent(userId)}/${encodeURIComponent(workspaceId)}/`;
if (parsed.pathname.startsWith(prefix)) {
return decodeURIComponent(parsed.pathname.slice(prefix.length));
}
} catch {
return;
}
return undefined;
}
@CallMetric('ai', 'blob_delete')
async delete(userId: string, workspaceId: string, key: string) {
await this.rt.deleteObject('copilot', `${userId}/${workspaceId}/${key}`);
@@ -18,7 +18,8 @@ export const LegacyTranscriptionSchema = z.array(
);
export const AudioBlobInfoSchema = z.object({
url: z.string(),
key: z.string().min(1).optional(),
url: z.string().min(1),
mimeType: z.string(),
index: z.number().int().nullable().optional(),
});
@@ -46,10 +46,6 @@ export class CopilotTranscriptionService {
private readonly realtime: RealtimePublisher
) {}
private parseTaskPayload(payload: unknown): TranscriptionPayloadV2 {
return TranscriptPayloadSchema.parse(payload);
}
private buildTaskPublicMeta(payload: TranscriptionPayloadV2) {
return {
sourceAudio: payload.sourceAudio,
@@ -68,13 +64,10 @@ export class CopilotTranscriptionService {
const infos: AudioBlobInfos = [];
for (const [idx, blob] of blobs.entries()) {
const buffer = await readStream(blob.createReadStream());
const url = await this.storage.put(
userId,
workspaceId,
`${blobId}-${idx}`,
buffer
);
const key = `${blobId}-${idx}`;
const url = await this.storage.put(userId, workspaceId, key, buffer);
infos.push({
key,
url,
mimeType: sniffMime(buffer, blob.mimetype) || blob.mimetype,
index: idx,
@@ -103,6 +96,47 @@ export class CopilotTranscriptionService {
} satisfies TranscriptionPayloadV2;
}
private async resolveAttachmentUrl(
userId: string,
workspaceId: string,
info: AudioBlobInfos[number]
) {
if (info.url.startsWith('data:')) {
return info.url;
}
const key =
info.key ?? this.storage.keyFromUrl(userId, workspaceId, info.url);
if (!key) {
throw new Error('Transcript attachment cannot be resolved');
}
const signedUrl = await this.storage.presignGet(userId, workspaceId, key);
if (!signedUrl) {
throw new Error('Transcript attachment signing is not configured');
}
return signedUrl;
}
private async materializePayload(
userId: string,
workspaceId: string,
payload: TranscriptionPayloadV2
) {
return {
...payload,
infos: payload.infos
? await Promise.all(
payload.infos.map(async info => ({
url: await this.resolveAttachmentUrl(userId, workspaceId, info),
mimeType: info.mimeType,
index: info.index,
}))
)
: payload.infos,
} satisfies TranscriptionPayloadV2;
}
private async buildTranscriptActionMessages(payload: TranscriptionPayloadV2) {
const prompt = await this.prompts.get(TRANSCRIPT_PROMPT_REF);
if (!prompt) {
@@ -118,13 +152,12 @@ export class CopilotTranscriptionService {
index: info.index ?? null,
})) ?? null,
};
const attachments =
payload.infos?.map(info => ({
role: 'user' as const,
content: `Audio attachment ${info.index ?? 0}`,
attachments: [{ attachment: info.url, mimeType: info.mimeType }],
params: { mimetype: info.mimeType },
})) ?? [];
const attachments = (payload.infos ?? []).map(info => ({
role: 'user' as const,
content: `Audio attachment ${info.index ?? 0}`,
attachments: [{ attachment: info.url, mimeType: info.mimeType }],
params: { mimetype: info.mimeType },
}));
return [
...this.prompts.finish(prompt, {
content: JSON.stringify(metadata),
@@ -205,7 +238,7 @@ export class CopilotTranscriptionService {
);
}
const payload = this.parseTaskPayload(task.protectedResult);
const payload = TranscriptPayloadSchema.parse(task.protectedResult);
await this.runtime.assertRoute(
'transcript.audio',
{},
@@ -287,14 +320,19 @@ export class CopilotTranscriptionService {
let bridgeFailed = false;
let bridgeError = 'transcript native recipe failed';
let finalResult: unknown = null;
const messages = await this.buildTranscriptActionMessages(payload);
const runtimePayload = await this.materializePayload(
task.userId,
task.workspaceId,
payload
);
const messages = await this.buildTranscriptActionMessages(runtimePayload);
for await (const event of this.actionBridge.runStream({
userId: task.userId,
workspaceId: task.workspaceId,
actionId: TRANSCRIPT_ACTION_ID,
actionVersion: TRANSCRIPT_ACTION_VERSION,
retryOf: retryOf ?? null,
inputSnapshot: payload,
inputSnapshot: runtimePayload,
onRunCreated: async ({ runId }) => {
await this.models.copilotTranscriptTask.markRunning(taskId, runId);
this.publishTaskChanged(
@@ -329,7 +367,10 @@ export class CopilotTranscriptionService {
if (bridgeFailed) {
throw new Error(bridgeError);
}
const parsedResult = TranscriptPayloadSchema.parse(finalResult);
const parsedResult = {
...TranscriptPayloadSchema.parse(finalResult),
infos: payload.infos,
} satisfies TranscriptionPayloadV2;
await this.models.copilotTranscriptTask.complete(taskId, {
status: 'ready',
actionRunId,