feat(server): realtime handle & migration (#15487)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
  * Retry failed Copilot transcription tasks in real time.
  * Retried tasks resume processing and report updated status.
* Managed Copilot provider models can be omitted to use provider
defaults.

* **Bug Fixes**
* Improved handling of incomplete BYOK profiles, including safe
replacement of legacy records.
  * Duplicate profile creation now returns a clear validation error.
* Improved transcript processing reliability by preventing duplicate or
stale dispatches.

* **Migration**
* Consolidated legacy managed-provider settings while preserving
existing profiles and defaults.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/plugins/copilot/config.ts

@@ -35,7 +35,7 @@ type CopilotProviderProfileCommon = {
   displayName?: string;
   priority?: number;
   enabled?: boolean;
-  models: string[];
+  models?: string[];
   middleware?: ProviderMiddlewareConfig;
 };
 

packages/backend/server/src/plugins/copilot/cron.ts

@@ -3,6 +3,7 @@ import { Cron, CronExpression } from '@nestjs/schedule';
 
 import { JobQueue, OneDay, OnJob } from '../../base';
 import { Models } from '../../models';
+import { CopilotTranscriptionRetryService } from './transcript/retry';
 
 const BACKGROUND_COPILOT_JOB_PRIORITY = 100;
 
@@ -19,9 +20,15 @@ export class CopilotCronJobs {
 
   constructor(
     private readonly models: Models,
-    private readonly jobs: JobQueue
+    private readonly jobs: JobQueue,
+    private readonly transcriptRetry: CopilotTranscriptionRetryService
   ) {}
 
+  @Cron(CronExpression.EVERY_MINUTE)
+  async reconcileTranscriptDispatches() {
+    await this.transcriptRetry.reconcileDispatches();
+  }
+
   @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
   async dailyCleanupJob() {
     await this.jobs.add(

packages/backend/server/src/plugins/copilot/module-providers.ts

@@ -42,6 +42,7 @@ import { CopilotStorage } from './storage';
 import {
   CopilotTranscriptionReader,
   CopilotTranscriptionResolver,
+  CopilotTranscriptionRetryService,
   CopilotTranscriptionService,
   CopilotTranscriptRealtimeProvider,
 } from './transcript';
@@ -89,6 +90,7 @@ export const COPILOT_RUNTIME_PROVIDERS = [
 
 export const COPILOT_TRANSCRIPT_REALTIME_PROVIDERS = [
   CopilotTranscriptionReader,
+  CopilotTranscriptionRetryService,
   CopilotTranscriptRealtimeProvider,
   CopilotEmbeddingRealtimeProvider,
   DelegatedEditorRealtimeProvider,

packages/backend/server/src/plugins/copilot/transcript/constants.ts

@@ -0,0 +1,3 @@
+export const TRANSCRIPT_ACTION_ID = 'transcript.audio';
+export const TRANSCRIPT_PROMPT_REF = 'Transcript audio structured';
+export const TRANSCRIPT_ACTION_VERSION = 'v1';

packages/backend/server/src/plugins/copilot/transcript/index.ts

@@ -2,4 +2,5 @@ export type { TranscriptionJob } from './job';
 export { CopilotTranscriptionReader } from './reader';
 export { CopilotTranscriptRealtimeProvider } from './realtime';
 export { CopilotTranscriptionResolver } from './resolver';
+export { CopilotTranscriptionRetryService } from './retry';
 export { CopilotTranscriptionService } from './service';

packages/backend/server/src/plugins/copilot/transcript/realtime.ts

@@ -11,12 +11,14 @@ import {
 } from '../../../core/realtime';
 import { assertCopilotEnabled } from '../availability';
 import { CopilotTranscriptionReader } from './reader';
+import { CopilotTranscriptionRetryService } from './retry';
 
 @Injectable()
 export class CopilotTranscriptRealtimeProvider implements OnModuleInit {
   constructor(
     private readonly ac: PermissionAccess,
     private readonly transcript: CopilotTranscriptionReader,
+    private readonly retry: CopilotTranscriptionRetryService,
     private readonly registry: RealtimeRegistry,
     private readonly config: Config
   ) {}
@@ -34,6 +36,24 @@ export class CopilotTranscriptRealtimeProvider implements OnModuleInit {
       taskId: z.string(),
     });
 
+    this.registry.registerRequest({
+      name: 'copilot.transcript.task.retry',
+      input: z.object({
+        workspaceId: z.string(),
+        taskId: z.string(),
+      }),
+      handle: async (user, input) => {
+        await this.assertCopilot(user.id, input.workspaceId);
+        return {
+          task: await this.retry.retryTask(
+            user.id,
+            input.workspaceId,
+            input.taskId
+          ),
+        };
+      },
+    });
+
     registerRealtimeLiveQuery(this.registry, {
       request: {
         name: 'copilot.transcript.task.get',

packages/backend/server/src/plugins/copilot/transcript/retry.ts

@@ -0,0 +1,186 @@
+import { randomUUID } from 'node:crypto';
+
+import { BadRequestException, Injectable, Logger } from '@nestjs/common';
+import { AiJobStatus } from '@prisma/client';
+
+import {
+  CopilotTranscriptionJobNotFound,
+  JobQueue,
+  OneHour,
+  OneMinute,
+} from '../../../base';
+import {
+  RealtimePublisher,
+  realtimeTranscriptTaskRoom,
+} from '../../../core/realtime';
+import { Models } from '../../../models';
+import { CapabilityRuntime } from '../runtime/capability-runtime';
+import { TRANSCRIPT_PROMPT_REF } from './constants';
+import { TranscriptPayloadSchema } from './schema';
+
+@Injectable()
+export class CopilotTranscriptionRetryService {
+  private readonly logger = new Logger(CopilotTranscriptionRetryService.name);
+
+  constructor(
+    private readonly models: Models,
+    private readonly job: JobQueue,
+    private readonly runtime: CapabilityRuntime,
+    private readonly realtime: RealtimePublisher
+  ) {}
+
+  async retryTask(userId: string, workspaceId: string, taskId: string) {
+    const task = await this.models.copilotTranscriptTask.getWithUser(
+      userId,
+      workspaceId,
+      taskId
+    );
+    if (!task) {
+      throw new CopilotTranscriptionJobNotFound();
+    }
+    if (task.status === 'ready' || task.status === 'settled') {
+      throw new BadRequestException(
+        'Ready or settled transcript tasks cannot be retried'
+      );
+    }
+    if (task.status !== 'failed') {
+      throw new BadRequestException(
+        'Only failed transcript tasks can be retried'
+      );
+    }
+
+    const payload = TranscriptPayloadSchema.parse(task.protectedResult);
+    await this.runtime.assertRoute(
+      'transcript.audio',
+      {},
+      {
+        user: userId,
+        workspace: workspaceId,
+        featureKind: 'transcript',
+        builtInRouteId: TRANSCRIPT_PROMPT_REF,
+      }
+    );
+    const generation = randomUUID();
+    const retryOf = task.actionRunId ?? null;
+    const claimed = await this.models.copilotTranscriptTask.claimRetry(
+      taskId,
+      userId,
+      workspaceId,
+      retryOf,
+      generation
+    );
+    if (!claimed) {
+      throw new BadRequestException(
+        'Only failed transcript tasks can be retried'
+      );
+    }
+    await this.enqueuePendingTask(taskId, payload, generation, retryOf);
+    this.realtime.publish(
+      'copilot.transcript.task.changed',
+      { workspaceId, taskId },
+      { taskId, status: AiJobStatus.pending },
+      { room: realtimeTranscriptTaskRoom(workspaceId, taskId) }
+    );
+    return {
+      id: taskId,
+      status: AiJobStatus.pending,
+      infos: payload.infos ?? undefined,
+    };
+  }
+
+  async enqueuePendingTask(
+    taskId: string,
+    payload: Jobs['copilot.transcript.task.submit']['payload'],
+    generation: string,
+    retryOf: string | null,
+    rollbackOnError = true
+  ) {
+    try {
+      await this.job.add(
+        'copilot.transcript.task.submit',
+        {
+          taskId,
+          payload,
+          generation,
+          retryOf: retryOf ?? undefined,
+        },
+        {
+          jobId: `copilot-transcript-task/${taskId}/${generation}`,
+          attempts: 1,
+          removeOnFail: true,
+        }
+      );
+    } catch (error) {
+      if (rollbackOnError) {
+        await this.models.copilotTranscriptTask.failPendingDispatch(
+          taskId,
+          generation,
+          error instanceof Error ? error.message : 'transcript_enqueue_failed'
+        );
+      }
+      throw error;
+    }
+  }
+
+  async reconcileDispatches() {
+    const pending = await this.models.copilotTranscriptTask.pendingDispatches(
+      new Date(Date.now() - OneMinute)
+    );
+    for (const task of pending) {
+      const generation = task.dispatchGeneration;
+      if (!generation) continue;
+      const parsed = TranscriptPayloadSchema.safeParse(
+        task.protectedResult ?? task.inputSnapshot
+      );
+      if (!parsed.success) {
+        await this.models.copilotTranscriptTask.failPendingDispatch(
+          task.id,
+          generation,
+          'invalid_transcript_dispatch_payload'
+        );
+        continue;
+      }
+      try {
+        await this.enqueuePendingTask(
+          task.id,
+          parsed.data,
+          generation,
+          task.actionRunId,
+          false
+        );
+      } catch (error) {
+        this.logger.warn(
+          `Failed to recover pending transcript task ${task.id}`,
+          error
+        );
+      }
+    }
+
+    const running =
+      await this.models.copilotTranscriptTask.staleRunningDispatches(
+        new Date(Date.now() - OneHour)
+      );
+    for (const task of running) {
+      const generation = task.dispatchGeneration;
+      if (!generation) continue;
+      const failed =
+        await this.models.copilotTranscriptTask.failRunningDispatch(
+          task.id,
+          generation,
+          'transcript_dispatch_timed_out'
+        );
+      if (failed) {
+        this.realtime.publish(
+          'copilot.transcript.task.changed',
+          { workspaceId: task.workspaceId, taskId: task.id },
+          {
+            taskId: task.id,
+            status: AiJobStatus.failed,
+            error: 'transcript_dispatch_timed_out',
+          },
+          { room: realtimeTranscriptTaskRoom(task.workspaceId, task.id) }
+        );
+      }
+    }
+  }
+}

packages/backend/server/src/plugins/copilot/transcript/service.ts

@@ -1,3 +1,5 @@
+import { randomUUID } from 'node:crypto';
+
 import { BadRequestException, Injectable } from '@nestjs/common';
 import { AiJobStatus } from '@prisma/client';
 
@@ -5,7 +7,6 @@ import {
   CopilotTranscriptionJobExists,
   CopilotTranscriptionJobNotFound,
   type FileUpload,
-  JobQueue,
   OnJob,
   sniffMime,
 } from '../../../base';
@@ -18,7 +19,13 @@ import { PromptService } from '../prompt';
 import { ActionRuntimeBridge } from '../runtime/action-runtime-bridge';
 import { CapabilityRuntime } from '../runtime/capability-runtime';
 import { CopilotStorage } from '../storage';
+import {
+  TRANSCRIPT_ACTION_ID,
+  TRANSCRIPT_ACTION_VERSION,
+  TRANSCRIPT_PROMPT_REF,
+} from './constants';
 import { taskToJob, type TranscriptionJob } from './job';
+import { CopilotTranscriptionRetryService } from './retry';
 import {
   TranscriptActionResultContract,
   TranscriptPayloadSchema,
@@ -30,20 +37,16 @@ import type {
 } from './types';
 import { readStream } from './utils';
 
-const TRANSCRIPT_ACTION_ID = 'transcript.audio';
-const TRANSCRIPT_PROMPT_REF = 'Transcript audio structured';
-const TRANSCRIPT_ACTION_VERSION = 'v1';
-
 @Injectable()
 export class CopilotTranscriptionService {
   constructor(
     private readonly models: Models,
-    private readonly job: JobQueue,
     private readonly storage: CopilotStorage,
     private readonly prompts: PromptService,
     private readonly actionBridge: ActionRuntimeBridge,
     private readonly runtime: CapabilityRuntime,
-    private readonly realtime: RealtimePublisher
+    private readonly realtime: RealtimePublisher,
+    private readonly retry: CopilotTranscriptionRetryService
   ) {}
 
   private buildTaskPublicMeta(payload: TranscriptionPayloadV2) {
@@ -198,69 +201,27 @@ export class CopilotTranscriptionService {
     );
     const infos = await this.persistUploads(userId, workspaceId, blobId, blobs);
     const payload = this.createCanonicalPayload(blobId, infos, input);
+    const generation = randomUUID();
     const task = await this.models.copilotTranscriptTask.create({
       userId,
       workspaceId,
       blobId,
       recipeId: TRANSCRIPT_ACTION_ID,
       recipeVersion: TRANSCRIPT_ACTION_VERSION,
+      dispatchGeneration: generation,
       inputSnapshot: payload,
       publicMeta: this.buildTaskPublicMeta(payload),
+      protectedResult: payload,
     });
 
-    await this.job.add('copilot.transcript.task.submit', {
-      taskId: task.id,
-      payload,
-    });
-    await this.models.copilotTranscriptTask.markRunning(task.id);
-    this.publishTaskChanged(workspaceId, task.id, AiJobStatus.running);
+    await this.retry.enqueuePendingTask(task.id, payload, generation, null);
+    this.publishTaskChanged(workspaceId, task.id, AiJobStatus.pending);
 
-    return { id: task.id, status: AiJobStatus.running, infos };
+    return { id: task.id, status: AiJobStatus.pending, infos };
   }
 
   async retryTask(userId: string, workspaceId: string, taskId: string) {
-    const task = await this.models.copilotTranscriptTask.getWithUser(
-      userId,
-      workspaceId,
-      taskId
-    );
-    if (!task) {
-      throw new CopilotTranscriptionJobNotFound();
-    }
-    if (task.status === 'ready' || task.status === 'settled') {
-      throw new BadRequestException(
-        'Ready or settled transcript tasks cannot be retried'
-      );
-    }
-    if (task.status !== 'failed') {
-      throw new BadRequestException(
-        'Only failed transcript tasks can be retried'
-      );
-    }
-
-    const payload = TranscriptPayloadSchema.parse(task.protectedResult);
-    await this.runtime.assertRoute(
-      'transcript.audio',
-      {},
-      {
-        user: userId,
-        workspace: workspaceId,
-        featureKind: 'transcript',
-        builtInRouteId: TRANSCRIPT_PROMPT_REF,
-      }
-    );
-    await this.job.add('copilot.transcript.task.submit', {
-      taskId,
-      payload,
-      retryOf: task.actionRunId ?? undefined,
-    });
-    await this.models.copilotTranscriptTask.markRunning(taskId);
-    this.publishTaskChanged(workspaceId, taskId, AiJobStatus.running);
-    return {
-      id: taskId,
-      status: AiJobStatus.running,
-      infos: payload.infos ?? undefined,
-    };
+    return await this.retry.retryTask(userId, workspaceId, taskId);
   }
 
   async settleTask(userId: string, workspaceId: string, taskId: string) {
@@ -308,14 +269,34 @@ export class CopilotTranscriptionService {
   async transcriptTask({
     taskId,
     payload,
+    generation: queuedGeneration,
     retryOf,
   }: Jobs['copilot.transcript.task.submit']) {
     const task = await this.models.copilotTranscriptTask.get(taskId);
     if (!task) {
       throw new CopilotTranscriptionJobNotFound();
     }
+    let actionRunId = retryOf ?? null;
+    const generation = queuedGeneration ?? randomUUID();
+    if (
+      !queuedGeneration &&
+      !(await this.models.copilotTranscriptTask.adoptLegacyDispatch(
+        taskId,
+        actionRunId,
+        generation
+      ))
+    ) {
+      return;
+    }
+    const claimed = await this.models.copilotTranscriptTask.claimDispatch(
+      taskId,
+      generation,
+      actionRunId
+    );
+    if (!claimed) {
+      return;
+    }
 
-    let actionRunId: string | null = null;
     try {
       let bridgeFailed = false;
       let bridgeError = 'transcript native recipe failed';
@@ -334,7 +315,17 @@ export class CopilotTranscriptionService {
         retryOf: retryOf ?? null,
         inputSnapshot: runtimePayload,
         onRunCreated: async ({ runId }) => {
-          await this.models.copilotTranscriptTask.markRunning(taskId, runId);
+          const attached =
+            await this.models.copilotTranscriptTask.attachActionRun(
+              taskId,
+              generation,
+              actionRunId,
+              runId
+            );
+          if (!attached) {
+            throw new Error('stale transcript dispatch generation');
+          }
+          actionRunId = runId;
           this.publishTaskChanged(
             task.workspaceId,
             taskId,
@@ -355,7 +346,6 @@ export class CopilotTranscriptionService {
           responseContract: TranscriptActionResultContract,
         },
       })) {
-        actionRunId = event.runId;
         if (event.type === 'error' || event.status === 'failed') {
           bridgeFailed = true;
           bridgeError = event.errorMessage ?? event.errorCode ?? bridgeError;
@@ -371,29 +361,43 @@ export class CopilotTranscriptionService {
         ...TranscriptPayloadSchema.parse(finalResult),
         infos: payload.infos,
       } satisfies TranscriptionPayloadV2;
-      await this.models.copilotTranscriptTask.complete(taskId, {
-        status: 'ready',
-        actionRunId,
-        publicMeta: this.buildTaskPublicMeta(parsedResult),
-        protectedResult: parsedResult,
-        errorCode: null,
-      });
-      this.publishTaskChanged(task.workspaceId, taskId, AiJobStatus.finished);
+      const completed =
+        await this.models.copilotTranscriptTask.completeDispatch(
+          taskId,
+          generation,
+          actionRunId,
+          {
+            status: 'ready',
+            publicMeta: this.buildTaskPublicMeta(parsedResult),
+            protectedResult: parsedResult,
+            errorCode: null,
+          }
+        );
+      if (completed) {
+        this.publishTaskChanged(task.workspaceId, taskId, AiJobStatus.finished);
+      }
     } catch (error) {
-      await this.models.copilotTranscriptTask.complete(taskId, {
-        status: 'failed',
-        actionRunId,
-        publicMeta: this.buildTaskPublicMeta(payload),
-        protectedResult: payload,
-        errorCode:
-          error instanceof Error ? error.message : 'transcript_task_failed',
-      });
-      this.publishTaskChanged(
-        task.workspaceId,
+      const errorCode =
+        error instanceof Error ? error.message : 'transcript_task_failed';
+      const failed = await this.models.copilotTranscriptTask.completeDispatch(
         taskId,
-        AiJobStatus.failed,
-        error instanceof Error ? error.message : 'transcript_task_failed'
+        generation,
+        actionRunId,
+        {
+          status: 'failed',
+          publicMeta: this.buildTaskPublicMeta(payload),
+          protectedResult: payload,
+          errorCode,
+        }
       );
+      if (failed) {
+        this.publishTaskChanged(
+          task.workspaceId,
+          taskId,
+          AiJobStatus.failed,
+          errorCode
+        );
+      }
       throw error;
     }
   }

packages/backend/server/src/plugins/copilot/transcript/types.ts

@@ -53,6 +53,7 @@ declare global {
     'copilot.transcript.task.submit': {
       taskId: string;
       payload: TranscriptionPayloadV2;
+      generation?: string;
       retryOf?: string;
     };
   }