feat(server): refactor record schema (#14729)

#### PR Dependency Tree * **PR #14729** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal)  ## Summary by CodeRabbit * **New Features** * Transcriptions now produce structured meeting summaries (strict JSON) and a normalized, speaker‑tagged, non‑overlapping transcript with legacy projection support. * **API** * Submission accepts richer transcription input; results return source‑audio metadata, slice manifest, quality indicators, normalized segments/transcript, and structured summary JSON. * **Frontend** * Recording flow stores transcription metadata and uploads preprocessed audio slices with slice/quality info; UI-side result normalization applied. * **Tests** * Expanded unit, contract, and e2e coverage for normalization, payload parsing, persistence/retry, and end‑to‑end transcription flows.
2026-07-22 12:36:24 +08:00 · 2026-03-26 21:32:36 +08:00
parent a3379c8979
commit 5b05c5a1b2
29 changed files with 2977 additions and 463 deletions
@@ -3,8 +3,7 @@ import {
  type TranscriptionBlockModel,
 } from '@affine/core/blocksuite/ai/blocks/transcription-block/model';
 import { insertFromMarkdown } from '@affine/core/blocksuite/utils';
-import { toArrayBuffer } from '@affine/core/utils/array-buffer';
-import { encodeAudioBlobToOpusSlices } from '@affine/core/utils/opus-encoding';
+import { preprocessAudioBlobForTranscription } from '@affine/core/utils/opus-encoding';
 import { DebugLogger } from '@affine/debug';
 import { AiJobStatus } from '@affine/graphql';
 import track from '@affine/track';
@@ -23,12 +22,22 @@ import { AudioTranscriptionJob } from './audio-transcription-job';
 import type { TranscriptionResult } from './types';

 const logger = new DebugLogger('audio-attachment-block');
+type TranscriptionBlockProps = TranscriptionBlockModel['props'];

 // BlockSuiteError: yText must not contain "\r" because it will break the range synchronization
 function sanitizeText(text: string) {
  return text.replace(/\r/g, '');
 }

+function requireTranscriptionBlockProps(
+  transcriptionBlockProps: TranscriptionBlockProps | undefined
+) {
+  if (!transcriptionBlockProps) {
+    throw new Error('No transcription block props');
+  }
+  return transcriptionBlockProps;
+}
+
 const colorOptions = [
  cssVarV2.text.highlight.fg.red,
  cssVarV2.text.highlight.fg.green,
@@ -124,26 +133,35 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {
      transcriptionBlockProps = this.transcriptionBlock$.value?.props;
    }

-    if (!transcriptionBlockProps) {
-      throw new Error('No transcription block props');
-    }
-
    const job = this.framework.createEntity(AudioTranscriptionJob, {
      blobId: this.props.props.sourceId,
-      blockProps: transcriptionBlockProps,
-      getAudioFiles: async () => {
+      blockProps: requireTranscriptionBlockProps(transcriptionBlockProps),
+      getAudioTranscriptionInput: async () => {
        const buffer = await this.audioMedia.getBuffer();
        if (!buffer) {
          throw new Error('No audio buffer available');
        }
-        const slices = await encodeAudioBlobToOpusSlices(buffer, 64000);
-        const files = slices.map((slice, index) => {
-          const blob = new Blob([toArrayBuffer(slice)], { type: 'audio/opus' });
-          return new File([blob], this.props.props.name + `-${index}.opus`, {
-            type: 'audio/opus',
+        const currentTranscriptionBlockProps = requireTranscriptionBlockProps(
+          this.transcriptionBlock$.value?.props
+        );
+        const { files, sourceAudio, sliceManifest } =
+          await preprocessAudioBlobForTranscription(buffer, {
+            fileNameBase: this.props.props.name,
+            sourceMimeType: this.props.props.type,
+            targetBitrate: 64000,
          });
-        });
-        return files;
+
+        return {
+          files,
+          input: {
+            sourceAudio: {
+              ...sourceAudio,
+              ...currentTranscriptionBlockProps.transcription.sourceAudio,
+            },
+            quality: currentTranscriptionBlockProps.transcription.quality,
+            sliceManifest,
+          },
+        };
      },
    });

@@ -12,7 +12,10 @@ import type { WorkspaceService } from '../../workspace';

 export class AudioTranscriptionJobStore extends Entity<{
  readonly blobId: string;
-  readonly getAudioFiles: () => Promise<File[]>;
+  readonly getAudioTranscriptionInput: () => Promise<{
+    files: File[];
+    input?: Record<string, unknown>;
+  }>;
 }> {
  constructor(
    private readonly workspaceService: WorkspaceService,
@@ -41,7 +44,7 @@ export class AudioTranscriptionJobStore extends Entity<{
    if (!graphqlService) {
      throw new Error('No graphql service available');
    }
-    const files = await this.props.getAudioFiles();
+    const { files, input } = await this.props.getAudioTranscriptionInput();
    const response = await graphqlService.gql({
      timeout: 0, // default 15s is too short for audio transcription
      query: submitAudioTranscriptionMutation,
@@ -49,6 +52,7 @@ export class AudioTranscriptionJobStore extends Entity<{
        workspaceId: this.currentWorkspaceId,
        blobId: this.props.blobId,
        blobs: files,
+        input,
      },
    });
    if (!response.submitAudioTranscription?.id) {
@@ -8,6 +8,7 @@ import { Entity, LiveData } from '@toeverything/infra';
 import type { DefaultServerService, WorkspaceServerService } from '../../cloud';
 import { AuthService } from '../../cloud/services/auth';
 import { AudioTranscriptionJobStore } from './audio-transcription-job-store';
+import { buildTranscriptionResult } from './transcription-result';
 import type { TranscriptionResult } from './types';

 // The UI status of the transcription job
@@ -46,7 +47,10 @@ const logger = new DebugLogger('audio-transcription-job');
 export class AudioTranscriptionJob extends Entity<{
  readonly blockProps: TranscriptionBlockProps;
  readonly blobId: string;
-  readonly getAudioFiles: () => Promise<File[]>;
+  readonly getAudioTranscriptionInput: () => Promise<{
+    files: File[];
+    input?: Record<string, unknown>;
+  }>;
 }> {
  constructor(
    private readonly workspaceServerService: WorkspaceServerService,
@@ -68,7 +72,7 @@ export class AudioTranscriptionJob extends Entity<{
    AudioTranscriptionJobStore,
    {
      blobId: this.props.blobId,
-      getAudioFiles: this.props.getAudioFiles,
+      getAudioTranscriptionInput: this.props.getAudioTranscriptionInput,
    }
  );

@@ -241,18 +245,7 @@ export class AudioTranscriptionJob extends Entity<{
      logger.debug('Successfully claimed job', {
        jobId: this.props.blockProps.jobId,
      });
-      const result: TranscriptionResult = {
-        summary: claimedJob.summary ?? '',
-        title: claimedJob.title ?? '',
-        actions: claimedJob.actions ?? '',
-        segments:
-          claimedJob.transcription?.map(segment => ({
-            speaker: segment.speaker,
-            start: segment.start,
-            end: segment.end,
-            transcription: segment.transcription,
-          })) ?? [],
-      };
+      const result: TranscriptionResult = buildTranscriptionResult(claimedJob);

      this._status$.value = {
        status: AiJobStatus.claimed,
@@ -0,0 +1,117 @@
+import { describe, expect, test } from 'vitest';
+
+import {
+  actionItemsToMarkdown,
+  buildTranscriptionResult,
+  summaryJsonToMarkdown,
+} from './transcription-result';
+
+describe('transcription-result', () => {
+  test('prefers new fields and projects markdown from summaryJson', () => {
+    const result = buildTranscriptionResult({
+      summaryJson: {
+        title: 'Weekly Sync',
+        durationMinutes: 30,
+        attendees: ['A', 'B'],
+        keyPoints: ['Reviewed launch status'],
+        actionItems: [
+          {
+            description: 'Send recap',
+            owner: 'A',
+            deadline: 'Friday',
+          },
+        ],
+        decisions: ['Ship on Monday'],
+        openQuestions: ['Need final QA sign-off'],
+        blockers: ['Waiting on analytics'],
+      },
+      normalizedSegments: [
+        {
+          speaker: 'A',
+          startSec: 1,
+          endSec: 3,
+          start: '00:00:01',
+          end: '00:00:03',
+          text: 'Kickoff',
+        },
+      ],
+    });
+
+    expect(result).toEqual({
+      title: 'Weekly Sync',
+      summary: [
+        '- Reviewed launch status',
+        '## Decisions',
+        '- Ship on Monday',
+        '## Open Questions',
+        '- Need final QA sign-off',
+        '## Blockers',
+        '- Waiting on analytics',
+      ].join('\n'),
+      actions: '- [ ] Send recap (A · Friday)',
+      segments: [
+        {
+          speaker: 'A',
+          start: '00:00:01',
+          end: '00:00:03',
+          transcription: 'Kickoff',
+        },
+      ],
+    });
+  });
+
+  test('falls back to legacy fields when they exist', () => {
+    const result = buildTranscriptionResult({
+      title: 'Legacy title',
+      summary: 'legacy summary',
+      actions: 'legacy actions',
+      transcription: [
+        {
+          speaker: 'B',
+          start: '00:00:04',
+          end: '00:00:06',
+          transcription: 'Legacy line',
+        },
+      ],
+      summaryJson: {
+        title: 'New title',
+        durationMinutes: 5,
+        attendees: [],
+        keyPoints: ['new'],
+        actionItems: [],
+        decisions: [],
+        openQuestions: [],
+        blockers: [],
+      },
+      normalizedSegments: [
+        {
+          speaker: 'A',
+          startSec: 1,
+          endSec: 2,
+          start: '00:00:01',
+          end: '00:00:02',
+          text: 'new',
+        },
+      ],
+    });
+
+    expect(result).toEqual({
+      title: 'Legacy title',
+      summary: 'legacy summary',
+      actions: 'legacy actions',
+      segments: [
+        {
+          speaker: 'B',
+          start: '00:00:04',
+          end: '00:00:06',
+          transcription: 'Legacy line',
+        },
+      ],
+    });
+  });
+
+  test('returns empty markdown when summaryJson is absent', () => {
+    expect(summaryJsonToMarkdown(null)).toBe('');
+    expect(actionItemsToMarkdown(null)).toBe('');
+  });
+});
@@ -0,0 +1,89 @@
+import type {
+  MeetingSummaryV2Type,
+  NormalizedTranscriptSegmentType,
+} from '@affine/graphql';
+
+import type { TranscriptionResult } from './types';
+
+type TranscriptionPayloadLike = {
+  title?: string | null;
+  summary?: string | null;
+  actions?: string | null;
+  transcription?:
+    | {
+        speaker: string;
+        start: string;
+        end: string;
+        transcription: string;
+      }[]
+    | null;
+  normalizedSegments?: NormalizedTranscriptSegmentType[] | null;
+  summaryJson?: MeetingSummaryV2Type | null;
+};
+
+function formatSection(title: string, items: string[]) {
+  if (!items.length) {
+    return [];
+  }
+
+  return [`## ${title}`, ...items.map(item => `- ${item}`)];
+}
+
+export function summaryJsonToMarkdown(
+  summaryJson?: MeetingSummaryV2Type | null
+) {
+  if (!summaryJson) {
+    return '';
+  }
+
+  return [
+    ...summaryJson.keyPoints.map(item => `- ${item}`),
+    ...formatSection('Decisions', summaryJson.decisions),
+    ...formatSection('Open Questions', summaryJson.openQuestions),
+    ...formatSection('Blockers', summaryJson.blockers),
+  ]
+    .join('\n')
+    .trim();
+}
+
+export function actionItemsToMarkdown(
+  summaryJson?: MeetingSummaryV2Type | null
+) {
+  if (!summaryJson?.actionItems.length) {
+    return '';
+  }
+
+  return summaryJson.actionItems
+    .map(item => {
+      const suffix = [item.owner, item.deadline].filter(Boolean).join(' · ');
+      return `- [ ] ${item.description}${suffix ? ` (${suffix})` : ''}`;
+    })
+    .join('\n')
+    .trim();
+}
+
+function normalizedSegmentsToResult(
+  normalizedSegments?: NormalizedTranscriptSegmentType[] | null
+) {
+  return (
+    normalizedSegments?.map(segment => ({
+      speaker: segment.speaker,
+      start: segment.start,
+      end: segment.end,
+      transcription: segment.text,
+    })) ?? []
+  );
+}
+
+export function buildTranscriptionResult(
+  payload: TranscriptionPayloadLike
+): TranscriptionResult {
+  return {
+    title: payload.title ?? payload.summaryJson?.title ?? '',
+    summary: payload.summary ?? summaryJsonToMarkdown(payload.summaryJson),
+    actions: payload.actions ?? actionItemsToMarkdown(payload.summaryJson),
+    segments:
+      payload.transcription ??
+      normalizedSegmentsToResult(payload.normalizedSegments),
+  };
+}