feat(server): refactor record schema (#14729)

#### PR Dependency Tree


* **PR #14729** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Transcriptions now produce structured meeting summaries (strict JSON)
and a normalized, speaker‑tagged, non‑overlapping transcript with legacy
projection support.

* **API**
* Submission accepts richer transcription input; results return
source‑audio metadata, slice manifest, quality indicators, normalized
segments/transcript, and structured summary JSON.

* **Frontend**
* Recording flow stores transcription metadata and uploads preprocessed
audio slices with slice/quality info; UI-side result normalization
applied.

* **Tests**
* Expanded unit, contract, and e2e coverage for normalization, payload
parsing, persistence/retry, and end‑to‑end transcription flows.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/frontend/core/src/blocksuite/ai/blocks/transcription-block/model.ts

@@ -1,3 +1,7 @@
+import type {
+  TranscriptionQualityInput,
+  TranscriptionSourceAudioInput,
+} from '@affine/graphql';
 import {
   BlockModel,
   BlockSchemaExtension,
@@ -25,7 +29,10 @@ export const TranscriptionBlockSchema = defineBlockSchema({
 });
 
 export type TranscriptionBlockProps = {
-  transcription: Record<string, any>;
+  transcription: {
+    sourceAudio?: TranscriptionSourceAudioInput;
+    quality?: TranscriptionQualityInput;
+  };
   jobId?: string;
   createdBy?: string;
 };

packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts

@@ -3,8 +3,7 @@ import {
   type TranscriptionBlockModel,
 } from '@affine/core/blocksuite/ai/blocks/transcription-block/model';
 import { insertFromMarkdown } from '@affine/core/blocksuite/utils';
-import { toArrayBuffer } from '@affine/core/utils/array-buffer';
-import { encodeAudioBlobToOpusSlices } from '@affine/core/utils/opus-encoding';
+import { preprocessAudioBlobForTranscription } from '@affine/core/utils/opus-encoding';
 import { DebugLogger } from '@affine/debug';
 import { AiJobStatus } from '@affine/graphql';
 import track from '@affine/track';
@@ -23,12 +22,22 @@ import { AudioTranscriptionJob } from './audio-transcription-job';
 import type { TranscriptionResult } from './types';
 
 const logger = new DebugLogger('audio-attachment-block');
+type TranscriptionBlockProps = TranscriptionBlockModel['props'];
 
 // BlockSuiteError: yText must not contain "\r" because it will break the range synchronization
 function sanitizeText(text: string) {
   return text.replace(/\r/g, '');
 }
 
+function requireTranscriptionBlockProps(
+  transcriptionBlockProps: TranscriptionBlockProps | undefined
+) {
+  if (!transcriptionBlockProps) {
+    throw new Error('No transcription block props');
+  }
+  return transcriptionBlockProps;
+}
+
 const colorOptions = [
   cssVarV2.text.highlight.fg.red,
   cssVarV2.text.highlight.fg.green,
@@ -124,26 +133,35 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {
       transcriptionBlockProps = this.transcriptionBlock$.value?.props;
     }
 
-    if (!transcriptionBlockProps) {
-      throw new Error('No transcription block props');
-    }
-
     const job = this.framework.createEntity(AudioTranscriptionJob, {
       blobId: this.props.props.sourceId,
-      blockProps: transcriptionBlockProps,
-      getAudioFiles: async () => {
+      blockProps: requireTranscriptionBlockProps(transcriptionBlockProps),
+      getAudioTranscriptionInput: async () => {
         const buffer = await this.audioMedia.getBuffer();
         if (!buffer) {
           throw new Error('No audio buffer available');
         }
-        const slices = await encodeAudioBlobToOpusSlices(buffer, 64000);
-        const files = slices.map((slice, index) => {
-          const blob = new Blob([toArrayBuffer(slice)], { type: 'audio/opus' });
-          return new File([blob], this.props.props.name + `-${index}.opus`, {
-            type: 'audio/opus',
+        const currentTranscriptionBlockProps = requireTranscriptionBlockProps(
+          this.transcriptionBlock$.value?.props
+        );
+        const { files, sourceAudio, sliceManifest } =
+          await preprocessAudioBlobForTranscription(buffer, {
+            fileNameBase: this.props.props.name,
+            sourceMimeType: this.props.props.type,
+            targetBitrate: 64000,
           });
-        });
-        return files;
+
+        return {
+          files,
+          input: {
+            sourceAudio: {
+              ...sourceAudio,
+              ...currentTranscriptionBlockProps.transcription.sourceAudio,
+            },
+            quality: currentTranscriptionBlockProps.transcription.quality,
+            sliceManifest,
+          },
+        };
       },
     });
 

packages/frontend/core/src/modules/media/entities/audio-transcription-job-store.ts

@@ -12,7 +12,10 @@ import type { WorkspaceService } from '../../workspace';
 
 export class AudioTranscriptionJobStore extends Entity<{
   readonly blobId: string;
-  readonly getAudioFiles: () => Promise<File[]>;
+  readonly getAudioTranscriptionInput: () => Promise<{
+    files: File[];
+    input?: Record<string, unknown>;
+  }>;
 }> {
   constructor(
     private readonly workspaceService: WorkspaceService,
@@ -41,7 +44,7 @@ export class AudioTranscriptionJobStore extends Entity<{
     if (!graphqlService) {
       throw new Error('No graphql service available');
     }
-    const files = await this.props.getAudioFiles();
+    const { files, input } = await this.props.getAudioTranscriptionInput();
     const response = await graphqlService.gql({
       timeout: 0, // default 15s is too short for audio transcription
       query: submitAudioTranscriptionMutation,
@@ -49,6 +52,7 @@ export class AudioTranscriptionJobStore extends Entity<{
         workspaceId: this.currentWorkspaceId,
         blobId: this.props.blobId,
         blobs: files,
+        input,
       },
     });
     if (!response.submitAudioTranscription?.id) {

packages/frontend/core/src/modules/media/entities/audio-transcription-job.ts

@@ -8,6 +8,7 @@ import { Entity, LiveData } from '@toeverything/infra';
 import type { DefaultServerService, WorkspaceServerService } from '../../cloud';
 import { AuthService } from '../../cloud/services/auth';
 import { AudioTranscriptionJobStore } from './audio-transcription-job-store';
+import { buildTranscriptionResult } from './transcription-result';
 import type { TranscriptionResult } from './types';
 
 // The UI status of the transcription job
@@ -46,7 +47,10 @@ const logger = new DebugLogger('audio-transcription-job');
 export class AudioTranscriptionJob extends Entity<{
   readonly blockProps: TranscriptionBlockProps;
   readonly blobId: string;
-  readonly getAudioFiles: () => Promise<File[]>;
+  readonly getAudioTranscriptionInput: () => Promise<{
+    files: File[];
+    input?: Record<string, unknown>;
+  }>;
 }> {
   constructor(
     private readonly workspaceServerService: WorkspaceServerService,
@@ -68,7 +72,7 @@ export class AudioTranscriptionJob extends Entity<{
     AudioTranscriptionJobStore,
     {
       blobId: this.props.blobId,
-      getAudioFiles: this.props.getAudioFiles,
+      getAudioTranscriptionInput: this.props.getAudioTranscriptionInput,
     }
   );
 
@@ -241,18 +245,7 @@ export class AudioTranscriptionJob extends Entity<{
       logger.debug('Successfully claimed job', {
         jobId: this.props.blockProps.jobId,
       });
-      const result: TranscriptionResult = {
-        summary: claimedJob.summary ?? '',
-        title: claimedJob.title ?? '',
-        actions: claimedJob.actions ?? '',
-        segments:
-          claimedJob.transcription?.map(segment => ({
-            speaker: segment.speaker,
-            start: segment.start,
-            end: segment.end,
-            transcription: segment.transcription,
-          })) ?? [],
-      };
+      const result: TranscriptionResult = buildTranscriptionResult(claimedJob);
 
       this._status$.value = {
         status: AiJobStatus.claimed,

packages/frontend/core/src/modules/media/entities/transcription-result.spec.ts

@@ -0,0 +1,117 @@
+import { describe, expect, test } from 'vitest';
+
+import {
+  actionItemsToMarkdown,
+  buildTranscriptionResult,
+  summaryJsonToMarkdown,
+} from './transcription-result';
+
+describe('transcription-result', () => {
+  test('prefers new fields and projects markdown from summaryJson', () => {
+    const result = buildTranscriptionResult({
+      summaryJson: {
+        title: 'Weekly Sync',
+        durationMinutes: 30,
+        attendees: ['A', 'B'],
+        keyPoints: ['Reviewed launch status'],
+        actionItems: [
+          {
+            description: 'Send recap',
+            owner: 'A',
+            deadline: 'Friday',
+          },
+        ],
+        decisions: ['Ship on Monday'],
+        openQuestions: ['Need final QA sign-off'],
+        blockers: ['Waiting on analytics'],
+      },
+      normalizedSegments: [
+        {
+          speaker: 'A',
+          startSec: 1,
+          endSec: 3,
+          start: '00:00:01',
+          end: '00:00:03',
+          text: 'Kickoff',
+        },
+      ],
+    });
+
+    expect(result).toEqual({
+      title: 'Weekly Sync',
+      summary: [
+        '- Reviewed launch status',
+        '## Decisions',
+        '- Ship on Monday',
+        '## Open Questions',
+        '- Need final QA sign-off',
+        '## Blockers',
+        '- Waiting on analytics',
+      ].join('\n'),
+      actions: '- [ ] Send recap (A · Friday)',
+      segments: [
+        {
+          speaker: 'A',
+          start: '00:00:01',
+          end: '00:00:03',
+          transcription: 'Kickoff',
+        },
+      ],
+    });
+  });
+
+  test('falls back to legacy fields when they exist', () => {
+    const result = buildTranscriptionResult({
+      title: 'Legacy title',
+      summary: 'legacy summary',
+      actions: 'legacy actions',
+      transcription: [
+        {
+          speaker: 'B',
+          start: '00:00:04',
+          end: '00:00:06',
+          transcription: 'Legacy line',
+        },
+      ],
+      summaryJson: {
+        title: 'New title',
+        durationMinutes: 5,
+        attendees: [],
+        keyPoints: ['new'],
+        actionItems: [],
+        decisions: [],
+        openQuestions: [],
+        blockers: [],
+      },
+      normalizedSegments: [
+        {
+          speaker: 'A',
+          startSec: 1,
+          endSec: 2,
+          start: '00:00:01',
+          end: '00:00:02',
+          text: 'new',
+        },
+      ],
+    });
+
+    expect(result).toEqual({
+      title: 'Legacy title',
+      summary: 'legacy summary',
+      actions: 'legacy actions',
+      segments: [
+        {
+          speaker: 'B',
+          start: '00:00:04',
+          end: '00:00:06',
+          transcription: 'Legacy line',
+        },
+      ],
+    });
+  });
+
+  test('returns empty markdown when summaryJson is absent', () => {
+    expect(summaryJsonToMarkdown(null)).toBe('');
+    expect(actionItemsToMarkdown(null)).toBe('');
+  });
+});

packages/frontend/core/src/modules/media/entities/transcription-result.ts

@@ -0,0 +1,89 @@
+import type {
+  MeetingSummaryV2Type,
+  NormalizedTranscriptSegmentType,
+} from '@affine/graphql';
+
+import type { TranscriptionResult } from './types';
+
+type TranscriptionPayloadLike = {
+  title?: string | null;
+  summary?: string | null;
+  actions?: string | null;
+  transcription?:
+    | {
+        speaker: string;
+        start: string;
+        end: string;
+        transcription: string;
+      }[]
+    | null;
+  normalizedSegments?: NormalizedTranscriptSegmentType[] | null;
+  summaryJson?: MeetingSummaryV2Type | null;
+};
+
+function formatSection(title: string, items: string[]) {
+  if (!items.length) {
+    return [];
+  }
+
+  return [`## ${title}`, ...items.map(item => `- ${item}`)];
+}
+
+export function summaryJsonToMarkdown(
+  summaryJson?: MeetingSummaryV2Type | null
+) {
+  if (!summaryJson) {
+    return '';
+  }
+
+  return [
+    ...summaryJson.keyPoints.map(item => `- ${item}`),
+    ...formatSection('Decisions', summaryJson.decisions),
+    ...formatSection('Open Questions', summaryJson.openQuestions),
+    ...formatSection('Blockers', summaryJson.blockers),
+  ]
+    .join('\n')
+    .trim();
+}
+
+export function actionItemsToMarkdown(
+  summaryJson?: MeetingSummaryV2Type | null
+) {
+  if (!summaryJson?.actionItems.length) {
+    return '';
+  }
+
+  return summaryJson.actionItems
+    .map(item => {
+      const suffix = [item.owner, item.deadline].filter(Boolean).join(' · ');
+      return `- [ ] ${item.description}${suffix ? ` (${suffix})` : ''}`;
+    })
+    .join('\n')
+    .trim();
+}
+
+function normalizedSegmentsToResult(
+  normalizedSegments?: NormalizedTranscriptSegmentType[] | null
+) {
+  return (
+    normalizedSegments?.map(segment => ({
+      speaker: segment.speaker,
+      start: segment.start,
+      end: segment.end,
+      transcription: segment.text,
+    })) ?? []
+  );
+}
+
+export function buildTranscriptionResult(
+  payload: TranscriptionPayloadLike
+): TranscriptionResult {
+  return {
+    title: payload.title ?? payload.summaryJson?.title ?? '',
+    summary: payload.summary ?? summaryJsonToMarkdown(payload.summaryJson),
+    actions: payload.actions ?? actionItemsToMarkdown(payload.summaryJson),
+    segments:
+      payload.transcription ??
+      normalizedSegmentsToResult(payload.normalizedSegments),
+  };
+}

packages/frontend/core/src/utils/opus-encoding.ts

@@ -1,5 +1,9 @@
 import { DebugLogger } from '@affine/debug';
 import { apis } from '@affine/electron-api';
+import type {
+  AudioSliceManifestItemInput,
+  TranscriptionSourceAudioInput,
+} from '@affine/graphql';
 import { ArrayBufferTarget, Muxer } from 'mp4-muxer';
 
 import { isLink } from '../modules/navigation/utils';
@@ -16,6 +20,12 @@ interface AudioEncodingResult {
   config: AudioEncodingConfig;
 }
 
+interface EncodedAudioSlice {
+  data: Uint8Array;
+  startSec: number;
+  durationSec: number;
+}
+
 const logger = new DebugLogger('opus-encoding');
 const LOCAL_FILE_ASSET_URL = 'assets://local-file';
 
@@ -24,6 +34,12 @@ const DEFAULT_BITRATE = 64000;
 const MAX_SLICE_DURATION_SECONDS = 10 * 60; // 10 minutes
 const MIN_SLICE_DURATION_SECONDS = 5 * 60; // 5 minutes
 const AUDIO_LEVEL_THRESHOLD = 0.02; // Threshold for "silence" detection
+export const SLICE_FILE_EXT = 'm4a';
+export const SLICE_MIME_TYPE = 'audio/m4a';
+
+export function getSliceName(fileNameBase: string, index: number) {
+  return `${fileNameBase}-${index}.${SLICE_FILE_EXT}`;
+}
 
 /**
  * Converts various blob formats to ArrayBuffer
@@ -352,64 +368,118 @@ export async function encodeAudioBlobToOpusSlices(
   try {
     const arrayBuffer = await blobToArrayBuffer(blob);
     const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
-    const slices: Uint8Array[] = [];
-
-    // Define slicing parameters
-    const sampleRate = audioBuffer.sampleRate;
-    const numberOfChannels = audioBuffer.numberOfChannels;
-
-    // Calculate sizes in samples
-    const maxSliceSamples = MAX_SLICE_DURATION_SECONDS * sampleRate;
-    const minSliceSamples = MIN_SLICE_DURATION_SECONDS * sampleRate;
-    const totalSamples = audioBuffer.length;
-
-    // Start slicing
-    let startSample = 0;
-
-    while (startSample < totalSamples) {
-      // Determine end sample for this slice
-      let endSample = Math.min(startSample + maxSliceSamples, totalSamples);
-
-      // Find the best slice point based on audio levels
-      endSample = findSlicePoint(
-        audioBuffer,
-        startSample,
-        endSample,
-        minSliceSamples
-      );
-
-      // Create a slice from startSample to endSample
-      const audioData = extractAudioData(audioBuffer, startSample, endSample);
-
-      // Encode this slice to Opus
-      const { encoder, encodedChunks } = createOpusEncoder({
-        sampleRate,
-        numberOfChannels,
-        bitrate: targetBitrate,
-      });
-
-      await encodeAudioFrames({
-        audioData,
-        numberOfChannels,
-        sampleRate,
-        encoder,
-      });
-
-      // Mux to MP4 and add to slices
-      const mp4 = muxToMp4(encodedChunks, {
-        sampleRate,
-        numberOfChannels,
-        bitrate: targetBitrate,
-      });
-
-      slices.push(mp4);
-
-      // Move to next slice
-      startSample = endSample;
-    }
+    const slices = await encodeAudioBufferToOpusSliceData(
+      audioBuffer,
+      targetBitrate
+    );
 
     logger.debug(`Encoded audio blob to ${slices.length} Opus slices`);
-    return slices;
+    return slices.map(slice => slice.data);
+  } finally {
+    await audioContext.close();
+  }
+}
+
+async function encodeAudioBufferToOpusSliceData(
+  audioBuffer: AudioBuffer,
+  targetBitrate: number
+): Promise<EncodedAudioSlice[]> {
+  const slices: EncodedAudioSlice[] = [];
+  const sampleRate = audioBuffer.sampleRate;
+  const numberOfChannels = audioBuffer.numberOfChannels;
+  const maxSliceSamples = MAX_SLICE_DURATION_SECONDS * sampleRate;
+  const minSliceSamples = MIN_SLICE_DURATION_SECONDS * sampleRate;
+  const totalSamples = audioBuffer.length;
+
+  let startSample = 0;
+  while (startSample < totalSamples) {
+    let endSample = Math.min(startSample + maxSliceSamples, totalSamples);
+    endSample = findSlicePoint(
+      audioBuffer,
+      startSample,
+      endSample,
+      minSliceSamples
+    );
+
+    const audioData = extractAudioData(audioBuffer, startSample, endSample);
+    const { encoder, encodedChunks } = createOpusEncoder({
+      sampleRate,
+      numberOfChannels,
+      bitrate: targetBitrate,
+    });
+
+    await encodeAudioFrames({
+      audioData,
+      numberOfChannels,
+      sampleRate,
+      encoder,
+    });
+
+    slices.push({
+      data: muxToMp4(encodedChunks, {
+        sampleRate,
+        numberOfChannels,
+        bitrate: targetBitrate,
+      }),
+      startSec: startSample / sampleRate,
+      durationSec: (endSample - startSample) / sampleRate,
+    });
+
+    startSample = endSample;
+  }
+
+  return slices;
+}
+
+export async function preprocessAudioBlobForTranscription(
+  blob: Blob | ArrayBuffer | Uint8Array,
+  options: {
+    fileNameBase: string;
+    sourceMimeType?: string;
+    targetBitrate?: number;
+  }
+): Promise<{
+  files: File[];
+  sourceAudio: TranscriptionSourceAudioInput;
+  sliceManifest: AudioSliceManifestItemInput[];
+}> {
+  const audioContext = new AudioContext();
+
+  try {
+    const arrayBuffer = await blobToArrayBuffer(blob);
+    const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+    const encodedSlices = await encodeAudioBufferToOpusSliceData(
+      audioBuffer,
+      options.targetBitrate ?? DEFAULT_BITRATE
+    );
+
+    const files = encodedSlices.map((slice, index) => {
+      const fileName = getSliceName(options.fileNameBase, index);
+      const data = toArrayBuffer(slice.data);
+      return new File([new Blob([data], { type: SLICE_MIME_TYPE })], fileName, {
+        type: SLICE_MIME_TYPE,
+      });
+    });
+
+    return {
+      files,
+      sourceAudio: {
+        mimeType:
+          options.sourceMimeType ?? (blob instanceof Blob ? blob.type : null),
+        durationMs: Math.round(audioBuffer.duration * 1000),
+        sampleRate: audioBuffer.sampleRate,
+        channels: audioBuffer.numberOfChannels,
+      },
+      sliceManifest: encodedSlices.map((slice, index) => ({
+        index,
+        fileName:
+          files[index]?.name ?? getSliceName(options.fileNameBase, index),
+        mimeType: files[index]?.type || SLICE_MIME_TYPE,
+        startSec: slice.startSec,
+        durationSec: slice.durationSec,
+        byteSize: slice.data.byteLength,
+      })),
+    };
   } finally {
     await audioContext.close();
   }