feat(server): refactor record schema (#14729)

#### PR Dependency Tree


* **PR #14729** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Transcriptions now produce structured meeting summaries (strict JSON)
and a normalized, speaker‑tagged, non‑overlapping transcript with legacy
projection support.

* **API**
* Submission accepts richer transcription input; results return
source‑audio metadata, slice manifest, quality indicators, normalized
segments/transcript, and structured summary JSON.

* **Frontend**
* Recording flow stores transcription metadata and uploads preprocessed
audio slices with slice/quality info; UI-side result normalization
applied.

* **Tests**
* Expanded unit, contract, and e2e coverage for normalization, payload
parsing, persistence/retry, and end‑to‑end transcription flows.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/frontend/core/src/utils/opus-encoding.ts

@@ -1,5 +1,9 @@
 import { DebugLogger } from '@affine/debug';
 import { apis } from '@affine/electron-api';
+import type {
+  AudioSliceManifestItemInput,
+  TranscriptionSourceAudioInput,
+} from '@affine/graphql';
 import { ArrayBufferTarget, Muxer } from 'mp4-muxer';
 
 import { isLink } from '../modules/navigation/utils';
@@ -16,6 +20,12 @@ interface AudioEncodingResult {
   config: AudioEncodingConfig;
 }
 
+interface EncodedAudioSlice {
+  data: Uint8Array;
+  startSec: number;
+  durationSec: number;
+}
+
 const logger = new DebugLogger('opus-encoding');
 const LOCAL_FILE_ASSET_URL = 'assets://local-file';
 
@@ -24,6 +34,12 @@ const DEFAULT_BITRATE = 64000;
 const MAX_SLICE_DURATION_SECONDS = 10 * 60; // 10 minutes
 const MIN_SLICE_DURATION_SECONDS = 5 * 60; // 5 minutes
 const AUDIO_LEVEL_THRESHOLD = 0.02; // Threshold for "silence" detection
+export const SLICE_FILE_EXT = 'm4a';
+export const SLICE_MIME_TYPE = 'audio/m4a';
+
+export function getSliceName(fileNameBase: string, index: number) {
+  return `${fileNameBase}-${index}.${SLICE_FILE_EXT}`;
+}
 
 /**
  * Converts various blob formats to ArrayBuffer
@@ -352,64 +368,118 @@ export async function encodeAudioBlobToOpusSlices(
   try {
     const arrayBuffer = await blobToArrayBuffer(blob);
     const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
-    const slices: Uint8Array[] = [];
-
-    // Define slicing parameters
-    const sampleRate = audioBuffer.sampleRate;
-    const numberOfChannels = audioBuffer.numberOfChannels;
-
-    // Calculate sizes in samples
-    const maxSliceSamples = MAX_SLICE_DURATION_SECONDS * sampleRate;
-    const minSliceSamples = MIN_SLICE_DURATION_SECONDS * sampleRate;
-    const totalSamples = audioBuffer.length;
-
-    // Start slicing
-    let startSample = 0;
-
-    while (startSample < totalSamples) {
-      // Determine end sample for this slice
-      let endSample = Math.min(startSample + maxSliceSamples, totalSamples);
-
-      // Find the best slice point based on audio levels
-      endSample = findSlicePoint(
-        audioBuffer,
-        startSample,
-        endSample,
-        minSliceSamples
-      );
-
-      // Create a slice from startSample to endSample
-      const audioData = extractAudioData(audioBuffer, startSample, endSample);
-
-      // Encode this slice to Opus
-      const { encoder, encodedChunks } = createOpusEncoder({
-        sampleRate,
-        numberOfChannels,
-        bitrate: targetBitrate,
-      });
-
-      await encodeAudioFrames({
-        audioData,
-        numberOfChannels,
-        sampleRate,
-        encoder,
-      });
-
-      // Mux to MP4 and add to slices
-      const mp4 = muxToMp4(encodedChunks, {
-        sampleRate,
-        numberOfChannels,
-        bitrate: targetBitrate,
-      });
-
-      slices.push(mp4);
-
-      // Move to next slice
-      startSample = endSample;
-    }
+    const slices = await encodeAudioBufferToOpusSliceData(
+      audioBuffer,
+      targetBitrate
+    );
 
     logger.debug(`Encoded audio blob to ${slices.length} Opus slices`);
-    return slices;
+    return slices.map(slice => slice.data);
+  } finally {
+    await audioContext.close();
+  }
+}
+
+async function encodeAudioBufferToOpusSliceData(
+  audioBuffer: AudioBuffer,
+  targetBitrate: number
+): Promise<EncodedAudioSlice[]> {
+  const slices: EncodedAudioSlice[] = [];
+  const sampleRate = audioBuffer.sampleRate;
+  const numberOfChannels = audioBuffer.numberOfChannels;
+  const maxSliceSamples = MAX_SLICE_DURATION_SECONDS * sampleRate;
+  const minSliceSamples = MIN_SLICE_DURATION_SECONDS * sampleRate;
+  const totalSamples = audioBuffer.length;
+
+  let startSample = 0;
+  while (startSample < totalSamples) {
+    let endSample = Math.min(startSample + maxSliceSamples, totalSamples);
+    endSample = findSlicePoint(
+      audioBuffer,
+      startSample,
+      endSample,
+      minSliceSamples
+    );
+
+    const audioData = extractAudioData(audioBuffer, startSample, endSample);
+    const { encoder, encodedChunks } = createOpusEncoder({
+      sampleRate,
+      numberOfChannels,
+      bitrate: targetBitrate,
+    });
+
+    await encodeAudioFrames({
+      audioData,
+      numberOfChannels,
+      sampleRate,
+      encoder,
+    });
+
+    slices.push({
+      data: muxToMp4(encodedChunks, {
+        sampleRate,
+        numberOfChannels,
+        bitrate: targetBitrate,
+      }),
+      startSec: startSample / sampleRate,
+      durationSec: (endSample - startSample) / sampleRate,
+    });
+
+    startSample = endSample;
+  }
+
+  return slices;
+}
+
+export async function preprocessAudioBlobForTranscription(
+  blob: Blob | ArrayBuffer | Uint8Array,
+  options: {
+    fileNameBase: string;
+    sourceMimeType?: string;
+    targetBitrate?: number;
+  }
+): Promise<{
+  files: File[];
+  sourceAudio: TranscriptionSourceAudioInput;
+  sliceManifest: AudioSliceManifestItemInput[];
+}> {
+  const audioContext = new AudioContext();
+
+  try {
+    const arrayBuffer = await blobToArrayBuffer(blob);
+    const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+    const encodedSlices = await encodeAudioBufferToOpusSliceData(
+      audioBuffer,
+      options.targetBitrate ?? DEFAULT_BITRATE
+    );
+
+    const files = encodedSlices.map((slice, index) => {
+      const fileName = getSliceName(options.fileNameBase, index);
+      const data = toArrayBuffer(slice.data);
+      return new File([new Blob([data], { type: SLICE_MIME_TYPE })], fileName, {
+        type: SLICE_MIME_TYPE,
+      });
+    });
+
+    return {
+      files,
+      sourceAudio: {
+        mimeType:
+          options.sourceMimeType ?? (blob instanceof Blob ? blob.type : null),
+        durationMs: Math.round(audioBuffer.duration * 1000),
+        sampleRate: audioBuffer.sampleRate,
+        channels: audioBuffer.numberOfChannels,
+      },
+      sliceManifest: encodedSlices.map((slice, index) => ({
+        index,
+        fileName:
+          files[index]?.name ?? getSliceName(options.fileNameBase, index),
+        mimeType: files[index]?.type || SLICE_MIME_TYPE,
+        startSec: slice.startSec,
+        durationSec: slice.durationSec,
+        byteSize: slice.data.byteLength,
+      })),
+    };
   } finally {
     await audioContext.close();
   }