refactor(electron): encoding recording on the fly (#11457)

fix AF-2460, AF-2463

When recording is started, we start polling the pending raw buffers that are waiting for encoding. The buffers are determined by the cursor of the original raw buffer file. When recording is stopped, we will flush the pending buffers and wrap the encoded chunks into WebM.

```mermaid
sequenceDiagram
    participant App as App/UI
    participant RecordingFeature as Recording Feature
    participant StateMachine as State Machine
    participant FileSystem as File System
    participant StreamEncoder as Stream Encoder
    participant OpusEncoder as Opus Encoder
    participant WebM as WebM Muxer

    Note over App,WebM: Recording Start Flow
    App->>RecordingFeature: startRecording()
    RecordingFeature->>StateMachine: dispatch(START_RECORDING)
    StateMachine-->>RecordingFeature: status: 'recording'
    RecordingFeature->>StreamEncoder: createStreamEncoder(id, {sampleRate, channels})

    Note over App,WebM: Streaming Flow
    loop Audio Data Streaming
        RecordingFeature->>FileSystem: Write raw audio chunks to .raw file
        StreamEncoder->>FileSystem: Poll raw audio data
        FileSystem-->>StreamEncoder: Raw audio chunks
        StreamEncoder->>OpusEncoder: Encode chunks
        OpusEncoder-->>StreamEncoder: Encoded Opus frames
    end

    Note over App,WebM: Recording Stop Flow
    App->>RecordingFeature: stopRecording()
    RecordingFeature->>StateMachine: dispatch(STOP_RECORDING)
    StateMachine-->>RecordingFeature: status: 'stopped'
    StreamEncoder->>OpusEncoder: flush()
    StreamEncoder->>WebM: muxToWebM(encodedChunks)
    WebM-->>RecordingFeature: WebM buffer
    RecordingFeature->>FileSystem: Save as .opus file
    RecordingFeature->>StateMachine: dispatch(SAVE_RECORDING)
```

packages/frontend/apps/electron/src/main/recording/feature.ts

@@ -1,5 +1,6 @@
 /* oxlint-disable no-var-requires */
 import { execSync } from 'node:child_process';
+import fsp from 'node:fs/promises';
 import path from 'node:path';
 
 // Should not load @affine/native for unsupported platforms
@@ -240,7 +241,12 @@ function setupNewRunningAppGroup() {
   );
 }
 
-function createRecording(status: RecordingStatus) {
+export function createRecording(status: RecordingStatus) {
+  let recording = recordings.get(status.id);
+  if (recording) {
+    return recording;
+  }
+
   const bufferedFilePath = path.join(
     SAVED_RECORDINGS_DIR,
     `${status.appGroup?.bundleIdentifier ?? 'unknown'}-${status.id}-${status.startTime}.raw`
@@ -275,7 +281,7 @@ function createRecording(status: RecordingStatus) {
     ? status.app.rawInstance.tapAudio(tapAudioSamples)
     : ShareableContent.tapGlobalAudio(null, tapAudioSamples);
 
-  const recording: Recording = {
+  recording = {
     id: status.id,
     startTime: status.startTime,
     app: status.app,
@@ -284,6 +290,8 @@ function createRecording(status: RecordingStatus) {
     stream,
   };
 
+  recordings.set(status.id, recording);
+
   return recording;
 }
 
@@ -330,7 +338,6 @@ function setupRecordingListeners() {
           // create a recording if not exists
           if (!recording) {
             recording = createRecording(status);
-            recordings.set(status.id, recording);
           }
         } else if (status?.status === 'stopped') {
           const recording = recordings.get(status.id);
@@ -518,6 +525,10 @@ export function startRecording(
     appGroup: normalizeAppGroupInfo(appGroup),
   });
 
+  if (state?.status === 'recording') {
+    createRecording(state);
+  }
+
   // set a timeout to stop the recording after MAX_DURATION_FOR_TRANSCRIPTION
   setTimeout(() => {
     if (
@@ -544,7 +555,7 @@ export function resumeRecording(id: number) {
 export async function stopRecording(id: number) {
   const recording = recordings.get(id);
   if (!recording) {
-    logger.error(`Recording ${id} not found`);
+    logger.error(`stopRecording: Recording ${id} not found`);
     return;
   }
 
@@ -590,9 +601,6 @@ export async function stopRecording(id: number) {
     const recordingStatus = recordingStateMachine.dispatch({
       type: 'STOP_RECORDING',
       id,
-      filepath: String(recording.file.path),
-      sampleRate: recording.stream.sampleRate,
-      numberOfChannels: recording.stream.channels,
     });
 
     if (!recordingStatus) {
@@ -620,11 +628,35 @@ export async function stopRecording(id: number) {
   }
 }
 
+export async function getRawAudioBuffers(
+  id: number,
+  cursor?: number
+): Promise<{
+  buffer: Buffer;
+  nextCursor: number;
+}> {
+  const recording = recordings.get(id);
+  if (!recording) {
+    throw new Error(`getRawAudioBuffers: Recording ${id} not found`);
+  }
+  const start = cursor ?? 0;
+  const file = await fsp.open(recording.file.path, 'r');
+  const stats = await file.stat();
+  const buffer = Buffer.alloc(stats.size - start);
+  const result = await file.read(buffer, 0, buffer.length, start);
+  await file.close();
+
+  return {
+    buffer,
+    nextCursor: start + result.bytesRead,
+  };
+}
+
 export async function readyRecording(id: number, buffer: Buffer) {
   const recordingStatus = recordingStatus$.value;
   const recording = recordings.get(id);
   if (!recordingStatus || recordingStatus.id !== id || !recording) {
-    logger.error(`Recording ${id} not found`);
+    logger.error(`readyRecording: Recording ${id} not found`);
     return;
   }
 
@@ -635,6 +667,16 @@ export async function readyRecording(id: number, buffer: Buffer) {
 
   await fs.writeFile(filepath, buffer);
 
+  // can safely remove the raw file now
+  const rawFilePath = recording.file.path;
+  logger.info('remove raw file', rawFilePath);
+  if (rawFilePath) {
+    try {
+      await fs.unlink(rawFilePath);
+    } catch (err) {
+      logger.error('failed to remove raw file', err);
+    }
+  }
   // Update the status through the state machine
   recordingStateMachine.dispatch({
     type: 'SAVE_RECORDING',
@@ -689,7 +731,8 @@ export interface SerializedRecordingStatus {
 
 export function serializeRecordingStatus(
   status: RecordingStatus
-): SerializedRecordingStatus {
+): SerializedRecordingStatus | null {
+  const recording = recordings.get(status.id);
   return {
     id: status.id,
     status: status.status,
@@ -697,9 +740,10 @@ export function serializeRecordingStatus(
     appGroupId: status.appGroup?.processGroupId,
     icon: status.appGroup?.icon,
     startTime: status.startTime,
-    filepath: status.filepath,
-    sampleRate: status.sampleRate,
-    numberOfChannels: status.numberOfChannels,
+    filepath:
+      status.filepath ?? (recording ? String(recording.file.path) : undefined),
+    sampleRate: recording?.stream.sampleRate,
+    numberOfChannels: recording?.stream.channels,
   };
 }
 

packages/frontend/apps/electron/src/main/recording/index.ts

@@ -12,6 +12,7 @@ import {
   checkRecordingAvailable,
   checkScreenRecordingPermission,
   disableRecordingFeature,
+  getRawAudioBuffers,
   getRecording,
   handleBlockCreationFailed,
   handleBlockCreationSuccess,
@@ -47,6 +48,9 @@ export const recordingHandlers = {
   stopRecording: async (_, id: number) => {
     return stopRecording(id);
   },
+  getRawAudioBuffers: async (_, id: number, cursor?: number) => {
+    return getRawAudioBuffers(id, cursor);
+  },
   // save the encoded recording buffer to the file system
   readyRecording: async (_, id: number, buffer: Uint8Array) => {
     return readyRecording(id, Buffer.from(buffer));

packages/frontend/apps/electron/src/main/recording/state-machine.ts

@@ -9,15 +9,15 @@ import type { AppGroupInfo, RecordingStatus } from './types';
  */
 export type RecordingEvent =
   | { type: 'NEW_RECORDING'; appGroup?: AppGroupInfo }
-  | { type: 'START_RECORDING'; appGroup?: AppGroupInfo }
+  | {
+      type: 'START_RECORDING';
+      appGroup?: AppGroupInfo;
+    }
   | { type: 'PAUSE_RECORDING'; id: number }
   | { type: 'RESUME_RECORDING'; id: number }
   | {
       type: 'STOP_RECORDING';
       id: number;
-      filepath: string;
-      sampleRate: number;
-      numberOfChannels: number;
     }
   | {
       type: 'SAVE_RECORDING';
@@ -81,12 +81,7 @@ export class RecordingStateMachine {
         newStatus = this.handleResumeRecording();
         break;
       case 'STOP_RECORDING':
-        newStatus = this.handleStopRecording(
-          event.id,
-          event.filepath,
-          event.sampleRate,
-          event.numberOfChannels
-        );
+        newStatus = this.handleStopRecording(event.id);
         break;
       case 'SAVE_RECORDING':
         newStatus = this.handleSaveRecording(event.id, event.filepath);
@@ -208,12 +203,7 @@ export class RecordingStateMachine {
   /**
    * Handle the STOP_RECORDING event
    */
-  private handleStopRecording(
-    id: number,
-    filepath: string,
-    sampleRate: number,
-    numberOfChannels: number
-  ): RecordingStatus | null {
+  private handleStopRecording(id: number): RecordingStatus | null {
     const currentStatus = this.recordingStatus$.value;
 
     if (!currentStatus || currentStatus.id !== id) {
@@ -232,9 +222,6 @@ export class RecordingStateMachine {
     return {
       ...currentStatus,
       status: 'stopped',
-      filepath,
-      sampleRate,
-      numberOfChannels,
     };
   }
 

packages/frontend/apps/electron/src/main/recording/types.ts

@@ -29,6 +29,7 @@ export interface Recording {
   file: WriteStream;
   stream: AudioTapStream;
   startTime: number;
+  filepath?: string; // the filepath of the recording (only available when status is ready)
 }
 
 export interface RecordingStatus {
@@ -52,7 +53,5 @@ export interface RecordingStatus {
   app?: TappableAppInfo;
   appGroup?: AppGroupInfo;
   startTime: number; // 0 means not started yet
-  filepath?: string; // the filepath of the recording (only available when status is ready)
-  sampleRate?: number;
-  numberOfChannels?: number;
+  filepath?: string; // encoded file path
 }

packages/frontend/apps/electron/src/main/windows-manager/popup.ts

@@ -1,7 +1,11 @@
 import { join } from 'node:path';
 import { setTimeout } from 'node:timers/promises';
 
-import { BrowserWindow, type BrowserWindowConstructorOptions } from 'electron';
+import {
+  app,
+  BrowserWindow,
+  type BrowserWindowConstructorOptions,
+} from 'electron';
 import { BehaviorSubject } from 'rxjs';
 
 import { popupViewUrl } from '../constants';
@@ -96,6 +100,9 @@ abstract class PopupWindow {
       },
     });
 
+    // it seems that the dock will disappear when popup windows are shown
+    await app.dock?.show();
+
     // required to make the window transparent
     browserWindow.setBackgroundColor('#00000000');
     browserWindow.setVisibleOnAllWorkspaces(true, {