feat(editor): audio block (#10947)

AudioMedia entity for loading & controlling a single audio media
AudioMediaManagerService: Global audio state synchronization across tabs
AudioAttachmentService + AudioAttachmentBlock for manipulating AttachmentBlock in affine - e.g., filling transcription (using mock endpoint for now)
Added AudioBlock + AudioPlayer for rendering audio block in affine (new transcription block whose renderer is provided in affine)

fix AF-2292
fix AF-2337

packages/frontend/apps/electron/src/helper/index.ts

@@ -12,7 +12,7 @@ function setupRendererConnection(rendererPort: Electron.MessagePortMain) {
           try {
             const start = performance.now();
             const result = await handler(...args);
-            logger.info(
+            logger.debug(
               '[async-api]',
               `${namespace}.${name}`,
               args.filter(
@@ -74,7 +74,7 @@ function main() {
     if (e.data.channel === 'renderer-connect' && e.ports.length === 1) {
       const rendererPort = e.ports[0];
       setupRendererConnection(rendererPort);
-      logger.info('[helper] renderer connected');
+      logger.debug('[helper] renderer connected');
     }
   });
 }

packages/frontend/apps/electron/src/helper/logger.ts

@@ -3,3 +3,4 @@ import log from 'electron-log/main';
 export const logger = log.scope('helper');
 
 log.transports.file.level = 'info';
+log.transports.console.level = 'info';

packages/frontend/apps/electron/src/helper/main-rpc.ts

@@ -2,7 +2,6 @@ import { AsyncCall } from 'async-call-rpc';
 
 import type { HelperToMain, MainToHelper } from '../shared/type';
 import { exposed } from './provide';
-import { encodeToMp3 } from './recording/encode';
 
 const helperToMainServer: HelperToMain = {
   getMeta: () => {
@@ -11,8 +10,6 @@ const helperToMainServer: HelperToMain = {
     }
     return exposed;
   },
-  // allow main process encode audio samples to mp3 buffer (because it is slow and blocking)
-  encodeToMp3,
 };
 
 export const mainRPC = AsyncCall<MainToHelper>(helperToMainServer, {
@@ -33,4 +30,5 @@ export const mainRPC = AsyncCall<MainToHelper>(helperToMainServer, {
       process.parentPort.postMessage(data);
     },
   },
+  log: false,
 });

packages/frontend/apps/electron/src/helper/recording/encode.ts

@@ -1,16 +0,0 @@
-import { Mp3Encoder } from '@affine/native';
-
-// encode audio samples to mp3 buffer
-export function encodeToMp3(
-  samples: Float32Array,
-  opts: {
-    channels?: number;
-    sampleRate?: number;
-  } = {}
-): Uint8Array {
-  const mp3Encoder = new Mp3Encoder({
-    channels: opts.channels ?? 2,
-    sampleRate: opts.sampleRate ?? 44100,
-  });
-  return mp3Encoder.encode(samples);
-}

packages/frontend/apps/electron/src/main/cleanup.ts

@@ -1,15 +1,22 @@
+import type { MediaStats } from '@toeverything/infra';
 import { app } from 'electron';
 
 import { logger } from './logger';
+import { globalStateStorage } from './shared-storage/storage';
 
-const cleanupRegistry: (() => void)[] = [];
+const beforeAppQuitRegistry: (() => void)[] = [];
+const beforeTabCloseRegistry: ((tabId: string) => void)[] = [];
 
 export function beforeAppQuit(fn: () => void) {
-  cleanupRegistry.push(fn);
+  beforeAppQuitRegistry.push(fn);
+}
+
+export function beforeTabClose(fn: (tabId: string) => void) {
+  beforeTabCloseRegistry.push(fn);
 }
 
 app.on('before-quit', () => {
-  cleanupRegistry.forEach(fn => {
+  beforeAppQuitRegistry.forEach(fn => {
     // some cleanup functions might throw on quit and crash the app
     try {
       fn();
@@ -18,3 +25,32 @@ app.on('before-quit', () => {
     }
   });
 });
+
+export function onTabClose(tabId: string) {
+  beforeTabCloseRegistry.forEach(fn => {
+    try {
+      fn(tabId);
+    } catch (err) {
+      logger.warn('cleanup error on tab close', err);
+    }
+  });
+}
+
+app.on('ready', () => {
+  globalStateStorage.set('media:playback-state', null);
+  globalStateStorage.set('media:stats', null);
+});
+
+beforeAppQuit(() => {
+  globalStateStorage.set('media:playback-state', null);
+  globalStateStorage.set('media:stats', null);
+});
+
+// set audio play state
+beforeTabClose(tabId => {
+  const stats = globalStateStorage.get<MediaStats | null>('media:stats');
+  if (stats && stats.tabId === tabId) {
+    globalStateStorage.set('media:playback-state', null);
+    globalStateStorage.set('media:stats', null);
+  }
+});

packages/frontend/apps/electron/src/main/helper-process.ts

@@ -115,6 +115,7 @@ class HelperProcessManager {
         unknownMessage: false,
       },
       channel: new MessageEventChannel(this.#process),
+      log: false,
     });
   }
 }

packages/frontend/apps/electron/src/main/index.ts

@@ -15,7 +15,7 @@ import { registerHandlers } from './handlers';
 import { logger } from './logger';
 import { registerProtocol } from './protocol';
 import { setupRecording } from './recording';
-import { getTrayState } from './tray';
+import { setupTrayState } from './tray';
 import { registerUpdater } from './updater';
 import { launch } from './windows-manager/launcher';
 import { launchStage } from './windows-manager/stage';
@@ -89,7 +89,7 @@ app
   .then(launch)
   .then(setupRecording)
   .then(createApplicationMenu)
-  .then(getTrayState)
+  .then(setupTrayState)
   .then(registerUpdater)
   .catch(e => console.error('Failed create window:', e));
 

packages/frontend/apps/electron/src/main/logger.ts

@@ -7,6 +7,7 @@ log.initialize({
 });
 
 log.transports.file.level = 'info';
+log.transports.console.level = 'info';
 
 export function getLogFilePath() {
   return log.transports.file.getFile().path;

packages/frontend/apps/electron/src/main/protocol.ts

@@ -1,6 +1,6 @@
 import { join } from 'node:path';
 
-import { net, protocol, session } from 'electron';
+import { app, net, protocol, session } from 'electron';
 import cookieParser from 'set-cookie-parser';
 
 import { resourcesPath } from '../shared/utils';
@@ -43,8 +43,10 @@ function isNetworkResource(pathname: string) {
 async function handleFileRequest(request: Request) {
   const urlObject = new URL(request.url);
 
+  const isAbsolutePath = urlObject.host !== '.';
+
   // Redirect to webpack dev server if defined
-  if (process.env.DEV_SERVER_URL) {
+  if (process.env.DEV_SERVER_URL && !isAbsolutePath) {
     const devServerUrl = new URL(
       urlObject.pathname,
       process.env.DEV_SERVER_URL
@@ -56,20 +58,30 @@ async function handleFileRequest(request: Request) {
   });
   // this will be file types (in the web-static folder)
   let filepath = '';
-  // if is a file type, load the file in resources
-  if (urlObject.pathname.split('/').at(-1)?.includes('.')) {
-    // Sanitize pathname to prevent path traversal attacks
-    const decodedPath = decodeURIComponent(urlObject.pathname);
-    const normalizedPath = join(webStaticDir, decodedPath).normalize();
-    if (!normalizedPath.startsWith(webStaticDir)) {
-      // Attempted path traversal - reject by using empty path
-      filepath = join(webStaticDir, '');
+
+  // for relative path, load the file in resources
+  if (!isAbsolutePath) {
+    if (urlObject.pathname.split('/').at(-1)?.includes('.')) {
+      // Sanitize pathname to prevent path traversal attacks
+      const decodedPath = decodeURIComponent(urlObject.pathname);
+      const normalizedPath = join(webStaticDir, decodedPath).normalize();
+      if (!normalizedPath.startsWith(webStaticDir)) {
+        // Attempted path traversal - reject by using empty path
+        filepath = join(webStaticDir, '');
+      } else {
+        filepath = normalizedPath;
+      }
     } else {
-      filepath = normalizedPath;
+      // else, fallback to load the index.html instead
+      filepath = join(webStaticDir, 'index.html');
     }
   } else {
-    // else, fallback to load the index.html instead
-    filepath = join(webStaticDir, 'index.html');
+    filepath = decodeURIComponent(urlObject.pathname);
+    // security check if the filepath is within app.getPath('sessionData')
+    const sessionDataPath = app.getPath('sessionData');
+    if (!filepath.startsWith(sessionDataPath)) {
+      throw new Error('Invalid filepath');
+    }
   }
   return net.fetch('file://' + filepath, clonedRequest);
 }

packages/frontend/apps/electron/src/main/recording/index.ts

@@ -1,11 +1,13 @@
+import path from 'node:path';
+
 import { ShareableContent } from '@affine/native';
-import { nativeImage, Notification } from 'electron';
+import { app, nativeImage, Notification } from 'electron';
+import fs from 'fs-extra';
 import { debounce } from 'lodash-es';
 import { BehaviorSubject, distinctUntilChanged, groupBy, mergeMap } from 'rxjs';
 
 import { isMacOS } from '../../shared/utils';
 import { beforeAppQuit } from '../cleanup';
-import { ensureHelperProcess } from '../helper-process';
 import { logger } from '../logger';
 import type { NamespaceHandlers } from '../type';
 import { getMainWindow } from '../windows-manager';
@@ -18,6 +20,11 @@ import type {
 
 const subscribers: Subscriber[] = [];
 
+const SAVED_RECORDINGS_DIR = path.join(
+  app.getPath('sessionData'),
+  'recordings'
+);
+
 beforeAppQuit(() => {
   subscribers.forEach(subscriber => {
     try {
@@ -134,7 +141,9 @@ function setupNewRunningAppGroup() {
           recordingStatus$.value?.appGroup?.processGroupId ===
             currentGroup.processGroupId
         ) {
-          stopRecording();
+          stopRecording().catch(err => {
+            logger.error('failed to stop recording', err);
+          });
         }
       }
     })
@@ -142,7 +151,13 @@ function setupNewRunningAppGroup() {
 }
 
 function createRecording(status: RecordingStatus) {
-  const buffers: Float32Array[] = [];
+  const bufferedFilePath = path.join(
+    SAVED_RECORDINGS_DIR,
+    `${status.appGroup?.bundleIdentifier ?? 'unknown'}-${status.id}-${status.startTime}.raw`
+  );
+
+  fs.ensureDirSync(SAVED_RECORDINGS_DIR);
+  const file = fs.createWriteStream(bufferedFilePath);
 
   function tapAudioSamples(err: Error | null, samples: Float32Array) {
     const recordingStatus = recordingStatus$.getValue();
@@ -157,7 +172,9 @@ function createRecording(status: RecordingStatus) {
     if (err) {
       logger.error('failed to get audio samples', err);
     } else {
-      buffers.push(new Float32Array(samples));
+      // Writing raw Float32Array samples directly to file
+      // For stereo audio, samples are interleaved [L,R,L,R,...]
+      file.write(Buffer.from(samples.buffer));
     }
   }
 
@@ -170,44 +187,29 @@ function createRecording(status: RecordingStatus) {
     startTime: status.startTime,
     app: status.app,
     appGroup: status.appGroup,
-    buffers,
+    file,
     stream,
   };
 
   return recording;
 }
 
-function concatBuffers(buffers: Float32Array[]): Float32Array {
-  const totalSamples = buffers.reduce((acc, buf) => acc + buf.length, 0);
-  const buffer = new Float32Array(totalSamples);
-  let offset = 0;
-  buffers.forEach(buf => {
-    buffer.set(buf, offset);
-    offset += buf.length;
-  });
-  return buffer;
-}
-
-export async function saveRecording(id: number) {
+export async function getRecording(id: number) {
   const recording = recordings.get(id);
   if (!recording) {
     logger.error(`Recording ${id} not found`);
     return;
   }
-  const { buffers } = recording;
-  const helperProcessManager = await ensureHelperProcess();
-  const buffer = concatBuffers(buffers);
-  const mp3Buffer = await helperProcessManager.rpc?.encodeToMp3(buffer, {
-    channels: recording.stream.channels,
+  const rawFilePath = String(recording.file.path);
+  return {
+    id,
+    appGroup: recording.appGroup,
+    app: recording.app,
+    startTime: recording.startTime,
+    filepath: rawFilePath,
     sampleRate: recording.stream.sampleRate,
-  });
-
-  if (!mp3Buffer) {
-    logger.error('failed to encode audio samples to mp3');
-    return;
-  }
-  recordings.delete(recording.id);
-  return mp3Buffer;
+    numberOfChannels: recording.stream.channels,
+  };
 }
 
 function setupRecordingListeners() {
@@ -386,9 +388,15 @@ export function resumeRecording() {
   });
 }
 
-export function stopRecording() {
+export async function stopRecording() {
   const recordingStatus = recordingStatus$.value;
   if (!recordingStatus) {
+    logger.error('No recording status to stop');
+    return;
+  }
+  const recording = recordings.get(recordingStatus?.id);
+  if (!recording) {
+    logger.error(`Recording ${recordingStatus?.id} not found`);
     return;
   }
 
@@ -398,6 +406,15 @@ export function stopRecording() {
     status: 'stopped',
   });
 
+  const { file } = recording;
+  file.end();
+
+  await new Promise<void>(resolve => {
+    file.on('finish', () => {
+      resolve();
+    });
+  });
+
   // bring up the window
   getMainWindow()
     .then(mainWindow => {
@@ -411,8 +428,17 @@ export function stopRecording() {
 }
 
 export const recordingHandlers = {
-  saveRecording: async (_, id: number) => {
-    return saveRecording(id);
+  getRecording: async (_, id: number) => {
+    return getRecording(id);
+  },
+  deleteCachedRecording: async (_, id: number) => {
+    const recording = recordings.get(id);
+    if (recording) {
+      recording.stream.stop();
+      recordings.delete(id);
+      await fs.unlink(recording.file.path);
+    }
+    return true;
   },
 } satisfies NamespaceHandlers;
 

packages/frontend/apps/electron/src/main/recording/types.ts

@@ -1,3 +1,5 @@
+import type { WriteStream } from 'node:fs';
+
 import type { AudioTapStream, TappableApplication } from '@affine/native';
 
 export interface TappableAppInfo {
@@ -23,8 +25,8 @@ export interface Recording {
   // the app may not be available if the user choose to record system audio
   app?: TappableAppInfo;
   appGroup?: AppGroupInfo;
-  // the raw audio buffers that are already accumulated
-  buffers: Float32Array[];
+  // the buffered file that is being recorded streamed to
+  file: WriteStream;
   stream: AudioTapStream;
   startTime: number;
 }

packages/frontend/apps/electron/src/main/tray/index.ts

@@ -198,7 +198,9 @@ class TrayState {
           label: 'Stop',
           click: () => {
             logger.info('User action: Stop Recording');
-            stopRecording();
+            stopRecording().catch(err => {
+              logger.error('Failed to stop recording:', err);
+            });
           },
         },
       ],
@@ -290,7 +292,7 @@ class TrayState {
 
 let _trayState: TrayState | undefined;
 
-export const getTrayState = () => {
+export const setupTrayState = () => {
   if (!_trayState) {
     _trayState = new TrayState();
     _trayState.init();

packages/frontend/apps/electron/src/main/ui/events.ts

@@ -35,6 +35,12 @@ export const uiEvents = {
   },
   onTabsStatusChange,
   onActiveTabChanged,
+  onTabGoToRequest: (fn: (opts: { tabId: string; to: string }) => void) => {
+    const sub = uiSubjects.tabGoToRequest$.subscribe(fn);
+    return () => {
+      sub.unsubscribe();
+    };
+  },
   onTabShellViewActiveChange,
   onAuthenticationRequest: (fn: (state: AuthenticationRequest) => void) => {
     const sub = uiSubjects.authenticationRequest$.subscribe(fn);

packages/frontend/apps/electron/src/main/ui/handlers.ts

@@ -10,6 +10,7 @@ import {
   activateView,
   addTab,
   closeTab,
+  ensureTabLoaded,
   getMainWindow,
   getOnboardingWindow,
   getTabsStatus,
@@ -187,6 +188,12 @@ export const uiHandlers = {
   showTab: async (_, ...args: Parameters<typeof showTab>) => {
     await showTab(...args);
   },
+  tabGoTo: async (_, tabId: string, to: string) => {
+    uiSubjects.tabGoToRequest$.next({ tabId, to });
+  },
+  ensureTabLoaded: async (_, ...args: Parameters<typeof ensureTabLoaded>) => {
+    await ensureTabLoaded(...args);
+  },
   closeTab: async (_, ...args: Parameters<typeof closeTab>) => {
     await closeTab(...args);
   },

packages/frontend/apps/electron/src/main/ui/subject.ts

@@ -6,6 +6,7 @@ export const uiSubjects = {
   onMaximized$: new Subject<boolean>(),
   onFullScreen$: new Subject<boolean>(),
   onToggleRightSidebar$: new Subject<string>(),
+  tabGoToRequest$: new Subject<{ tabId: string; to: string }>(),
   authenticationRequest$: new Subject<AuthenticationRequest>(),
   // via menu -> close view (CMD+W)
   onCloseView$: new Subject<void>(),

packages/frontend/apps/electron/src/main/windows-manager/tab-views.ts

@@ -24,7 +24,7 @@ import {
 } from 'rxjs';
 
 import { isMacOS } from '../../shared/utils';
-import { beforeAppQuit } from '../cleanup';
+import { beforeAppQuit, onTabClose } from '../cleanup';
 import { mainWindowOrigin, shellViewUrl } from '../constants';
 import { ensureHelperProcess } from '../helper-process';
 import { logger } from '../logger';
@@ -400,6 +400,8 @@ export class WebContentViewsManager {
         });
       }
     }, 500); // delay a bit to get rid of the flicker
+
+    onTabClose(id);
   };
 
   undoCloseTab = async () => {
@@ -1052,6 +1054,13 @@ export const loadUrlInActiveTab = async (_url: string) => {
   // todo: implement
   throw new Error('loadUrlInActiveTab not implemented');
 };
+export const ensureTabLoaded = async (tabId: string) => {
+  const tab = WebContentViewsManager.instance.tabViewsMap.get(tabId);
+  if (tab) {
+    return tab;
+  }
+  return WebContentViewsManager.instance.loadTab(tabId);
+};
 export const showTab = WebContentViewsManager.instance.showTab;
 export const closeTab = WebContentViewsManager.instance.closeTab;
 export const undoCloseTab = WebContentViewsManager.instance.undoCloseTab;

packages/frontend/apps/electron/src/shared/type.ts

@@ -17,13 +17,6 @@ export interface HelperToRenderer {
 // helper <-> main
 export interface HelperToMain {
   getMeta: () => ExposedMeta;
-  encodeToMp3: (
-    samples: Float32Array,
-    opts?: {
-      channels?: number;
-      sampleRate?: number;
-    }
-  ) => Uint8Array;
 }
 
 export type MainToHelper = Pick<