feat(electron): recording popups (#11016)

Added a recording popup UI for the audio recording feature in the desktop app, improving the user experience when capturing audio from applications. ### What changed? - Created a new popup window system for displaying recording controls - Added a dedicated recording UI with start/stop controls and status indicators - Moved audio encoding logic from the main app to a dedicated module - Implemented smooth animations for popup appearance/disappearance - Updated the recording workflow to show visual feedback during recording process - Added internationalization support for recording-related text - Modified the recording status flow to include new states: new, recording, stopped, ready fix AF-2340
2026-08-19 02:51:47 +08:00 · 2025-03-26 04:53:43 +00:00
parent 96e83a2141
commit 61c0d01da3
38 changed files with 3611 additions and 383 deletions
@@ -5,134 +5,21 @@ import { AudioAttachmentService } from '@affine/core/modules/media/services/audi
 import { WorkbenchService } from '@affine/core/modules/workbench';
 import { DebugLogger } from '@affine/debug';
 import { apis, events } from '@affine/electron-api';
+import { i18nTime } from '@affine/i18n';
 import type { AttachmentBlockModel } from '@blocksuite/affine/model';
 import { Text } from '@blocksuite/affine/store';
 import type { BlobEngine } from '@blocksuite/affine/sync';
 import type { FrameworkProvider } from '@toeverything/infra';
-import { ArrayBufferTarget, Muxer } from 'webm-muxer';

 import { getCurrentWorkspace } from './utils';

 const logger = new DebugLogger('electron-renderer:recording');

-/**
- * Encodes raw audio data to Opus in WebM container.
- */
-async function encodeRawBufferToOpus({
-  filepath,
-  sampleRate,
-  numberOfChannels,
-}: {
-  filepath: string;
-  sampleRate: number;
-  numberOfChannels: number;
-}): Promise<Uint8Array> {
-  // Use streams to process audio data incrementally
-  const response = await fetch(new URL(filepath, location.origin));
-  if (!response.body) {
-    throw new Error('Response body is null');
-  }
-
-  // Setup Opus encoder
-  const encodedChunks: EncodedAudioChunk[] = [];
-  const encoder = new AudioEncoder({
-    output: chunk => {
-      encodedChunks.push(chunk);
-    },
-    error: err => {
-      throw new Error(`Encoding error: ${err}`);
-    },
-  });
-
-  // Configure Opus encoder
-  encoder.configure({
-    codec: 'opus',
-    sampleRate: sampleRate,
-    numberOfChannels: numberOfChannels,
-    bitrate: 96000, // 96 kbps is good for stereo audio
-  });
-
-  // Process the stream
-  const reader = response.body.getReader();
-  let offset = 0;
-  const CHUNK_SIZE = numberOfChannels * 1024; // Process 1024 samples per channel at a time
-
-  try {
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done) break;
-
-      // Convert the chunk to Float32Array
-      const float32Data = new Float32Array(value.buffer);
-
-      // Process in smaller chunks to avoid large frames
-      for (let i = 0; i < float32Data.length; i += CHUNK_SIZE) {
-        const chunkSize = Math.min(CHUNK_SIZE, float32Data.length - i);
-        const chunk = float32Data.subarray(i, i + chunkSize);
-
-        // Create and encode frame
-        const frame = new AudioData({
-          format: 'f32',
-          sampleRate: sampleRate,
-          numberOfFrames: chunk.length / numberOfChannels,
-          numberOfChannels: numberOfChannels,
-          timestamp: (offset * 1000000) / sampleRate, // timestamp in microseconds
-          data: chunk,
-        });
-
-        encoder.encode(frame);
-        frame.close();
-
-        offset += chunk.length / numberOfChannels;
-      }
-    }
-  } finally {
-    await encoder.flush();
-    encoder.close();
-  }
-
-  if (encodedChunks.length === 0) {
-    throw new Error('No chunks were produced during encoding');
-  }
-
-  // Initialize WebM muxer
-  const target = new ArrayBufferTarget();
-  const muxer = new Muxer({
-    target,
-    audio: {
-      codec: 'A_OPUS',
-      sampleRate: sampleRate,
-      numberOfChannels: numberOfChannels,
-    },
-  });
-
-  // Add all chunks to the muxer
-  for (const chunk of encodedChunks) {
-    muxer.addAudioChunk(chunk, {});
-  }
-
-  // Finalize and get WebM container
-  muxer.finalize();
-  const { buffer: webmBuffer } = target;
-
-  return new Uint8Array(webmBuffer);
-}
-
-async function saveRecordingBlob(
-  blobEngine: BlobEngine,
-  recording: {
-    id: number;
-    filepath: string;
-    sampleRate: number;
-    numberOfChannels: number;
-  }
-) {
-  logger.debug('Saving recording', recording.id);
-  const opusBuffer = await encodeRawBufferToOpus({
-    filepath: recording.filepath,
-    sampleRate: recording.sampleRate,
-    numberOfChannels: recording.numberOfChannels,
-  });
+async function saveRecordingBlob(blobEngine: BlobEngine, filepath: string) {
+  logger.debug('Saving recording', filepath);
+  const opusBuffer = await fetch(new URL(filepath, location.origin)).then(res =>
+    res.arrayBuffer()
+  );
  const blob = new Blob([opusBuffer], {
    type: 'audio/webm',
  });
@@ -144,7 +31,7 @@ async function saveRecordingBlob(
 export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
  events?.recording.onRecordingStatusChanged(status => {
    (async () => {
-      if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') {
+      if ((await apis?.ui.isActiveTab()) && status?.status === 'ready') {
        using currentWorkspace = getCurrentWorkspace(frameworkProvider);
        if (!currentWorkspace) {
          return;
@@ -155,30 +42,28 @@ export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
        const docsService = workspace.scope.get(DocsService);
        const editorSetting = editorSettingService.editorSetting;

+        const timestamp = i18nTime(status.startTime, {
+          absolute: {
+            accuracy: 'minute',
+            noYear: true,
+          },
+        });
+
        const docProps: DocProps = {
          note: editorSetting.get('affine:note'),
          page: {
            title: new Text(
              'Recording ' +
-                (status.appGroup?.name ?? 'System Audio') +
+                (status.appName ?? 'System Audio') +
                ' ' +
-                new Date(status.startTime).toISOString()
+                timestamp
            ),
          },
          onStoreLoad: (doc, { noteId }) => {
            (async () => {
-              const recording = await apis?.recording.getRecording(status.id);
-              if (!recording) {
-                logger.error('Failed to save recording');
-                return;
-              }
-
              // name + timestamp(readable) + extension
              const attachmentName =
-                (status.appGroup?.name ?? 'System Audio') +
-                ' ' +
-                new Date(status.startTime).toISOString() +
-                '.webm';
+                (status.appName ?? 'System Audio') + ' ' + timestamp + '.webm';

              // add size and sourceId to the attachment later
              const attachmentId = doc.addBlock(
@@ -193,11 +78,11 @@ export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
              const model = doc.getBlock(attachmentId)
                ?.model as AttachmentBlockModel;

-              if (model) {
+              if (model && status.filepath) {
                // it takes a while to save the blob, so we show the attachment first
                const { blobId, blob } = await saveRecordingBlob(
                  doc.workspace.blobSync,
-                  recording
+                  status.filepath
                );

                model.props.size = blob.size;
@@ -0,0 +1,16 @@
+import { globalStyle, style } from '@vanilla-extract/css';
+
+globalStyle('html', {
+  backgroundColor: 'transparent',
+  userSelect: 'none',
+});
+
+globalStyle('body', {
+  backgroundColor: 'transparent',
+});
+
+export const root = style({
+  backgroundColor: 'transparent',
+  width: '100%',
+  height: '100%',
+});
@@ -0,0 +1,37 @@
+import { ThemeProvider } from '@affine/core/components/theme-provider';
+import { configureDesktopApiModule } from '@affine/core/modules/desktop-api';
+import { configureI18nModule, I18nProvider } from '@affine/core/modules/i18n';
+import {
+  configureElectronStateStorageImpls,
+  configureStorageModule,
+} from '@affine/core/modules/storage';
+import { configureEssentialThemeModule } from '@affine/core/modules/theme';
+import { appInfo } from '@affine/electron-api';
+import { Framework, FrameworkRoot } from '@toeverything/infra';
+
+import * as styles from './app.css';
+import { Recording } from './recording';
+
+const framework = new Framework();
+configureI18nModule(framework);
+configureEssentialThemeModule(framework);
+configureStorageModule(framework);
+configureElectronStateStorageImpls(framework);
+configureDesktopApiModule(framework);
+const frameworkProvider = framework.provider();
+
+const mode = appInfo?.windowName as 'notification' | 'recording';
+
+export function App() {
+  return (
+    <FrameworkRoot framework={frameworkProvider}>
+      <ThemeProvider>
+        <I18nProvider>
+          <div className={styles.root}>
+            {mode === 'recording' && <Recording />}
+          </div>
+        </I18nProvider>
+      </ThemeProvider>
+    </FrameworkRoot>
+  );
+}
@@ -0,0 +1,24 @@
+import './setup';
+
+import { StrictMode } from 'react';
+import { createRoot } from 'react-dom/client';
+
+import { App } from './app';
+
+function main() {
+  mountApp();
+}
+
+function mountApp() {
+  const root = document.getElementById('app');
+  if (!root) {
+    throw new Error('Root element not found');
+  }
+  createRoot(root).render(
+    <StrictMode>
+      <App />
+    </StrictMode>
+  );
+}
+
+main();
@@ -0,0 +1,100 @@
+import { ArrayBufferTarget, Muxer } from 'webm-muxer';
+
+/**
+ * Encodes raw audio data to Opus in WebM container.
+ */
+export async function encodeRawBufferToOpus({
+  filepath,
+  sampleRate,
+  numberOfChannels,
+}: {
+  filepath: string;
+  sampleRate: number;
+  numberOfChannels: number;
+}): Promise<Uint8Array> {
+  // Use streams to process audio data incrementally
+  const response = await fetch(new URL(filepath, location.origin));
+  if (!response.body) {
+    throw new Error('Response body is null');
+  }
+
+  // Setup Opus encoder
+  const encodedChunks: EncodedAudioChunk[] = [];
+  const encoder = new AudioEncoder({
+    output: chunk => {
+      encodedChunks.push(chunk);
+    },
+    error: err => {
+      throw new Error(`Encoding error: ${err}`);
+    },
+  });
+
+  // Configure Opus encoder
+  encoder.configure({
+    codec: 'opus',
+    sampleRate: sampleRate,
+    numberOfChannels: numberOfChannels,
+    bitrate: 128000,
+  });
+
+  // Process the stream
+  const reader = response.body.getReader();
+  let offset = 0;
+  const CHUNK_SIZE = numberOfChannels * 1024; // Process 1024 samples per channel at a time
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+
+      // Convert the chunk to Float32Array
+      const float32Data = new Float32Array(value.buffer);
+
+      // Process in smaller chunks to avoid large frames
+      for (let i = 0; i < float32Data.length; i += CHUNK_SIZE) {
+        const chunkSize = Math.min(CHUNK_SIZE, float32Data.length - i);
+        const chunk = float32Data.subarray(i, i + chunkSize);
+
+        // Create and encode frame
+        const frame = new AudioData({
+          format: 'f32',
+          sampleRate: sampleRate,
+          numberOfFrames: chunk.length / numberOfChannels,
+          numberOfChannels: numberOfChannels,
+          timestamp: (offset * 1000000) / sampleRate, // timestamp in microseconds
+          data: chunk,
+        });
+
+        encoder.encode(frame);
+        frame.close();
+
+        offset += chunk.length / numberOfChannels;
+      }
+    }
+  } finally {
+    await encoder.flush();
+    encoder.close();
+  }
+
+  // Initialize WebM muxer
+  const target = new ArrayBufferTarget();
+  const muxer = new Muxer({
+    target,
+    audio: {
+      codec: 'A_OPUS',
+      sampleRate: sampleRate,
+      numberOfChannels: numberOfChannels,
+    },
+  });
+
+  // Add all chunks to the muxer
+  for (const chunk of encodedChunks) {
+    muxer.addAudioChunk(chunk, {});
+  }
+
+  // Finalize and get WebM container
+  muxer.finalize();
+  const { buffer: webmBuffer } = target;
+
+  return new Uint8Array(webmBuffer);
+}
@@ -0,0 +1,183 @@
+import { Button } from '@affine/component';
+import { useAsyncCallback } from '@affine/core/components/hooks/affine-async-hooks';
+import { appIconMap } from '@affine/core/utils';
+import { apis, events } from '@affine/electron-api';
+import { useI18n } from '@affine/i18n';
+import { useEffect, useMemo, useState } from 'react';
+
+import { encodeRawBufferToOpus } from './encode';
+import * as styles from './styles.css';
+
+type Status = {
+  id: number;
+  status: 'new' | 'recording' | 'paused' | 'stopped' | 'ready';
+  appName?: string;
+  appGroupId?: number;
+  icon?: Buffer;
+};
+
+export const useRecordingStatus = () => {
+  const [status, setStatus] = useState<Status | null>(null);
+
+  useEffect(() => {
+    // Get initial status
+    apis?.recording
+      .getCurrentRecording()
+      .then(status => setStatus(status as Status))
+      .catch(console.error);
+
+    // Subscribe to status changes
+    const unsubscribe = events?.recording.onRecordingStatusChanged(status =>
+      setStatus(status as Status)
+    );
+
+    return () => {
+      unsubscribe?.();
+    };
+  }, []);
+
+  return status;
+};
+
+const appIcon = appIconMap[BUILD_CONFIG.appBuildType];
+
+export function Recording() {
+  const status = useRecordingStatus();
+
+  const t = useI18n();
+  const textElement = useMemo(() => {
+    if (!status) {
+      return null;
+    }
+    if (status.status === 'new') {
+      return t['com.affine.recording.new']();
+    } else if (status.status === 'ready') {
+      return t['com.affine.recording.ready']();
+    } else if (status.appName) {
+      return t['com.affine.recording.recording']({
+        appName: status.appName,
+      });
+    } else {
+      return t['com.affine.recording.recording.unnamed']();
+    }
+  }, [status, t]);
+
+  const handleDismiss = useAsyncCallback(async () => {
+    await apis?.popup?.dismissCurrentRecording();
+  }, []);
+
+  const handleStopRecording = useAsyncCallback(async () => {
+    if (!status) {
+      return;
+    }
+    await apis?.recording?.stopRecording(status.id);
+  }, [status]);
+
+  const handleProcessStoppedRecording = useAsyncCallback(async () => {
+    let id: number | undefined;
+    try {
+      const result = await apis?.recording?.getCurrentRecording();
+      if (!result) {
+        return;
+      }
+
+      id = result.id;
+
+      const { filepath, sampleRate, numberOfChannels } = result;
+      if (!filepath || !sampleRate || !numberOfChannels) {
+        return;
+      }
+      const [buffer] = await Promise.all([
+        encodeRawBufferToOpus({
+          filepath,
+          sampleRate,
+          numberOfChannels,
+        }),
+        new Promise<void>(resolve => {
+          setTimeout(() => {
+            resolve();
+          }, 1000); // wait at least 1 second for better user experience
+        }),
+      ]);
+      await apis?.recording.readyRecording(result.id, buffer);
+    } catch (error) {
+      console.error('Failed to stop recording', error);
+      await apis?.popup?.dismissCurrentRecording();
+      if (id) {
+        await apis?.recording.removeRecording(id);
+      }
+    }
+  }, []);
+
+  useEffect(() => {
+    // allow processing stopped event in tray menu as well:
+    return events?.recording.onRecordingStatusChanged(status => {
+      if (status?.status === 'stopped') {
+        handleProcessStoppedRecording();
+      }
+    });
+  }, [handleProcessStoppedRecording]);
+
+  const handleStartRecording = useAsyncCallback(async () => {
+    if (!status) {
+      return;
+    }
+    await apis?.recording?.startRecording(status.appGroupId);
+  }, [status]);
+
+  const controlsElement = useMemo(() => {
+    if (!status) {
+      return null;
+    }
+    if (status.status === 'new') {
+      return (
+        <>
+          <Button variant="plain" onClick={handleDismiss}>
+            {t['com.affine.recording.dismiss']()}
+          </Button>
+          <Button
+            onClick={handleStartRecording}
+            variant="primary"
+            prefix={<div className={styles.recordingIcon} />}
+          >
+            {t['com.affine.recording.start']()}
+          </Button>
+        </>
+      );
+    } else if (status.status === 'recording') {
+      return (
+        <Button variant="error" onClick={handleStopRecording}>
+          {t['com.affine.recording.stop']()}
+        </Button>
+      );
+    } else if (status.status === 'stopped') {
+      return (
+        <Button
+          variant="error"
+          onClick={handleDismiss}
+          loading={true}
+          disabled
+        />
+      );
+    } else if (status.status === 'ready') {
+      return (
+        <Button variant="primary" onClick={handleDismiss}>
+          {t['com.affine.recording.ready']()}
+        </Button>
+      );
+    }
+    return null;
+  }, [handleDismiss, handleStartRecording, handleStopRecording, status, t]);
+
+  if (!status) {
+    return null;
+  }
+
+  return (
+    <div className={styles.root}>
+      <img className={styles.affineIcon} src={appIcon} alt="AFFiNE" />
+      <div className={styles.text}>{textElement}</div>
+      <div className={styles.controls}>{controlsElement}</div>
+    </div>
+  );
+}
@@ -0,0 +1,38 @@
+import { cssVar } from '@toeverything/theme';
+import { cssVarV2 } from '@toeverything/theme/v2';
+import { style } from '@vanilla-extract/css';
+
+export const root = style({
+  width: '100%',
+  height: '100%',
+  padding: 4,
+  display: 'flex',
+  gap: 4,
+  alignItems: 'center',
+});
+
+export const affineIcon = style({
+  width: 28,
+  height: 28,
+});
+
+export const recordingIcon = style({
+  width: 8,
+  height: 8,
+  borderRadius: '50%',
+  backgroundColor: cssVarV2('layer/pureWhite'),
+});
+
+export const text = style({
+  fontSize: cssVar('fontSm'),
+  overflow: 'hidden',
+  textOverflow: 'ellipsis',
+  whiteSpace: 'nowrap',
+  fontWeight: 600,
+  flex: 1,
+});
+
+export const controls = style({
+  display: 'flex',
+  gap: 2,
+});
@@ -0,0 +1,2 @@
+import '@affine/core/bootstrap/electron';
+import '@affine/component/theme';
@@ -0,0 +1,18 @@
+import { useEffect, useState } from 'react';
+
+export const useBlobUrl = (buffer?: Buffer) => {
+  const [blobUrl, setBlobUrl] = useState<string | null>(null);
+
+  useEffect(() => {
+    if (!buffer) {
+      return;
+    }
+    const url = URL.createObjectURL(new Blob([buffer]));
+    setBlobUrl(url);
+    return () => {
+      URL.revokeObjectURL(url);
+    };
+  }, [buffer]);
+
+  return blobUrl;
+};