feat(electron): audio capture permissions and settings (#11185)

fix AF-2420, AF-2391, AF-2265

packages/frontend/core/src/modules/dialogs/constant.ts

@@ -13,6 +13,7 @@ export type SettingTab =
   | 'experimental-features'
   | 'editor'
   | 'account'
+  | 'meetings'
   | `workspace:${'preference' | 'properties' | 'members' | 'storage' | 'billing' | 'license' | 'integrations'}`;
 
 export type GLOBAL_DIALOG_SCHEMA = {

packages/frontend/core/src/modules/feature-flag/constant.ts

@@ -225,15 +225,6 @@ export const AFFINE_FLAGS = {
     configurable: !isMobile,
     defaultState: false,
   },
-  enable_audio_block: {
-    category: 'affine',
-    displayName:
-      'com.affine.settings.workspace.experimental-features.enable-audio-block.name',
-    description:
-      'com.affine.settings.workspace.experimental-features.enable-audio-block.description',
-    configurable: !isMobile,
-    defaultState: false,
-  },
   enable_editor_rtl: {
     category: 'affine',
     displayName:
@@ -274,6 +265,24 @@ export const AFFINE_FLAGS = {
     configurable: isCanaryBuild,
     defaultState: false,
   },
+  enable_audio_block: {
+    category: 'affine',
+    displayName:
+      'com.affine.settings.workspace.experimental-features.enable-audio-block.name',
+    description:
+      'com.affine.settings.workspace.experimental-features.enable-audio-block.description',
+    configurable: !isMobile,
+    defaultState: false,
+  },
+  enable_meetings: {
+    category: 'affine',
+    displayName:
+      'com.affine.settings.workspace.experimental-features.enable-meetings.name',
+    description:
+      'com.affine.settings.workspace.experimental-features.enable-meetings.description',
+    configurable: !isMobile && environment.isMacOs,
+    defaultState: false,
+  },
 } satisfies { [key in string]: FlagInfo };
 
 // oxlint-disable-next-line no-redeclare

packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts

@@ -1,3 +1,4 @@
+import { encodeAudioBlobToOpus } from '@affine/core/utils/webm-encoding';
 import { DebugLogger } from '@affine/debug';
 import { AiJobStatus } from '@affine/graphql';
 import {
@@ -118,7 +119,8 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {
         if (!buffer) {
           throw new Error('No audio buffer available');
         }
-        const blob = new Blob([buffer], { type: this.props.props.type });
+        const encodedBuffer = await encodeAudioBlobToOpus(buffer, 64000);
+        const blob = new Blob([encodedBuffer], { type: this.props.props.type });
         const file = new File([blob], this.props.props.name, {
           type: this.props.props.type,
         });

packages/frontend/core/src/modules/media/index.ts

@@ -1,8 +1,7 @@
 import type { Framework } from '@toeverything/infra';
 
 import { DefaultServerService, WorkspaceServerService } from '../cloud';
-import { DesktopApiService } from '../desktop-api';
-import { GlobalState } from '../storage';
+import { GlobalState, GlobalStateService } from '../storage';
 import { WorkbenchService } from '../workbench';
 import { WorkspaceScope, WorkspaceService } from '../workspace';
 import { AudioAttachmentBlock } from './entities/audio-attachment-block';
@@ -16,9 +15,11 @@ import {
 } from './providers/global-audio-state';
 import { AudioAttachmentService } from './services/audio-attachment';
 import { AudioMediaManagerService } from './services/audio-media-manager';
+import { MeetingSettingsService } from './services/meeting-settings';
 
 export function configureMediaModule(framework: Framework) {
   framework
+    .service(MeetingSettingsService, [GlobalStateService])
     .scope(WorkspaceScope)
     .entity(AudioMedia, [WorkspaceService])
     .entity(AudioAttachmentBlock, [AudioMediaManagerService, WorkspaceService])
@@ -31,27 +32,18 @@ export function configureMediaModule(framework: Framework) {
       WorkspaceServerService,
       DefaultServerService,
     ])
-    .service(AudioAttachmentService);
+    .service(AudioAttachmentService)
+    .service(AudioMediaManagerService, [
+      GlobalMediaStateProvider,
+      WorkbenchService,
+    ]);
 
   if (BUILD_CONFIG.isElectron) {
-    framework
-      .impl(GlobalMediaStateProvider, ElectronGlobalMediaStateProvider, [
-        GlobalState,
-      ])
-      .scope(WorkspaceScope)
-      .service(AudioMediaManagerService, [
-        GlobalMediaStateProvider,
-        WorkbenchService,
-        DesktopApiService,
-      ]);
+    framework.impl(GlobalMediaStateProvider, ElectronGlobalMediaStateProvider, [
+      GlobalState,
+    ]);
   } else {
-    framework
-      .impl(GlobalMediaStateProvider, WebGlobalMediaStateProvider)
-      .scope(WorkspaceScope)
-      .service(AudioMediaManagerService, [
-        GlobalMediaStateProvider,
-        WorkbenchService,
-      ]);
+    framework.impl(GlobalMediaStateProvider, WebGlobalMediaStateProvider);
   }
 }
 

packages/frontend/core/src/modules/media/services/audio-media-manager.ts

@@ -13,7 +13,7 @@ import {
 import { clamp } from 'lodash-es';
 import { distinctUntilChanged } from 'rxjs';
 
-import type { DesktopApiService } from '../../desktop-api';
+import { DesktopApiService } from '../../desktop-api';
 import type { WorkbenchService } from '../../workbench';
 import { AudioMedia } from '../entities/audio-media';
 import type { BaseGlobalMediaStateProvider } from '../providers/global-audio-state';
@@ -36,18 +36,13 @@ export class AudioMediaManagerService extends Service {
   });
 
   private readonly mediaDisposables = new WeakMap<AudioMedia, (() => void)[]>();
+  private readonly desktopApi = this.framework.getOptional(DesktopApiService);
 
   constructor(
     private readonly globalMediaState: BaseGlobalMediaStateProvider,
-    private readonly workbench: WorkbenchService,
-    private readonly desktopApi?: DesktopApiService
+    private readonly workbench: WorkbenchService
   ) {
     super();
-
-    if (!BUILD_CONFIG.isElectron) {
-      this.desktopApi = undefined;
-    }
-
     this.disposables.push(() => {
       this.mediaPool.clear();
     });

packages/frontend/core/src/modules/media/services/meeting-settings.ts

@@ -0,0 +1,133 @@
+import type {
+  MeetingSettingsKey,
+  MeetingSettingsSchema,
+} from '@affine/electron/main/shared-state-schema';
+import { LiveData, Service } from '@toeverything/infra';
+import { defaults } from 'lodash-es';
+
+import { DesktopApiService } from '../../desktop-api';
+import type { GlobalStateService } from '../../storage';
+
+const MEETING_SETTINGS_KEY: typeof MeetingSettingsKey = 'meetingSettings';
+
+const defaultMeetingSettings: MeetingSettingsSchema = {
+  enabled: false,
+  recordingSavingMode: 'new-doc',
+  autoTranscription: true,
+  recordingMode: 'prompt',
+};
+
+export class MeetingSettingsService extends Service {
+  constructor(private readonly globalStateService: GlobalStateService) {
+    super();
+  }
+
+  private readonly desktopApiService =
+    this.framework.getOptional(DesktopApiService);
+
+  readonly settings$ = LiveData.computed(get => {
+    const value = get(
+      LiveData.from(
+        this.globalStateService.globalState.watch<MeetingSettingsSchema>(
+          MEETING_SETTINGS_KEY
+        ),
+        undefined
+      )
+    );
+    return defaults(value, defaultMeetingSettings);
+  });
+
+  get settings() {
+    return this.settings$.value;
+  }
+
+  // we do not want the caller to directly set the settings,
+  // there could be some side effects when the settings are changed.
+  async setEnabled(enabled: boolean) {
+    const currentEnabled = this.settings.enabled;
+    if (currentEnabled === enabled) {
+      return;
+    }
+
+    if (!(await this.isRecordingFeatureAvailable())) {
+      return;
+    }
+
+    // when the user enable the recording feature the first time,
+    // the app may prompt the user to allow the recording feature by MacOS.
+    // when the user allows the recording feature, the app may be required to restart.
+    if (enabled) {
+      // if the user already enabled the recording feature, we need to disable it
+      const successful =
+        await this.desktopApiService?.handler.recording.setupRecordingFeature();
+      if (!successful) {
+        throw new Error('Failed to setup recording feature');
+      }
+    } else {
+      // check if there is any ongoing recording
+      const ongoingRecording =
+        await this.desktopApiService?.handler.recording.getCurrentRecording();
+      if (
+        ongoingRecording &&
+        ongoingRecording.status !== 'new' &&
+        ongoingRecording.status !== 'ready'
+      ) {
+        throw new Error('There is an ongoing recording, please stop it first');
+      }
+      // if the user disabled the recording feature, we need to setup the recording feature
+      await this.desktopApiService?.handler.recording.disableRecordingFeature();
+    }
+
+    // Only update the state after successful feature setup/disable
+    this.globalStateService.globalState.set(MEETING_SETTINGS_KEY, {
+      ...this.settings$.value,
+      enabled,
+    });
+  }
+
+  setRecordingSavingMode(mode: MeetingSettingsSchema['recordingSavingMode']) {
+    this.globalStateService.globalState.set(MEETING_SETTINGS_KEY, {
+      ...this.settings$.value,
+      recordingSavingMode: mode,
+    });
+  }
+
+  setAutoTranscription(autoTranscription: boolean) {
+    this.globalStateService.globalState.set(MEETING_SETTINGS_KEY, {
+      ...this.settings$.value,
+      autoTranscription,
+    });
+  }
+
+  // this is a desktop-only feature for MacOS version 14.2 and above
+  async isRecordingFeatureAvailable() {
+    return this.desktopApiService?.handler.recording.checkRecordingAvailable();
+  }
+
+  async checkScreenRecordingPermission() {
+    return this.desktopApiService?.handler.recording.checkScreenRecordingPermission();
+  }
+
+  // the following methods are only available on MacOS right?
+  async showScreenRecordingPermissionSetting() {
+    return this.desktopApiService?.handler.recording.showScreenRecordingPermissionSetting();
+  }
+
+  setRecordingMode = (mode: MeetingSettingsSchema['recordingMode']) => {
+    const currentMode = this.settings.recordingMode;
+
+    if (currentMode === mode) {
+      return;
+    }
+
+    this.globalStateService.globalState.set(MEETING_SETTINGS_KEY, {
+      ...this.settings,
+      recordingMode: mode,
+    });
+  };
+
+  async openSavedRecordings() {
+    // todo: open the saved recordings folder
+    await this.desktopApiService?.handler.recording.showSavedRecordings();
+  }
+}

packages/frontend/core/src/modules/media/views/use-attachment-media.ts

@@ -1,31 +0,0 @@
-import type { AttachmentBlockModel } from '@blocksuite/affine/model';
-import { useService } from '@toeverything/infra';
-import { useEffect, useState } from 'react';
-
-import type { AudioAttachmentBlock } from '../entities/audio-attachment-block';
-import { AudioAttachmentService } from '../services/audio-attachment';
-
-export const useAttachmentMediaBlock = (model: AttachmentBlockModel) => {
-  const audioAttachmentService = useService(AudioAttachmentService);
-  const [audioAttachmentBlock, setAttachmentMedia] = useState<
-    AudioAttachmentBlock | undefined
-  >(undefined);
-
-  useEffect(() => {
-    if (!model.props.sourceId) {
-      return;
-    }
-    const entity = audioAttachmentService.get(model);
-    if (!entity) {
-      return;
-    }
-    const audioAttachmentBlock = entity.obj;
-    setAttachmentMedia(audioAttachmentBlock);
-    audioAttachmentBlock.mount();
-    return () => {
-      audioAttachmentBlock.unmount();
-      entity.release();
-    };
-  }, [audioAttachmentService, model]);
-  return audioAttachmentBlock;
-};