diff --git a/packages/frontend/apps/electron-renderer/package.json b/packages/frontend/apps/electron-renderer/package.json index 9c25d72403..e0aa60b27a 100644 --- a/packages/frontend/apps/electron-renderer/package.json +++ b/packages/frontend/apps/electron-renderer/package.json @@ -13,6 +13,7 @@ "@affine/electron-api": "workspace:*", "@affine/i18n": "workspace:*", "@affine/nbstore": "workspace:*", + "@blocksuite/affine": "workspace:*", "@emotion/react": "^11.14.0", "@sentry/react": "^9.2.0", "@toeverything/infra": "workspace:*", diff --git a/packages/frontend/apps/electron-renderer/src/app.tsx b/packages/frontend/apps/electron-renderer/src/app.tsx index 234d75d221..88079245ae 100644 --- a/packages/frontend/apps/electron-renderer/src/app.tsx +++ b/packages/frontend/apps/electron-renderer/src/app.tsx @@ -1,3 +1,4 @@ +import type { DocProps } from '@affine/core/blocksuite/initialization'; import { AffineContext } from '@affine/core/components/context'; import { WindowsAppControls } from '@affine/core/components/pure/header/windows-app-controls'; import { AppContainer } from '@affine/core/desktop/components/app-container'; @@ -38,6 +39,8 @@ import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspac import createEmotionCache from '@affine/core/utils/create-emotion-cache'; import { apis, events } from '@affine/electron-api'; import { StoreManagerClient } from '@affine/nbstore/worker/client'; +import type { AttachmentBlockProps } from '@blocksuite/affine/model'; +import { Text } from '@blocksuite/affine/store'; import { CacheProvider } from '@emotion/react'; import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra'; import { OpClient } from '@toeverything/infra/op'; @@ -174,32 +177,80 @@ events?.applicationMenu.openAboutPageInSettingModal(() => { }); events?.applicationMenu.onNewPageAction(type => { - const currentWorkspace = getCurrentWorkspace(); - if (!currentWorkspace) { - return; - } - const { workspace, dispose } = currentWorkspace; - const editorSettingService = frameworkProvider.get(EditorSettingService); - const docsService = workspace.scope.get(DocsService); - const editorSetting = editorSettingService.editorSetting; - - const docProps = { - note: editorSetting.get('affine:note'), - }; apis?.ui .isActiveTab() .then(isActive => { if (!isActive) { return; } + const currentWorkspace = getCurrentWorkspace(); + if (!currentWorkspace) { + return; + } + const { workspace, dispose } = currentWorkspace; + const editorSettingService = frameworkProvider.get(EditorSettingService); + const docsService = workspace.scope.get(DocsService); + const editorSetting = editorSettingService.editorSetting; + + const docProps = { + note: editorSetting.get('affine:note'), + }; const page = docsService.createDoc({ docProps, primaryMode: type }); workspace.scope.get(WorkbenchService).workbench.openDoc(page.id); + dispose(); }) .catch(err => { console.error(err); }); +}); - dispose(); +events?.recording.onRecordingStatusChanged(status => { + (async () => { + if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') { + const currentWorkspace = getCurrentWorkspace(); + if (!currentWorkspace) { + return; + } + const { workspace, dispose } = currentWorkspace; + const editorSettingService = frameworkProvider.get(EditorSettingService); + const docsService = workspace.scope.get(DocsService); + const editorSetting = editorSettingService.editorSetting; + + const docProps: DocProps = { + note: editorSetting.get('affine:note'), + page: { + title: new Text( + 'Recording ' + + (status.appGroup?.name ?? 'System Audio') + + ' ' + + new Date(status.startTime).toISOString() + ), + }, + onStoreLoad: (doc, { noteId }) => { + (async () => { + const data = await apis?.recording.saveRecording(status.id); + if (!data) { + return; + } + const blob = new Blob([data], { type: 'audio/mp3' }); + const blobId = await doc.workspace.blobSync.set(blob); + const attachmentProps: Partial = { + name: 'Recording', + size: blob.size, + type: 'audio/mp3', + sourceId: blobId, + embed: true, + }; + doc.addBlock('affine:attachment', attachmentProps, noteId); + })().catch(console.error); + }, + }; + const page = docsService.createDoc({ docProps, primaryMode: 'page' }); + workspace.scope.get(WorkbenchService).workbench.openDoc(page.id); + + dispose(); + } + })().catch(console.error); }); events?.applicationMenu.onOpenJournal(() => { diff --git a/packages/frontend/apps/electron-renderer/tsconfig.json b/packages/frontend/apps/electron-renderer/tsconfig.json index 19af95cc4f..422502d628 100644 --- a/packages/frontend/apps/electron-renderer/tsconfig.json +++ b/packages/frontend/apps/electron-renderer/tsconfig.json @@ -13,6 +13,7 @@ { "path": "../../electron-api" }, { "path": "../../i18n" }, { "path": "../../../common/nbstore" }, + { "path": "../../../../blocksuite/affine/all" }, { "path": "../../../common/infra" }, { "path": "../../../../tools/utils" } ] diff --git a/packages/frontend/apps/electron/resources/icons/monitor.png b/packages/frontend/apps/electron/resources/icons/monitor.png new file mode 100644 index 0000000000..e4305b8093 Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/monitor.png differ diff --git a/packages/frontend/apps/electron/resources/icons/pause.png b/packages/frontend/apps/electron/resources/icons/pause.png new file mode 100644 index 0000000000..f4ff7ee83d Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/pause.png differ diff --git a/packages/frontend/apps/electron/resources/icons/stop.png b/packages/frontend/apps/electron/resources/icons/stop.png new file mode 100644 index 0000000000..32977c4f2b Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/stop.png differ diff --git a/packages/frontend/apps/electron/resources/icons/waveform-recording.png b/packages/frontend/apps/electron/resources/icons/waveform-recording.png new file mode 100644 index 0000000000..19e91f03f9 Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/waveform-recording.png differ diff --git a/packages/frontend/apps/electron/resources/icons/waveform.png b/packages/frontend/apps/electron/resources/icons/waveform.png new file mode 100644 index 0000000000..11a4f05d38 Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/waveform.png differ diff --git a/packages/frontend/apps/electron/src/helper/main-rpc.ts b/packages/frontend/apps/electron/src/helper/main-rpc.ts index 59be05958f..e6026f5d30 100644 --- a/packages/frontend/apps/electron/src/helper/main-rpc.ts +++ b/packages/frontend/apps/electron/src/helper/main-rpc.ts @@ -2,6 +2,7 @@ import { AsyncCall } from 'async-call-rpc'; import type { HelperToMain, MainToHelper } from '../shared/type'; import { exposed } from './provide'; +import { encodeToMp3 } from './recording/encode'; const helperToMainServer: HelperToMain = { getMeta: () => { @@ -10,6 +11,8 @@ const helperToMainServer: HelperToMain = { } return exposed; }, + // allow main process encode audio samples to mp3 buffer (because it is slow and blocking) + encodeToMp3, }; export const mainRPC = AsyncCall(helperToMainServer, { diff --git a/packages/frontend/apps/electron/src/helper/recording/encode.ts b/packages/frontend/apps/electron/src/helper/recording/encode.ts new file mode 100644 index 0000000000..0aa3464876 --- /dev/null +++ b/packages/frontend/apps/electron/src/helper/recording/encode.ts @@ -0,0 +1,16 @@ +import { Mp3Encoder } from '@affine/native'; + +// encode audio samples to mp3 buffer +export function encodeToMp3( + samples: Float32Array, + opts: { + channels?: number; + sampleRate?: number; + } = {} +): Uint8Array { + const mp3Encoder = new Mp3Encoder({ + channels: opts.channels ?? 2, + sampleRate: opts.sampleRate ?? 44100, + }); + return mp3Encoder.encode(samples); +} diff --git a/packages/frontend/apps/electron/src/main/events.ts b/packages/frontend/apps/electron/src/main/events.ts index 51b5c1b07e..fec2f1dc78 100644 --- a/packages/frontend/apps/electron/src/main/events.ts +++ b/packages/frontend/apps/electron/src/main/events.ts @@ -4,6 +4,7 @@ import { AFFINE_EVENT_CHANNEL_NAME } from '../shared/type'; import { applicationMenuEvents } from './application-menu'; import { beforeAppQuit } from './cleanup'; import { logger } from './logger'; +import { recordingEvents } from './recording'; import { sharedStorageEvents } from './shared-storage'; import { uiEvents } from './ui/events'; import { updaterEvents } from './updater/event'; @@ -13,6 +14,7 @@ export const allEvents = { updater: updaterEvents, ui: uiEvents, sharedStorage: sharedStorageEvents, + recording: recordingEvents, }; function getActiveWindows() { diff --git a/packages/frontend/apps/electron/src/main/handlers.ts b/packages/frontend/apps/electron/src/main/handlers.ts index 59ef7a0d55..70d0fcf40a 100644 --- a/packages/frontend/apps/electron/src/main/handlers.ts +++ b/packages/frontend/apps/electron/src/main/handlers.ts @@ -5,6 +5,7 @@ import { clipboardHandlers } from './clipboard'; import { configStorageHandlers } from './config-storage'; import { findInPageHandlers } from './find-in-page'; import { getLogFilePath, logger, revealLogFile } from './logger'; +import { recordingHandlers } from './recording'; import { sharedStorageHandlers } from './shared-storage'; import { uiHandlers } from './ui/handlers'; import { updaterHandlers } from './updater'; @@ -29,6 +30,7 @@ export const allHandlers = { findInPage: findInPageHandlers, sharedStorage: sharedStorageHandlers, worker: workerHandlers, + recording: recordingHandlers, }; export const registerHandlers = () => { diff --git a/packages/frontend/apps/electron/src/main/index.ts b/packages/frontend/apps/electron/src/main/index.ts index e1f1d71ebb..f67e9dc9d7 100644 --- a/packages/frontend/apps/electron/src/main/index.ts +++ b/packages/frontend/apps/electron/src/main/index.ts @@ -14,6 +14,8 @@ import { registerEvents } from './events'; import { registerHandlers } from './handlers'; import { logger } from './logger'; import { registerProtocol } from './protocol'; +import { setupRecording } from './recording'; +import { getTrayState } from './tray'; import { registerUpdater } from './updater'; import { launch } from './windows-manager/launcher'; import { launchStage } from './windows-manager/stage'; @@ -85,7 +87,9 @@ app .then(registerHandlers) .then(registerEvents) .then(launch) + .then(setupRecording) .then(createApplicationMenu) + .then(getTrayState) .then(registerUpdater) .catch(e => console.error('Failed create window:', e)); diff --git a/packages/frontend/apps/electron/src/main/recording/index.ts b/packages/frontend/apps/electron/src/main/recording/index.ts index abcc9bd24a..ed9dde168a 100644 --- a/packages/frontend/apps/electron/src/main/recording/index.ts +++ b/packages/frontend/apps/electron/src/main/recording/index.ts @@ -1,38 +1,30 @@ -import { ShareableContent, TappableApplication } from '@affine/native'; -import { Notification } from 'electron'; -import { - BehaviorSubject, - distinctUntilChanged, - pairwise, - startWith, -} from 'rxjs'; +import { ShareableContent } from '@affine/native'; +import { nativeImage, Notification } from 'electron'; +import { debounce } from 'lodash-es'; +import { BehaviorSubject, distinctUntilChanged, groupBy, mergeMap } from 'rxjs'; import { isMacOS } from '../../shared/utils'; import { beforeAppQuit } from '../cleanup'; +import { ensureHelperProcess } from '../helper-process'; import { logger } from '../logger'; - -interface TappableAppInfo { - rawInstance: TappableApplication; - isRunning: boolean; - processId: number; - processGroupId: number; - bundleIdentifier: string; - name: string; -} - -interface AppGroupInfo { - processGroupId: number; - apps: TappableAppInfo[]; - name: string; - icon: Buffer | undefined; - isRunning: boolean; -} +import type { NamespaceHandlers } from '../type'; +import { getMainWindow } from '../windows-manager'; +import type { + AppGroupInfo, + Recording, + RecordingStatus, + TappableAppInfo, +} from './types'; const subscribers: Subscriber[] = []; beforeAppQuit(() => { subscribers.forEach(subscriber => { - subscriber.unsubscribe(); + try { + subscriber.unsubscribe(); + } catch { + // ignore unsubscribe error + } }); }); @@ -41,8 +33,41 @@ let shareableContent: ShareableContent | null = null; export const applications$ = new BehaviorSubject([]); export const appGroups$ = new BehaviorSubject([]); -if (isMacOS()) { - // Update appGroups$ whenever applications$ changes +// recording id -> recording +// recordings will be saved in memory before consumed and created as an audio block to user's doc +const recordings = new Map(); + +// there should be only one active recording at a time +export const recordingStatus$ = new BehaviorSubject( + null +); + +function createAppGroup(processGroupId: number): AppGroupInfo | undefined { + const groupProcess = + shareableContent?.applicationWithProcessId(processGroupId); + if (!groupProcess) { + return; + } + return { + processGroupId: processGroupId, + apps: [], // leave it empty for now. + name: groupProcess.name, + bundleIdentifier: groupProcess.bundleIdentifier, + // icon should be lazy loaded + get icon() { + try { + return groupProcess.icon; + } catch (error) { + logger.error(`Failed to get icon for ${groupProcess.name}`, error); + return undefined; + } + }, + isRunning: false, + }; +} + +// pipe applications$ to appGroups$ +function setupAppGroups() { subscribers.push( applications$.pipe(distinctUntilChanged()).subscribe(apps => { const appGroups: AppGroupInfo[] = []; @@ -52,69 +77,160 @@ if (isMacOS()) { ); if (!appGroup) { - const groupProcess = shareableContent?.applicationWithProcessId( - app.processGroupId - ); - if (!groupProcess) { - return; + appGroup = createAppGroup(app.processGroupId); + if (appGroup) { + appGroups.push(appGroup); } - appGroup = { - processGroupId: app.processGroupId, - apps: [], - name: groupProcess.name, - // icon will be lazy loaded - get icon() { - try { - return groupProcess.icon; - } catch (error) { - logger.error( - `Failed to get icon for ${groupProcess.name}`, - error - ); - return undefined; - } - }, - get isRunning() { - return this.apps.some(app => app.rawInstance.isRunning); - }, - }; - appGroups.push(appGroup); } if (appGroup) { appGroup.apps.push(app); } }); + + appGroups.forEach(appGroup => { + appGroup.isRunning = appGroup.apps.some(app => app.isRunning); + }); + appGroups$.next(appGroups); }) ); +} +function setupNewRunningAppGroup() { + const appGroupRunningChanged$ = appGroups$.pipe( + mergeMap(groups => groups), + groupBy(group => group.processGroupId), + mergeMap(groupStream$ => + groupStream$.pipe( + distinctUntilChanged((prev, curr) => prev.isRunning === curr.isRunning) + ) + ) + ); subscribers.push( - appGroups$ - .pipe(startWith([] as AppGroupInfo[]), pairwise()) - .subscribe(([previousGroups, currentGroups]) => { - currentGroups.forEach(currentGroup => { - const previousGroup = previousGroups.find( - group => group.processGroupId === currentGroup.processGroupId - ); - if (previousGroup?.isRunning !== currentGroup.isRunning) { - console.log( - 'appgroup running changed', - currentGroup.name, - currentGroup.isRunning - ); - if (currentGroup.isRunning) { - new Notification({ - title: 'Recording Meeting', - body: `Recording meeting with ${currentGroup.name}`, - }).show(); - } - } + appGroupRunningChanged$.subscribe(currentGroup => { + if (currentGroup.isRunning) { + // TODO(@pengx17): stub impl. will be replaced with a real one later + const notification = new Notification({ + icon: currentGroup.icon + ? nativeImage.createFromBuffer(currentGroup.icon) + : undefined, + title: 'Recording Meeting', + body: `Recording meeting with ${currentGroup.name}`, + actions: [ + { + type: 'button', + text: 'Start', + }, + ], }); - }) + notification.on('action', () => { + startRecording(currentGroup); + }); + notification.show(); + } else { + // if the group is not running, we should stop the recording (if it is recording) + if ( + recordingStatus$.value?.status === 'recording' && + recordingStatus$.value?.appGroup?.processGroupId === + currentGroup.processGroupId + ) { + stopRecording(); + } + } + }) ); } -async function getAllApps(): Promise { +function createRecording(status: RecordingStatus) { + const buffers: Float32Array[] = []; + + function tapAudioSamples(err: Error | null, samples: Float32Array) { + const recordingStatus = recordingStatus$.getValue(); + if ( + !recordingStatus || + recordingStatus.id !== status.id || + recordingStatus.status === 'paused' + ) { + return; + } + + if (err) { + logger.error('failed to get audio samples', err); + } else { + buffers.push(new Float32Array(samples)); + } + } + + const stream = status.app + ? status.app.rawInstance.tapAudio(tapAudioSamples) + : ShareableContent.tapGlobalAudio(null, tapAudioSamples); + + const recording: Recording = { + id: status.id, + startTime: status.startTime, + app: status.app, + appGroup: status.appGroup, + buffers, + stream, + }; + + return recording; +} + +function concatBuffers(buffers: Float32Array[]): Float32Array { + const totalSamples = buffers.reduce((acc, buf) => acc + buf.length, 0); + const buffer = new Float32Array(totalSamples); + let offset = 0; + buffers.forEach(buf => { + buffer.set(buf, offset); + offset += buf.length; + }); + return buffer; +} + +export async function saveRecording(id: number) { + const recording = recordings.get(id); + if (!recording) { + logger.error(`Recording ${id} not found`); + return; + } + const { buffers } = recording; + const helperProcessManager = await ensureHelperProcess(); + const buffer = concatBuffers(buffers); + const mp3Buffer = await helperProcessManager.rpc?.encodeToMp3(buffer, { + channels: recording.stream.channels, + sampleRate: recording.stream.sampleRate, + }); + + if (!mp3Buffer) { + logger.error('failed to encode audio samples to mp3'); + return; + } + recordings.delete(recording.id); + return mp3Buffer; +} + +function setupRecordingListeners() { + subscribers.push( + recordingStatus$.pipe(distinctUntilChanged()).subscribe(status => { + if (status?.status === 'recording') { + let recording = recordings.get(status.id); + // create a recording if not exists + if (!recording) { + recording = createRecording(status); + recordings.set(status.id, recording); + } + } else if (status?.status === 'stopped') { + const recording = recordings.get(status.id); + if (recording) { + recording.stream.stop(); + } + } + }) + ); +} + +function getAllApps(): TappableAppInfo[] { if (!shareableContent) { return []; } @@ -150,62 +266,37 @@ type Subscriber = { }; function setupMediaListeners() { + applications$.next(getAllApps()); subscribers.push( ShareableContent.onApplicationListChanged(() => { - getAllApps() - .then(apps => { - applications$.next(apps); - }) - .catch(err => { - logger.error('failed to get apps', err); - }); + applications$.next(getAllApps()); }) ); - getAllApps() - .then(apps => { - applications$.next(apps); - }) - .catch(err => { - logger.error('failed to get apps', err); - }); - let appStateSubscribers: Subscriber[] = []; subscribers.push( applications$.subscribe(apps => { appStateSubscribers.forEach(subscriber => { - subscriber.unsubscribe(); + try { + subscriber.unsubscribe(); + } catch { + // ignore unsubscribe error + } }); const _appStateSubscribers: Subscriber[] = []; apps.forEach(app => { try { - // Try to create a TappableApplication with a default audio object ID - // In a real implementation, you would need to get the actual audio object ID - // This is just a placeholder value that seems to work for testing - const tappableApp = TappableApplication.fromApplication( - app.rawInstance, - 1 + const tappableApp = app.rawInstance; + const debouncedAppStateChanged = debounce(() => { + applications$.next(getAllApps()); + }, 100); + _appStateSubscribers.push( + ShareableContent.onAppStateChanged(tappableApp, () => { + debouncedAppStateChanged(); + }) ); - - if (tappableApp) { - _appStateSubscribers.push( - ShareableContent.onAppStateChanged(tappableApp, () => { - setTimeout(() => { - const apps = applications$.getValue(); - applications$.next( - apps.map(_app => { - if (_app.processId === app.processId) { - return { ..._app, isRunning: tappableApp.isRunning }; - } - return _app; - }) - ); - }, 10); - }) - ); - } } catch (error) { logger.error( `Failed to convert app ${app.name} to TappableApplication`, @@ -217,15 +308,23 @@ function setupMediaListeners() { appStateSubscribers = _appStateSubscribers; return () => { _appStateSubscribers.forEach(subscriber => { - subscriber.unsubscribe(); + try { + subscriber.unsubscribe(); + } catch { + // ignore unsubscribe error + } }); }; }) ); } -export function getShareableContent() { - if (!shareableContent && isMacOS()) { +export function setupRecording() { + if (!isMacOS()) { + return; + } + + if (!shareableContent) { try { shareableContent = new ShareableContent(); setupMediaListeners(); @@ -233,5 +332,99 @@ export function getShareableContent() { logger.error('failed to get shareable content', error); } } - return shareableContent; + setupAppGroups(); + setupNewRunningAppGroup(); + setupRecordingListeners(); } + +let recordingId = 0; + +export function startRecording( + appGroup?: AppGroupInfo +): RecordingStatus | undefined { + if (!shareableContent) { + return; // likely called on unsupported platform + } + + // hmm, is it possible that there are multiple apps running (listening) in the same group? + const appInfo = appGroup?.apps.find(app => app.isRunning); + + const recordingStatus: RecordingStatus = { + id: recordingId++, + status: 'recording', + startTime: Date.now(), + app: appInfo, + appGroup, + }; + + recordingStatus$.next(recordingStatus); + + return recordingStatus; +} + +export function pauseRecording() { + const recordingStatus = recordingStatus$.value; + if (!recordingStatus) { + return; + } + + recordingStatus$.next({ + ...recordingStatus, + status: 'paused', + }); +} + +export function resumeRecording() { + const recordingStatus = recordingStatus$.value; + if (!recordingStatus) { + return; + } + + recordingStatus$.next({ + ...recordingStatus, + status: 'recording', + }); +} + +export function stopRecording() { + const recordingStatus = recordingStatus$.value; + if (!recordingStatus) { + return; + } + + // do not remove the last recordingStatus from recordingStatus$ + recordingStatus$.next({ + ...recordingStatus, + status: 'stopped', + }); + + // bring up the window + getMainWindow() + .then(mainWindow => { + if (mainWindow) { + mainWindow.show(); + } + }) + .catch(err => { + logger.error('failed to bring up the window', err); + }); +} + +export const recordingHandlers = { + saveRecording: async (_, id: number) => { + return saveRecording(id); + }, +} satisfies NamespaceHandlers; + +export const recordingEvents = { + onRecordingStatusChanged: (fn: (status: RecordingStatus | null) => void) => { + const sub = recordingStatus$.subscribe(fn); + return () => { + try { + sub.unsubscribe(); + } catch { + // ignore unsubscribe error + } + }; + }, +}; diff --git a/packages/frontend/apps/electron/src/main/recording/types.ts b/packages/frontend/apps/electron/src/main/recording/types.ts new file mode 100644 index 0000000000..d308b5b11b --- /dev/null +++ b/packages/frontend/apps/electron/src/main/recording/types.ts @@ -0,0 +1,38 @@ +import type { AudioTapStream, TappableApplication } from '@affine/native'; + +export interface TappableAppInfo { + rawInstance: TappableApplication; + isRunning: boolean; + processId: number; + processGroupId: number; + bundleIdentifier: string; + name: string; +} + +export interface AppGroupInfo { + processGroupId: number; + apps: TappableAppInfo[]; + name: string; + bundleIdentifier: string; + icon: Buffer | undefined; + isRunning: boolean; +} + +export interface Recording { + id: number; + // the app may not be available if the user choose to record system audio + app?: TappableAppInfo; + appGroup?: AppGroupInfo; + // the raw audio buffers that are already accumulated + buffers: Float32Array[]; + stream: AudioTapStream; + startTime: number; +} + +export interface RecordingStatus { + id: number; // corresponds to the recording id + status: 'recording' | 'paused' | 'stopped'; + app?: TappableAppInfo; + appGroup?: AppGroupInfo; + startTime: number; +} diff --git a/packages/frontend/apps/electron/src/main/tray/icons.ts b/packages/frontend/apps/electron/src/main/tray/icons.ts new file mode 100644 index 0000000000..26d7a962ad --- /dev/null +++ b/packages/frontend/apps/electron/src/main/tray/icons.ts @@ -0,0 +1,13 @@ +import { join } from 'node:path'; + +import { resourcesPath } from '../../shared/utils'; + +export const icons = { + record: join(resourcesPath, 'icons/waveform.png'), + recording: join(resourcesPath, 'icons/waveform-recording.png'), + tray: join(resourcesPath, 'icons/tray-icon.png'), + journal: join(resourcesPath, 'icons/journal-today.png'), + page: join(resourcesPath, 'icons/doc-page.png'), + edgeless: join(resourcesPath, 'icons/doc-edgeless.png'), + monitor: join(resourcesPath, 'icons/monitor.png'), +}; diff --git a/packages/frontend/apps/electron/src/main/tray/index.ts b/packages/frontend/apps/electron/src/main/tray/index.ts index 249c441b7b..a6d8b21d0b 100644 --- a/packages/frontend/apps/electron/src/main/tray/index.ts +++ b/packages/frontend/apps/electron/src/main/tray/index.ts @@ -1,25 +1,34 @@ -import { join } from 'node:path'; - import { app, Menu, MenuItem, + type MenuItemConstructorOptions, type NativeImage, nativeImage, Tray, } from 'electron'; -import { isMacOS, resourcesPath } from '../../shared/utils'; +import { isMacOS } from '../../shared/utils'; import { applicationMenuSubjects } from '../application-menu'; import { beforeAppQuit } from '../cleanup'; -import { appGroups$ } from '../recording'; +import { logger } from '../logger'; +import { + appGroups$, + pauseRecording, + recordingStatus$, + resumeRecording, + startRecording, + stopRecording, +} from '../recording'; import { getMainWindow } from '../windows-manager'; +import { icons } from './icons'; export interface TrayMenuConfigItem { label: string; click?: () => void; icon?: NativeImage | string | Buffer; disabled?: boolean; + submenu?: TrayMenuConfig; } export type TrayMenuConfig = Array; @@ -35,7 +44,35 @@ function showMainWindow() { .then(w => { w.show(); }) - .catch(err => console.error(err)); + .catch(err => logger.error('Failed to show main window:', err)); +} + +function buildMenuConfig(config: TrayMenuConfig): MenuItemConstructorOptions[] { + const menuConfig: MenuItemConstructorOptions[] = []; + config.forEach(item => { + if (item === 'separator') { + menuConfig.push({ type: 'separator' }); + } else { + const { icon, disabled, submenu, ...rest } = item; + let nativeIcon: NativeImage | undefined; + if (typeof icon === 'string') { + nativeIcon = nativeImage.createFromPath(icon); + } else if (Buffer.isBuffer(icon)) { + nativeIcon = nativeImage.createFromBuffer(icon); + } + if (nativeIcon) { + nativeIcon = nativeIcon.resize({ width: 20, height: 20 }); + } + const submenuConfig = submenu ? buildMenuConfig(submenu) : undefined; + menuConfig.push({ + ...rest, + enabled: !disabled, + icon: nativeIcon, + submenu: submenuConfig, + }); + } + }); + return menuConfig; } class TrayState { @@ -43,7 +80,7 @@ class TrayState { // tray's icon icon: NativeImage = nativeImage - .createFromPath(join(resourcesPath, 'icons/tray-icon.png')) + .createFromPath(icons.tray) .resize({ width: 16, height: 16 }); // tray's tooltip @@ -60,24 +97,27 @@ class TrayState { getConfig: () => [ { label: 'Open Journal', - icon: join(resourcesPath, 'icons/journal-today.png'), + icon: icons.journal, click: () => { + logger.info('User action: Open Journal'); showMainWindow(); applicationMenuSubjects.openJournal$.next(); }, }, { label: 'New Page', - icon: join(resourcesPath, 'icons/doc-page.png'), + icon: icons.page, click: () => { + logger.info('User action: New Page'); showMainWindow(); applicationMenuSubjects.newPageAction$.next('page'); }, }, { label: 'New Edgeless', - icon: join(resourcesPath, 'icons/doc-edgeless.png'), + icon: icons.edgeless, click: () => { + logger.info('User action: New Edgeless'); showMainWindow(); applicationMenuSubjects.newPageAction$.next('edgeless'); }, @@ -89,20 +129,78 @@ class TrayState { getRecordingMenuProvider(): TrayMenuProvider { const appGroups = appGroups$.value; const runningAppGroups = appGroups.filter(appGroup => appGroup.isRunning); + + const recordingStatus = recordingStatus$.value; + + if (!recordingStatus || recordingStatus?.status === 'stopped') { + return { + key: 'recording', + getConfig: () => [ + { + label: 'Start Recording Meeting', + icon: icons.record, + submenu: [ + { + label: 'System audio (all audio will be recorded)', + icon: icons.monitor, + click: () => { + logger.info( + 'User action: Start Recording Meeting (System audio)' + ); + startRecording(); + }, + }, + ...runningAppGroups.map(appGroup => ({ + label: appGroup.name, + icon: appGroup.icon || undefined, + click: () => { + logger.info( + `User action: Start Recording Meeting (${appGroup.name})` + ); + startRecording(appGroup); + }, + })), + ], + }, + ], + }; + } + + const recordingLabel = recordingStatus.appGroup?.name + ? `Recording (${recordingStatus.appGroup?.name})` + : 'Recording'; + + // recording is either started or paused return { key: 'recording', getConfig: () => [ { - label: 'Start Recording Meeting', + label: recordingLabel, + icon: icons.recording, disabled: true, }, - ...runningAppGroups.map(appGroup => ({ - label: appGroup.name, - icon: appGroup.icon || undefined, + recordingStatus.status === 'paused' + ? { + label: 'Resume', + click: () => { + logger.info('User action: Resume Recording'); + resumeRecording(); + }, + } + : { + label: 'Pause', + click: () => { + logger.info('User action: Pause Recording'); + pauseRecording(); + }, + }, + { + label: 'Stop', click: () => { - console.log(appGroup); + logger.info('User action: Stop Recording'); + stopRecording(); }, - })), + }, ], }; } @@ -114,12 +212,13 @@ class TrayState { { label: 'Open AFFiNE', click: () => { + logger.info('User action: Open AFFiNE'); getMainWindow() .then(w => { w.show(); }) .catch(err => { - console.error(err); + logger.error('Failed to open AFFiNE:', err); }); }, }, @@ -127,6 +226,7 @@ class TrayState { { label: 'Quit AFFiNE Completely...', click: () => { + logger.info('User action: Quit AFFiNE Completely'); app.quit(); }, }, @@ -137,32 +237,9 @@ class TrayState { buildMenu(providers: TrayMenuProvider[]) { const menu = new Menu(); providers.forEach((provider, index) => { - provider.getConfig().forEach(item => { - if (item === 'separator') { - menu.append(new MenuItem({ type: 'separator' })); - } else { - const { icon, disabled, ...rest } = item; - let nativeIcon: NativeImage | undefined; - if (typeof icon === 'string') { - nativeIcon = nativeImage.createFromPath(icon); - } else if (Buffer.isBuffer(icon)) { - try { - nativeIcon = nativeImage.createFromBuffer(icon); - } catch (error) { - console.error('Failed to create icon from buffer', error); - } - } - if (nativeIcon) { - nativeIcon = nativeIcon.resize({ width: 20, height: 20 }); - } - menu.append( - new MenuItem({ - ...rest, - enabled: !disabled, - icon: nativeIcon, - }) - ); - } + const config = provider.getConfig(); + buildMenuConfig(config).forEach(item => { + menu.append(new MenuItem(item)); }); if (index !== providers.length - 1) { menu.append(new MenuItem({ type: 'separator' })); @@ -176,15 +253,22 @@ class TrayState { this.tray = new Tray(this.icon); this.tray.setToolTip(this.tooltip); const clickHandler = () => { + logger.debug('User clicked on tray icon'); this.update(); if (!isMacOS()) { this.tray?.popUpContextMenu(); } }; this.tray.on('click', clickHandler); + const appGroupsSubscription = appGroups$.subscribe(() => { + logger.debug('App groups updated, refreshing tray menu'); + this.update(); + }); beforeAppQuit(() => { + logger.info('Cleaning up tray before app quit'); this.tray?.off('click', clickHandler); this.tray?.destroy(); + appGroupsSubscription.unsubscribe(); }); } @@ -199,6 +283,7 @@ class TrayState { } init() { + logger.info('Initializing tray'); this.update(); } } diff --git a/packages/frontend/apps/electron/src/shared/type.ts b/packages/frontend/apps/electron/src/shared/type.ts index 231ee3baf1..900a561035 100644 --- a/packages/frontend/apps/electron/src/shared/type.ts +++ b/packages/frontend/apps/electron/src/shared/type.ts @@ -17,6 +17,13 @@ export interface HelperToRenderer { // helper <-> main export interface HelperToMain { getMeta: () => ExposedMeta; + encodeToMp3: ( + samples: Float32Array, + opts?: { + channels?: number; + sampleRate?: number; + } + ) => Uint8Array; } export type MainToHelper = Pick< diff --git a/packages/frontend/core/src/blocksuite/initialization/index.ts b/packages/frontend/core/src/blocksuite/initialization/index.ts index 85a4a8f969..6e79c3b753 100644 --- a/packages/frontend/core/src/blocksuite/initialization/index.ts +++ b/packages/frontend/core/src/blocksuite/initialization/index.ts @@ -12,6 +12,14 @@ export interface DocProps { surface?: Partial; note?: Partial; paragraph?: Partial; + onStoreLoad?: ( + doc: Store, + props: { + noteId: string; + paragraphId: string; + surfaceId: string; + } + ) => void; } export function initDocFromProps(doc: Store, props?: DocProps) { @@ -20,7 +28,11 @@ export function initDocFromProps(doc: Store, props?: DocProps) { 'affine:page', props?.page || { title: new Text('') } ); - doc.addBlock('affine:surface' as never, props?.surface || {}, pageBlockId); + const surfaceId = doc.addBlock( + 'affine:surface' as never, + props?.surface || {}, + pageBlockId + ); const noteBlockId = doc.addBlock( 'affine:note', { @@ -29,7 +41,16 @@ export function initDocFromProps(doc: Store, props?: DocProps) { }, pageBlockId ); - doc.addBlock('affine:paragraph', props?.paragraph || {}, noteBlockId); + const paragraphBlockId = doc.addBlock( + 'affine:paragraph', + props?.paragraph || {}, + noteBlockId + ); + props?.onStoreLoad?.(doc, { + noteId: noteBlockId, + paragraphId: paragraphBlockId, + surfaceId, + }); doc.history.clear(); }); } diff --git a/packages/frontend/media-capture-playground/server/main.ts b/packages/frontend/media-capture-playground/server/main.ts index ebffc0b208..a22d6b8494 100644 --- a/packages/frontend/media-capture-playground/server/main.ts +++ b/packages/frontend/media-capture-playground/server/main.ts @@ -28,12 +28,13 @@ console.log(`📁 Ensuring recordings directory exists at ${RECORDING_DIR}`); // Types interface Recording { - app: TappableApplication; + app: TappableApplication | null; appGroup: Application | null; buffers: Float32Array[]; stream: AudioTapStream; startTime: number; isWriting: boolean; + isGlobal?: boolean; } interface RecordingStatus { @@ -54,6 +55,7 @@ interface RecordingMetadata { sampleRate: number; channels: number; totalSamples: number; + isGlobal?: boolean; } interface AppInfo { @@ -118,7 +120,7 @@ app.use( async function saveRecording(recording: Recording): Promise { try { recording.isWriting = true; - const app = recording.appGroup || recording.app; + const app = recording.isGlobal ? null : recording.appGroup || recording.app; const totalSamples = recording.buffers.reduce( (acc, buf) => acc + buf.length, @@ -133,9 +135,19 @@ async function saveRecording(recording: Recording): Promise { const channelCount = recording.stream.channels; const expectedSamples = recordingDuration * actualSampleRate; - console.log(`💾 Saving recording for ${app.name}:`); - console.log(`- Process ID: ${app.processId}`); - console.log(`- Bundle ID: ${app.bundleIdentifier}`); + if (recording.isGlobal) { + console.log('💾 Saving global recording:'); + } else { + const appName = app?.name ?? 'Unknown App'; + const processId = app?.processId ?? 0; + const bundleId = app?.bundleIdentifier ?? 'unknown'; + console.log(`💾 Saving recording for ${appName}:`); + if (app) { + console.log(`- Process ID: ${processId}`); + console.log(`- Bundle ID: ${bundleId}`); + } + } + console.log(`- Actual duration: ${recordingDuration.toFixed(2)}s`); console.log(`- Sample rate: ${actualSampleRate}Hz`); console.log(`- Channels: ${channelCount}`); @@ -156,7 +168,9 @@ async function saveRecording(recording: Recording): Promise { await fs.ensureDir(RECORDING_DIR); const timestamp = Date.now(); - const baseFilename = `${recording.app.bundleIdentifier}-${recording.app.processId}-${timestamp}`; + const baseFilename = recording.isGlobal + ? `global-recording-${timestamp}` + : `${app?.bundleIdentifier ?? 'unknown'}-${app?.processId ?? 0}-${timestamp}`; const recordingDir = `${RECORDING_DIR}/${baseFilename}`; await fs.ensureDir(recordingDir); @@ -189,7 +203,7 @@ async function saveRecording(recording: Recording): Promise { console.log('✅ Transcription MP3 file written successfully'); // Save app icon if available - if (app.icon) { + if (app?.icon) { console.log(`📝 Writing app icon to ${iconFilename}`); await fs.writeFile(iconFilename, app.icon); console.log('✅ App icon written successfully'); @@ -198,15 +212,16 @@ async function saveRecording(recording: Recording): Promise { console.log(`📝 Writing metadata to ${metadataFilename}`); // Save metadata with the actual sample rate from the stream const metadata: RecordingMetadata = { - appName: app.name, - bundleIdentifier: app.bundleIdentifier, - processId: app.processId, + appName: app?.name ?? 'Global Recording', + bundleIdentifier: app?.bundleIdentifier ?? 'system.global', + processId: app?.processId ?? -1, recordingStartTime: recording.startTime, recordingEndTime, recordingDuration, sampleRate: actualSampleRate, channels: channelCount, totalSamples, + isGlobal: recording.isGlobal, }; await fs.writeJson(metadataFilename, metadata, { spaces: 2 }); @@ -222,8 +237,8 @@ async function saveRecording(recording: Recording): Promise { function getRecordingStatus(): RecordingStatus[] { return Array.from(recordingMap.entries()).map(([processId, recording]) => ({ processId, - bundleIdentifier: recording.app.bundleIdentifier, - name: recording.app.name, + bundleIdentifier: recording.app?.bundleIdentifier ?? 'system.global', + name: recording.app?.name ?? 'Global Recording', startTime: recording.startTime, duration: Date.now() - recording.startTime, })); @@ -289,8 +304,11 @@ async function stopRecording(processId: number) { } const app = recording.appGroup || recording.app; + const appName = + app?.name ?? (recording.isGlobal ? 'Global Recording' : 'Unknown App'); + const appPid = app?.processId ?? processId; - console.log(`âšī¸ Stopping recording for ${app.name} (PID: ${app.processId})`); + console.log(`âšī¸ Stopping recording for ${appName} (PID: ${appPid})`); console.log( `âąī¸ Recording duration: ${((Date.now() - recording.startTime) / 1000).toFixed(2)}s` ); @@ -302,7 +320,7 @@ async function stopRecording(processId: number) { if (filename) { console.log(`✅ Recording saved successfully to ${filename}`); } else { - console.error(`❌ Failed to save recording for ${app.name}`); + console.error(`❌ Failed to save recording for ${appName}`); } emitRecordingStatus(); @@ -541,7 +559,13 @@ function listenToAppStateChanges(apps: AppInfo[]) { appsSubscriber(); appsSubscriber = () => { - subscribers.forEach(subscriber => subscriber.unsubscribe()); + subscribers.forEach(subscriber => { + try { + subscriber.unsubscribe(); + } catch { + // ignore unsubscribe error + } + }); }; } @@ -606,8 +630,8 @@ app.get('/apps/saved', rateLimiter, async (_req, res) => { // Utility function to validate and sanitize folder name function validateAndSanitizeFolderName(folderName: string): string | null { // Allow alphanumeric characters, hyphens, dots (for bundle IDs) - // Format: bundleId-processId-timestamp - if (!/^[\w.-]+-\d+-\d+$/.test(folderName)) { + // Format: bundleId-processId-timestamp OR global-recording-timestamp + if (!/^([\w.-]+-\d+-\d+|global-recording-\d+)$/.test(folderName)) { return null; } @@ -788,6 +812,65 @@ app.post( } ); +async function startGlobalRecording() { + const GLOBAL_RECORDING_ID = -1; + if (recordingMap.has(GLOBAL_RECORDING_ID)) { + console.log('âš ī¸ Global recording already in progress'); + return; + } + + try { + console.log('đŸŽ™ī¸ Starting global recording'); + + const buffers: Float32Array[] = []; + const stream = ShareableContent.tapGlobalAudio( + null, + (err: Error | null, samples: Float32Array) => { + if (err) { + console.error('❌ Global audio stream error:', err); + return; + } + const recording = recordingMap.get(GLOBAL_RECORDING_ID); + if (recording && !recording.isWriting) { + buffers.push(new Float32Array(samples)); + } + } + ); + + recordingMap.set(GLOBAL_RECORDING_ID, { + app: null, + appGroup: null, + buffers, + stream, + startTime: Date.now(), + isWriting: false, + isGlobal: true, + }); + + console.log('✅ Global recording started successfully'); + emitRecordingStatus(); + } catch (error) { + console.error('❌ Error starting global recording:', error); + } +} + +// Add API endpoint for global recording +app.post('/global/record', async (_req, res) => { + try { + await startGlobalRecording(); + res.json({ success: true }); + } catch (error) { + console.error('❌ Error starting global recording:', error); + res.status(500).json({ error: 'Failed to start global recording' }); + } +}); + +app.post('/global/stop', async (_req, res) => { + const GLOBAL_RECORDING_ID = -1; + await stopRecording(GLOBAL_RECORDING_ID); + res.json({ success: true }); +}); + // Start server httpServer.listen(PORT, () => { console.log(` diff --git a/packages/frontend/media-capture-playground/web/app.tsx b/packages/frontend/media-capture-playground/web/app.tsx index a3aaa7ff4d..7b8987f5e6 100644 --- a/packages/frontend/media-capture-playground/web/app.tsx +++ b/packages/frontend/media-capture-playground/web/app.tsx @@ -1,4 +1,5 @@ import { AppList } from './components/app-list'; +import { GlobalRecordButton } from './components/global-record-button'; import { SavedRecordings } from './components/saved-recordings'; export function App() { @@ -6,11 +7,15 @@ export function App() {
-

- Running Applications -

+
+

+ Running Applications +

+ +

- Select an application to start recording its audio + Select an application to start recording its audio, or use global + recording for system-wide audio

diff --git a/packages/frontend/media-capture-playground/web/components/global-record-button.tsx b/packages/frontend/media-capture-playground/web/components/global-record-button.tsx new file mode 100644 index 0000000000..6884b8b010 --- /dev/null +++ b/packages/frontend/media-capture-playground/web/components/global-record-button.tsx @@ -0,0 +1,70 @@ +import { useCallback, useEffect, useState } from 'react'; + +import { socket } from '../utils'; + +export function GlobalRecordButton() { + const [isRecording, setIsRecording] = useState(false); + const [isLoading, setIsLoading] = useState(false); + + useEffect(() => { + function handleRecordingStatus(data: { + recordings: Array<{ processId: number }>; + }) { + // Global recording uses processId -1 + setIsRecording(data.recordings.some(r => r.processId === -1)); + } + + socket.on('apps:recording', handleRecordingStatus); + return () => { + socket.off('apps:recording', handleRecordingStatus); + }; + }, []); + + const handleClick = useCallback(() => { + setIsLoading(true); + const endpoint = isRecording ? '/api/global/stop' : '/api/global/record'; + fetch(endpoint, { method: 'POST' }) + .then(response => { + if (!response.ok) { + throw new Error('Failed to toggle global recording'); + } + }) + .catch(error => { + console.error('Error toggling global recording:', error); + }) + .finally(() => { + setIsLoading(false); + }); + }, [isRecording]); + + return ( + + ); +} diff --git a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx index 217d300d49..61da0adfd4 100644 --- a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx +++ b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx @@ -378,11 +378,12 @@ function RecordingHeader({ transcriptionError: string | null; }): ReactElement { const [imgError, setImgError] = React.useState(false); + const isGlobalRecording = metadata?.isGlobal; return (
- {!imgError ? ( + {!imgError && !isGlobalRecording ? ( {metadata?.appName {metadata?.appName || 'Unknown Application'} - + {isGlobalRecording && ( + + System Audio + + )} + {duration}
diff --git a/packages/frontend/media-capture-playground/web/types.ts b/packages/frontend/media-capture-playground/web/types.ts index ced4bd4e23..c892c419a0 100644 --- a/packages/frontend/media-capture-playground/web/types.ts +++ b/packages/frontend/media-capture-playground/web/types.ts @@ -17,6 +17,8 @@ export interface RecordingStatus { bundleIdentifier: string; name: string; startTime: number; + duration: number; + isGlobal?: boolean; } export interface RecordingMetadata { @@ -31,6 +33,7 @@ export interface RecordingMetadata { totalSamples: number; icon?: Uint8Array; mp3: string; + isGlobal?: boolean; } export interface TranscriptionMetadata { diff --git a/packages/frontend/native/media_capture/src/macos/tap_audio.rs b/packages/frontend/native/media_capture/src/macos/tap_audio.rs index 1004b7dfef..cfce74c201 100644 --- a/packages/frontend/native/media_capture/src/macos/tap_audio.rs +++ b/packages/frontend/native/media_capture/src/macos/tap_audio.rs @@ -1,4 +1,4 @@ -use std::{ffi::c_void, sync::Arc}; +use std::{ffi::c_void, ptr, sync::Arc}; use block2::{Block, RcBlock}; use core_foundation::{ @@ -13,12 +13,15 @@ use coreaudio::sys::{ kAudioAggregateDeviceIsPrivateKey, kAudioAggregateDeviceIsStackedKey, kAudioAggregateDeviceMainSubDeviceKey, kAudioAggregateDeviceNameKey, kAudioAggregateDeviceSubDeviceListKey, kAudioAggregateDeviceTapAutoStartKey, - kAudioAggregateDeviceTapListKey, kAudioAggregateDeviceUIDKey, kAudioHardwareNoError, - kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultSystemOutputDevice, - kAudioSubDeviceUIDKey, kAudioSubTapDriftCompensationKey, kAudioSubTapUIDKey, - AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID, AudioDeviceIOProcID, - AudioDeviceStart, AudioDeviceStop, AudioHardwareCreateAggregateDevice, - AudioHardwareDestroyAggregateDevice, AudioObjectID, AudioTimeStamp, OSStatus, + kAudioAggregateDeviceTapListKey, kAudioAggregateDeviceUIDKey, + kAudioDevicePropertyAvailableNominalSampleRates, kAudioDevicePropertyNominalSampleRate, + kAudioHardwareNoError, kAudioHardwarePropertyDefaultInputDevice, + kAudioHardwarePropertyDefaultSystemOutputDevice, kAudioObjectPropertyElementMain, + kAudioObjectPropertyScopeGlobal, kAudioSubDeviceUIDKey, kAudioSubTapDriftCompensationKey, + kAudioSubTapUIDKey, AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID, + AudioDeviceIOProcID, AudioDeviceStart, AudioDeviceStop, AudioHardwareCreateAggregateDevice, + AudioHardwareDestroyAggregateDevice, AudioObjectGetPropertyData, AudioObjectGetPropertyDataSize, + AudioObjectID, AudioObjectPropertyAddress, AudioObjectSetPropertyData, AudioTimeStamp, OSStatus, }; use napi::{ bindgen_prelude::Float32Array, @@ -30,8 +33,11 @@ use objc2::{runtime::AnyObject, Encode, Encoding, RefEncode}; use crate::{ audio_stream_basic_desc::read_audio_stream_basic_description, - ca_tap_description::CATapDescription, device::get_device_uid, error::CoreAudioError, - queue::create_audio_tap_queue, screen_capture_kit::TappableApplication, + ca_tap_description::CATapDescription, + device::{get_device_audio_id, get_device_uid}, + error::CoreAudioError, + queue::create_audio_tap_queue, + screen_capture_kit::TappableApplication, }; extern "C" { @@ -53,6 +59,14 @@ pub struct AudioBuffer { pub mData: *mut c_void, } +// Define a struct to represent sample rate ranges +#[repr(C)] +#[allow(non_snake_case)] +struct AudioValueRange { + mMinimum: f64, + mMaximum: f64, +} + unsafe impl Encode for AudioBuffer { const ENCODING: Encoding = Encoding::Struct( "AudioBuffer", @@ -94,6 +108,10 @@ pub struct AggregateDevice { pub tap_id: AudioObjectID, pub id: AudioObjectID, pub audio_stats: Option, + pub input_device_id: Option, + pub output_device_id: Option, + pub input_proc_id: Option, + pub output_proc_id: Option, } impl AggregateDevice { @@ -128,6 +146,10 @@ impl AggregateDevice { tap_id, id: aggregate_device_id, audio_stats: None, + input_device_id: None, + output_device_id: None, + input_proc_id: None, + output_proc_id: None, }) } @@ -160,6 +182,10 @@ impl AggregateDevice { tap_id, id: aggregate_device_id, audio_stats: None, + input_device_id: None, + output_device_id: None, + input_proc_id: None, + output_proc_id: None, }) } @@ -173,6 +199,12 @@ impl AggregateDevice { return Err(CoreAudioError::CreateProcessTapFailed(status).into()); } + // Get the default input device (microphone) UID and ID + let input_device_id = get_device_audio_id(kAudioHardwarePropertyDefaultInputDevice)?; + + // Get the default output device ID + let output_device_id = get_device_audio_id(kAudioHardwarePropertyDefaultSystemOutputDevice)?; + let description_dict = Self::create_aggregate_description(tap_id, tap_description.get_uuid()?)?; let mut aggregate_device_id: AudioObjectID = 0; @@ -189,30 +221,246 @@ impl AggregateDevice { return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into()); } - Ok(Self { + // Create a device with stored device IDs + let mut device = Self { tap_id, id: aggregate_device_id, audio_stats: None, - }) + input_device_id: Some(input_device_id), + output_device_id: Some(output_device_id), + input_proc_id: None, + output_proc_id: None, + }; + + // Configure the aggregate device to ensure proper handling of both input and + // output + device.configure_aggregate_device()?; + + // Activate both the input and output devices and store their proc IDs + let input_proc_id = device.activate_audio_device(input_device_id)?; + let output_proc_id = device.activate_audio_device(output_device_id)?; + + device.input_proc_id = Some(input_proc_id); + device.output_proc_id = Some(output_proc_id); + + Ok(device) + } + + // Configures the aggregate device to ensure proper handling of both input and + // output streams + fn configure_aggregate_device(&self) -> Result { + // Read the current audio format to ensure it's properly configured + let audio_format = read_audio_stream_basic_description(self.tap_id)?; + + // Create initial audio stats with the actual sample rate but always use mono + let initial_sample_rate = audio_format.0.mSampleRate; + let mut audio_stats = AudioStats { + sample_rate: initial_sample_rate, + channels: 1, // Always set to 1 channel (mono) + }; + + // Set the preferred sample rate on the device + // This is similar to how Screen Capture Kit allows setting the sample rate + let preferred_sample_rate = initial_sample_rate; // Use the device's current sample rate + + // First, check if the preferred sample rate is available + let mut is_sample_rate_available = false; + let mut best_available_rate = preferred_sample_rate; // Default to preferred rate + + unsafe { + // Get the available sample rates + let address = AudioObjectPropertyAddress { + mSelector: kAudioDevicePropertyAvailableNominalSampleRates, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }; + + // Get the size of the property data + let mut data_size: u32 = 0; + let status = AudioObjectGetPropertyDataSize( + self.id, + &address as *const AudioObjectPropertyAddress, + 0, + std::ptr::null(), + &mut data_size as *mut u32, + ); + + if status == 0 && data_size > 0 { + // Calculate how many ranges we have + let range_count = data_size as usize / std::mem::size_of::(); + + // Allocate memory for the ranges + let mut ranges: Vec = Vec::with_capacity(range_count); + ranges.set_len(range_count); + + // Get the available sample rates + let status = AudioObjectGetPropertyData( + self.id, + &address as *const AudioObjectPropertyAddress, + 0, + std::ptr::null(), + &mut data_size as *mut u32, + ranges.as_mut_ptr() as *mut std::ffi::c_void, + ); + + if status == 0 { + // Check if our preferred sample rate is within any of the available ranges + for range in &ranges { + if preferred_sample_rate >= range.mMinimum && preferred_sample_rate <= range.mMaximum { + is_sample_rate_available = true; + break; + } + } + + // If not available, find the best available rate + if !is_sample_rate_available && !ranges.is_empty() { + // Common preferred sample rates in order of preference + let common_rates = [48000.0, 44100.0, 96000.0, 88200.0, 24000.0, 22050.0]; + let mut found_common_rate = false; + + // First try to find a common rate that's available + for &rate in &common_rates { + for range in &ranges { + if rate >= range.mMinimum && rate <= range.mMaximum { + best_available_rate = rate; + found_common_rate = true; + break; + } + } + if found_common_rate { + break; + } + } + + // If no common rate is available, use the highest available rate + if !found_common_rate { + // Find the highest available rate + for range in &ranges { + // Use the maximum of the range as our best available rate + if range.mMaximum > best_available_rate { + best_available_rate = range.mMaximum; + } + } + } + } + } + } + } + + // Set the sample rate to either the preferred rate or the best available rate + let sample_rate_to_set = if is_sample_rate_available { + preferred_sample_rate + } else { + best_available_rate + }; + + let status = unsafe { + // Note on scope usage: + // We use kAudioObjectPropertyScopeGlobal here because it works reliably for + // setting the nominal sample rate on the device. While + // kAudioObjectPropertyScopeInput or kAudioObjectPropertyScopeOutput might + // also work in some cases (as mentioned in the comments), + // kAudioObjectPropertyScopeGlobal is the most consistent approach. + // + // The CoreAudio documentation doesn't explicitly specify which scope to use + // with kAudioDevicePropertyNominalSampleRate, but in practice, + // kAudioObjectPropertyScopeGlobal ensures the sample rate is set for the + // entire device, affecting both input and output. + let address = AudioObjectPropertyAddress { + mSelector: kAudioDevicePropertyNominalSampleRate, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }; + + // Set the sample rate property + AudioObjectSetPropertyData( + self.id, + &address as *const AudioObjectPropertyAddress, + 0, + std::ptr::null(), + std::mem::size_of::() as u32, + &sample_rate_to_set as *const f64 as *const std::ffi::c_void, + ) + }; + + // Update the audio_stats with the actual sample rate that was set if successful + if status == 0 { + audio_stats.sample_rate = sample_rate_to_set; + + // Verify the actual sample rate by reading it back + unsafe { + let address = AudioObjectPropertyAddress { + mSelector: kAudioDevicePropertyNominalSampleRate, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }; + + let mut actual_rate: f64 = 0.0; + let mut data_size = std::mem::size_of::() as u32; + + let status = AudioObjectGetPropertyData( + self.id, + &address as *const AudioObjectPropertyAddress, + 0, + std::ptr::null(), + &mut data_size as *mut u32, + &mut actual_rate as *mut f64 as *mut std::ffi::c_void, + ); + + if status == 0 { + // Update with the verified rate + audio_stats.sample_rate = actual_rate; + } + } + } + + Ok(audio_stats) + } + + // Activates an audio device by creating a dummy IO proc + fn activate_audio_device(&self, device_id: AudioObjectID) -> Result { + // Create a simple no-op dummy proc + let dummy_block = RcBlock::new( + |_: *mut c_void, _: *mut c_void, _: *mut c_void, _: *mut c_void, _: *mut c_void| { + // No-op function that just returns success + kAudioHardwareNoError as i32 + }, + ); + + let mut dummy_proc_id: AudioDeviceIOProcID = None; + + // Create the IO proc with our dummy block + let status = unsafe { + AudioDeviceCreateIOProcIDWithBlock( + &mut dummy_proc_id, + device_id, + ptr::null_mut(), + (&*dummy_block.copy() as *const Block i32>) + .cast_mut() + .cast(), + ) + }; + + if status != 0 { + return Err(CoreAudioError::CreateIOProcIDWithBlockFailed(status).into()); + } + + // Start the device to activate it + let status = unsafe { AudioDeviceStart(device_id, dummy_proc_id) }; + if status != 0 { + return Err(CoreAudioError::AudioDeviceStartFailed(status).into()); + } + + // Return the proc ID for later cleanup + Ok(dummy_proc_id) } pub fn start( &mut self, audio_stream_callback: Arc>, ) -> Result { - // Read and log the audio format before starting the device - let mut audio_stats = AudioStats { - sample_rate: 44100.0, - channels: 1, // Always set to 1 channel (mono) - }; - - if let Ok(audio_format) = read_audio_stream_basic_description(self.tap_id) { - // Store the audio format information - audio_stats.sample_rate = audio_format.0.mSampleRate; - // Always use 1 channel regardless of what the system reports - audio_stats.channels = 1; - } - + // Configure the aggregate device and get audio stats before starting + let audio_stats = self.configure_aggregate_device()?; self.audio_stats = Some(audio_stats); let audio_stats_clone = audio_stats; @@ -242,7 +490,7 @@ impl AggregateDevice { }] = mBuffers; // Only create slice if we have valid data if !mData.is_null() && *mDataByteSize > 0 { - // Calculate total number of samples (accounting for interleaved stereo) + // Calculate total number of samples (total bytes / bytes per sample) let total_samples = *mDataByteSize as usize / 4; // 4 bytes per f32 // Create a slice of all samples @@ -253,27 +501,18 @@ impl AggregateDevice { let channel_count = *mNumberChannels as usize; // Process the audio based on channel count - let mut processed_samples: Vec; + let processed_samples: Vec; if channel_count > 1 { // For stereo, samples are interleaved: [L, R, L, R, ...] // We need to average each pair to get mono - let frame_count = total_samples / channel_count; - processed_samples = Vec::with_capacity(frame_count); - - for i in 0..frame_count { - let mut frame_sum = 0.0; - for c in 0..channel_count { - frame_sum += samples[i * channel_count + c]; - } - processed_samples.push(frame_sum / (channel_count as f32)); - } + processed_samples = process_mixed_audio(samples, channel_count); } else { - // Already mono, just copy the samples + // For mono, just copy the samples processed_samples = samples.to_vec(); } - // Pass the processed samples to the callback + // Send the processed audio data to JavaScript audio_stream_callback.call( Ok(processed_samples.into()), ThreadsafeFunctionCallMode::NonBlocking, @@ -309,7 +548,11 @@ impl AggregateDevice { device_id: self.id, in_proc_id, stop_called: false, - audio_stats: audio_stats_clone, + audio_stats: audio_stats_clone, // Use the updated audio_stats with the actual sample rate + input_device_id: self.input_device_id, + output_device_id: self.output_device_id, + input_proc_id: self.input_proc_id, + output_proc_id: self.output_proc_id, }) } @@ -325,15 +568,29 @@ impl AggregateDevice { let aggregate_device_uid_string = aggregate_device_uid.to_string(); // Sub-device UID key and dictionary - let sub_device_output_dict = CFDictionary::from_CFType_pairs(&[( - cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(), - system_output_uid.as_CFType(), - )]); + let sub_device_output_dict = CFDictionary::from_CFType_pairs(&[ + ( + cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(), + system_output_uid.as_CFType(), + ), + // Explicitly mark this as an output device + ( + CFString::new("com.apple.audio.roles").as_CFType(), + CFString::new("output").as_CFType(), + ), + ]); - let sub_device_input_dict = CFDictionary::from_CFType_pairs(&[( - cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(), - default_input_uid.as_CFType(), - )]); + let sub_device_input_dict = CFDictionary::from_CFType_pairs(&[ + ( + cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(), + default_input_uid.as_CFType(), + ), + // Explicitly mark this as an input device + ( + CFString::new("com.apple.audio.roles").as_CFType(), + CFString::new("input").as_CFType(), + ), + ]); let tap_device_dict = CFDictionary::from_CFType_pairs(&[ ( @@ -346,6 +603,7 @@ impl AggregateDevice { ), ]); + // Put input device first in the list to prioritize it let capture_device_list = vec![sub_device_input_dict, sub_device_output_dict]; // Sub-device list @@ -353,7 +611,8 @@ impl AggregateDevice { let tap_list = CFArray::from_CFTypes(&[tap_device_dict]); - // Create the aggregate device description dictionary + // Create the aggregate device description dictionary with a balanced + // configuration let description_dict = CFDictionary::from_CFType_pairs(&[ ( cfstring_from_bytes_with_nul(kAudioAggregateDeviceNameKey).as_CFType(), @@ -365,7 +624,9 @@ impl AggregateDevice { ), ( cfstring_from_bytes_with_nul(kAudioAggregateDeviceMainSubDeviceKey).as_CFType(), - system_output_uid.as_CFType(), + // Use a balanced approach that includes both input and output + // but prioritize input for microphone capture + default_input_uid.as_CFType(), ), ( cfstring_from_bytes_with_nul(kAudioAggregateDeviceIsPrivateKey).as_CFType(), @@ -398,6 +659,10 @@ pub struct AudioTapStream { in_proc_id: AudioDeviceIOProcID, stop_called: bool, audio_stats: AudioStats, + input_device_id: Option, + output_device_id: Option, + input_proc_id: Option, + output_proc_id: Option, } #[napi] @@ -408,22 +673,47 @@ impl AudioTapStream { return Ok(()); } self.stop_called = true; + + // Stop the main aggregate device let status = unsafe { AudioDeviceStop(self.device_id, self.in_proc_id) }; if status != 0 { return Err(CoreAudioError::AudioDeviceStopFailed(status).into()); } + + // Stop the input device if it was activated + if let Some(input_id) = self.input_device_id { + if let Some(proc_id) = self.input_proc_id { + let _ = unsafe { AudioDeviceStop(input_id, proc_id) }; + let _ = unsafe { AudioDeviceDestroyIOProcID(input_id, proc_id) }; + } + } + + // Stop the output device if it was activated + if let Some(output_id) = self.output_device_id { + if let Some(proc_id) = self.output_proc_id { + let _ = unsafe { AudioDeviceStop(output_id, proc_id) }; + let _ = unsafe { AudioDeviceDestroyIOProcID(output_id, proc_id) }; + } + } + + // Destroy the main IO proc let status = unsafe { AudioDeviceDestroyIOProcID(self.device_id, self.in_proc_id) }; if status != 0 { return Err(CoreAudioError::AudioDeviceDestroyIOProcIDFailed(status).into()); } + + // Destroy the aggregate device let status = unsafe { AudioHardwareDestroyAggregateDevice(self.device_id) }; if status != 0 { return Err(CoreAudioError::AudioHardwareDestroyAggregateDeviceFailed(status).into()); } + + // Destroy the process tap let status = unsafe { AudioHardwareDestroyProcessTap(self.device_id) }; if status != 0 { return Err(CoreAudioError::AudioHardwareDestroyProcessTapFailed(status).into()); } + Ok(()) } @@ -445,3 +735,21 @@ fn cfstring_from_bytes_with_nul(bytes: &'static [u8]) -> CFString { .as_ref(), ) } + +// Process mixed audio from multiple channels +fn process_mixed_audio(samples: &[f32], channel_count: usize) -> Vec { + // For stereo or multi-channel audio, we need to mix down to mono + let samples_per_channel = samples.len() / channel_count; + let mut mixed_samples = Vec::with_capacity(samples_per_channel); + + for i in 0..samples_per_channel { + let mut sample_sum = 0.0; + for c in 0..channel_count { + sample_sum += samples[i * channel_count + c]; + } + // Average the samples from all channels + mixed_samples.push(sample_sum / channel_count as f32); + } + + mixed_samples +} diff --git a/tests/affine-desktop/playwright.config.ts b/tests/affine-desktop/playwright.config.ts index e28f551d15..323b52e5db 100644 --- a/tests/affine-desktop/playwright.config.ts +++ b/tests/affine-desktop/playwright.config.ts @@ -29,7 +29,7 @@ const config: PlaywrightTestConfig = { if (process.env.CI) { config.retries = 5; - config.workers = 2; + config.workers = 1; } if (process.env.DEV_SERVER_URL) { diff --git a/tests/kit/src/electron.ts b/tests/kit/src/electron.ts index 64a3762d61..02bce1683a 100644 --- a/tests/kit/src/electron.ts +++ b/tests/kit/src/electron.ts @@ -1,4 +1,5 @@ import crypto from 'node:crypto'; +import { setTimeout } from 'node:timers/promises'; import { Package } from '@affine-tools/utils/workspace'; import { expect, type Page } from '@playwright/test'; @@ -71,7 +72,7 @@ export const test = base.extend<{ return electronApp.windows().length > 1; }, { - timeout: 50000, + timeout: 10000, } ) .toBeTruthy(); @@ -83,7 +84,7 @@ export const test = base.extend<{ return !!page; }, { - timeout: 50000, + timeout: 10000, } ) .toBeTruthy(); @@ -147,12 +148,25 @@ export const test = base.extend<{ }); await use(electronApp); - const pages = electronApp.windows(); - for (const page of pages) { - await page.close(); - } - await electronApp.close(); - await removeWithRetry(clonedDist); + const cleanup = async () => { + const pages = electronApp.windows(); + for (const page of pages) { + if (page.isClosed()) { + continue; + } + await page.close(); + } + await electronApp.close(); + await removeWithRetry(clonedDist); + }; + await Promise.race([ + // cleanup may stuck and fail the test, but it should be fine. + cleanup(), + setTimeout(10000).then(() => { + // kill the electron app if it is not closed after 10 seconds + electronApp.process().kill(); + }), + ]); } catch (error) { console.log(error); } diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts index 443d9eeb74..5b6df13d07 100644 --- a/tools/utils/src/workspace.gen.ts +++ b/tools/utils/src/workspace.gen.ts @@ -733,6 +733,7 @@ export const PackageList = [ 'packages/frontend/electron-api', 'packages/frontend/i18n', 'packages/common/nbstore', + 'blocksuite/affine/all', 'packages/common/infra', 'tools/utils', ], diff --git a/yarn.lock b/yarn.lock index aec8ceeb57..b83142a87a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -496,6 +496,7 @@ __metadata: "@affine/electron-api": "workspace:*" "@affine/i18n": "workspace:*" "@affine/nbstore": "workspace:*" + "@blocksuite/affine": "workspace:*" "@emotion/react": "npm:^11.14.0" "@sentry/react": "npm:^9.2.0" "@toeverything/infra": "workspace:*"