feat(electron): create recording through tray (#10526)

- added tray menu for controlling recording status
- recording watcher for monitoring system audio input events
This commit is contained in:
pengx17
2025-03-18 04:12:30 +00:00
parent 05329e96c7
commit a016630a82
29 changed files with 1186 additions and 258 deletions

View File

@@ -2,6 +2,7 @@ import { AsyncCall } from 'async-call-rpc';
import type { HelperToMain, MainToHelper } from '../shared/type';
import { exposed } from './provide';
import { encodeToMp3 } from './recording/encode';
const helperToMainServer: HelperToMain = {
getMeta: () => {
@@ -10,6 +11,8 @@ const helperToMainServer: HelperToMain = {
}
return exposed;
},
// allow main process encode audio samples to mp3 buffer (because it is slow and blocking)
encodeToMp3,
};
export const mainRPC = AsyncCall<MainToHelper>(helperToMainServer, {

View File

@@ -0,0 +1,16 @@
import { Mp3Encoder } from '@affine/native';
// encode audio samples to mp3 buffer
export function encodeToMp3(
samples: Float32Array,
opts: {
channels?: number;
sampleRate?: number;
} = {}
): Uint8Array {
const mp3Encoder = new Mp3Encoder({
channels: opts.channels ?? 2,
sampleRate: opts.sampleRate ?? 44100,
});
return mp3Encoder.encode(samples);
}

View File

@@ -4,6 +4,7 @@ import { AFFINE_EVENT_CHANNEL_NAME } from '../shared/type';
import { applicationMenuEvents } from './application-menu';
import { beforeAppQuit } from './cleanup';
import { logger } from './logger';
import { recordingEvents } from './recording';
import { sharedStorageEvents } from './shared-storage';
import { uiEvents } from './ui/events';
import { updaterEvents } from './updater/event';
@@ -13,6 +14,7 @@ export const allEvents = {
updater: updaterEvents,
ui: uiEvents,
sharedStorage: sharedStorageEvents,
recording: recordingEvents,
};
function getActiveWindows() {

View File

@@ -5,6 +5,7 @@ import { clipboardHandlers } from './clipboard';
import { configStorageHandlers } from './config-storage';
import { findInPageHandlers } from './find-in-page';
import { getLogFilePath, logger, revealLogFile } from './logger';
import { recordingHandlers } from './recording';
import { sharedStorageHandlers } from './shared-storage';
import { uiHandlers } from './ui/handlers';
import { updaterHandlers } from './updater';
@@ -29,6 +30,7 @@ export const allHandlers = {
findInPage: findInPageHandlers,
sharedStorage: sharedStorageHandlers,
worker: workerHandlers,
recording: recordingHandlers,
};
export const registerHandlers = () => {

View File

@@ -14,6 +14,8 @@ import { registerEvents } from './events';
import { registerHandlers } from './handlers';
import { logger } from './logger';
import { registerProtocol } from './protocol';
import { setupRecording } from './recording';
import { getTrayState } from './tray';
import { registerUpdater } from './updater';
import { launch } from './windows-manager/launcher';
import { launchStage } from './windows-manager/stage';
@@ -85,7 +87,9 @@ app
.then(registerHandlers)
.then(registerEvents)
.then(launch)
.then(setupRecording)
.then(createApplicationMenu)
.then(getTrayState)
.then(registerUpdater)
.catch(e => console.error('Failed create window:', e));

View File

@@ -1,38 +1,30 @@
import { ShareableContent, TappableApplication } from '@affine/native';
import { Notification } from 'electron';
import {
BehaviorSubject,
distinctUntilChanged,
pairwise,
startWith,
} from 'rxjs';
import { ShareableContent } from '@affine/native';
import { nativeImage, Notification } from 'electron';
import { debounce } from 'lodash-es';
import { BehaviorSubject, distinctUntilChanged, groupBy, mergeMap } from 'rxjs';
import { isMacOS } from '../../shared/utils';
import { beforeAppQuit } from '../cleanup';
import { ensureHelperProcess } from '../helper-process';
import { logger } from '../logger';
interface TappableAppInfo {
rawInstance: TappableApplication;
isRunning: boolean;
processId: number;
processGroupId: number;
bundleIdentifier: string;
name: string;
}
interface AppGroupInfo {
processGroupId: number;
apps: TappableAppInfo[];
name: string;
icon: Buffer | undefined;
isRunning: boolean;
}
import type { NamespaceHandlers } from '../type';
import { getMainWindow } from '../windows-manager';
import type {
AppGroupInfo,
Recording,
RecordingStatus,
TappableAppInfo,
} from './types';
const subscribers: Subscriber[] = [];
beforeAppQuit(() => {
subscribers.forEach(subscriber => {
subscriber.unsubscribe();
try {
subscriber.unsubscribe();
} catch {
// ignore unsubscribe error
}
});
});
@@ -41,8 +33,41 @@ let shareableContent: ShareableContent | null = null;
export const applications$ = new BehaviorSubject<TappableAppInfo[]>([]);
export const appGroups$ = new BehaviorSubject<AppGroupInfo[]>([]);
if (isMacOS()) {
// Update appGroups$ whenever applications$ changes
// recording id -> recording
// recordings will be saved in memory before consumed and created as an audio block to user's doc
const recordings = new Map<number, Recording>();
// there should be only one active recording at a time
export const recordingStatus$ = new BehaviorSubject<RecordingStatus | null>(
null
);
function createAppGroup(processGroupId: number): AppGroupInfo | undefined {
const groupProcess =
shareableContent?.applicationWithProcessId(processGroupId);
if (!groupProcess) {
return;
}
return {
processGroupId: processGroupId,
apps: [], // leave it empty for now.
name: groupProcess.name,
bundleIdentifier: groupProcess.bundleIdentifier,
// icon should be lazy loaded
get icon() {
try {
return groupProcess.icon;
} catch (error) {
logger.error(`Failed to get icon for ${groupProcess.name}`, error);
return undefined;
}
},
isRunning: false,
};
}
// pipe applications$ to appGroups$
function setupAppGroups() {
subscribers.push(
applications$.pipe(distinctUntilChanged()).subscribe(apps => {
const appGroups: AppGroupInfo[] = [];
@@ -52,69 +77,160 @@ if (isMacOS()) {
);
if (!appGroup) {
const groupProcess = shareableContent?.applicationWithProcessId(
app.processGroupId
);
if (!groupProcess) {
return;
appGroup = createAppGroup(app.processGroupId);
if (appGroup) {
appGroups.push(appGroup);
}
appGroup = {
processGroupId: app.processGroupId,
apps: [],
name: groupProcess.name,
// icon will be lazy loaded
get icon() {
try {
return groupProcess.icon;
} catch (error) {
logger.error(
`Failed to get icon for ${groupProcess.name}`,
error
);
return undefined;
}
},
get isRunning() {
return this.apps.some(app => app.rawInstance.isRunning);
},
};
appGroups.push(appGroup);
}
if (appGroup) {
appGroup.apps.push(app);
}
});
appGroups.forEach(appGroup => {
appGroup.isRunning = appGroup.apps.some(app => app.isRunning);
});
appGroups$.next(appGroups);
})
);
}
function setupNewRunningAppGroup() {
const appGroupRunningChanged$ = appGroups$.pipe(
mergeMap(groups => groups),
groupBy(group => group.processGroupId),
mergeMap(groupStream$ =>
groupStream$.pipe(
distinctUntilChanged((prev, curr) => prev.isRunning === curr.isRunning)
)
)
);
subscribers.push(
appGroups$
.pipe(startWith([] as AppGroupInfo[]), pairwise())
.subscribe(([previousGroups, currentGroups]) => {
currentGroups.forEach(currentGroup => {
const previousGroup = previousGroups.find(
group => group.processGroupId === currentGroup.processGroupId
);
if (previousGroup?.isRunning !== currentGroup.isRunning) {
console.log(
'appgroup running changed',
currentGroup.name,
currentGroup.isRunning
);
if (currentGroup.isRunning) {
new Notification({
title: 'Recording Meeting',
body: `Recording meeting with ${currentGroup.name}`,
}).show();
}
}
appGroupRunningChanged$.subscribe(currentGroup => {
if (currentGroup.isRunning) {
// TODO(@pengx17): stub impl. will be replaced with a real one later
const notification = new Notification({
icon: currentGroup.icon
? nativeImage.createFromBuffer(currentGroup.icon)
: undefined,
title: 'Recording Meeting',
body: `Recording meeting with ${currentGroup.name}`,
actions: [
{
type: 'button',
text: 'Start',
},
],
});
})
notification.on('action', () => {
startRecording(currentGroup);
});
notification.show();
} else {
// if the group is not running, we should stop the recording (if it is recording)
if (
recordingStatus$.value?.status === 'recording' &&
recordingStatus$.value?.appGroup?.processGroupId ===
currentGroup.processGroupId
) {
stopRecording();
}
}
})
);
}
async function getAllApps(): Promise<TappableAppInfo[]> {
function createRecording(status: RecordingStatus) {
const buffers: Float32Array[] = [];
function tapAudioSamples(err: Error | null, samples: Float32Array) {
const recordingStatus = recordingStatus$.getValue();
if (
!recordingStatus ||
recordingStatus.id !== status.id ||
recordingStatus.status === 'paused'
) {
return;
}
if (err) {
logger.error('failed to get audio samples', err);
} else {
buffers.push(new Float32Array(samples));
}
}
const stream = status.app
? status.app.rawInstance.tapAudio(tapAudioSamples)
: ShareableContent.tapGlobalAudio(null, tapAudioSamples);
const recording: Recording = {
id: status.id,
startTime: status.startTime,
app: status.app,
appGroup: status.appGroup,
buffers,
stream,
};
return recording;
}
function concatBuffers(buffers: Float32Array[]): Float32Array {
const totalSamples = buffers.reduce((acc, buf) => acc + buf.length, 0);
const buffer = new Float32Array(totalSamples);
let offset = 0;
buffers.forEach(buf => {
buffer.set(buf, offset);
offset += buf.length;
});
return buffer;
}
export async function saveRecording(id: number) {
const recording = recordings.get(id);
if (!recording) {
logger.error(`Recording ${id} not found`);
return;
}
const { buffers } = recording;
const helperProcessManager = await ensureHelperProcess();
const buffer = concatBuffers(buffers);
const mp3Buffer = await helperProcessManager.rpc?.encodeToMp3(buffer, {
channels: recording.stream.channels,
sampleRate: recording.stream.sampleRate,
});
if (!mp3Buffer) {
logger.error('failed to encode audio samples to mp3');
return;
}
recordings.delete(recording.id);
return mp3Buffer;
}
function setupRecordingListeners() {
subscribers.push(
recordingStatus$.pipe(distinctUntilChanged()).subscribe(status => {
if (status?.status === 'recording') {
let recording = recordings.get(status.id);
// create a recording if not exists
if (!recording) {
recording = createRecording(status);
recordings.set(status.id, recording);
}
} else if (status?.status === 'stopped') {
const recording = recordings.get(status.id);
if (recording) {
recording.stream.stop();
}
}
})
);
}
function getAllApps(): TappableAppInfo[] {
if (!shareableContent) {
return [];
}
@@ -150,62 +266,37 @@ type Subscriber = {
};
function setupMediaListeners() {
applications$.next(getAllApps());
subscribers.push(
ShareableContent.onApplicationListChanged(() => {
getAllApps()
.then(apps => {
applications$.next(apps);
})
.catch(err => {
logger.error('failed to get apps', err);
});
applications$.next(getAllApps());
})
);
getAllApps()
.then(apps => {
applications$.next(apps);
})
.catch(err => {
logger.error('failed to get apps', err);
});
let appStateSubscribers: Subscriber[] = [];
subscribers.push(
applications$.subscribe(apps => {
appStateSubscribers.forEach(subscriber => {
subscriber.unsubscribe();
try {
subscriber.unsubscribe();
} catch {
// ignore unsubscribe error
}
});
const _appStateSubscribers: Subscriber[] = [];
apps.forEach(app => {
try {
// Try to create a TappableApplication with a default audio object ID
// In a real implementation, you would need to get the actual audio object ID
// This is just a placeholder value that seems to work for testing
const tappableApp = TappableApplication.fromApplication(
app.rawInstance,
1
const tappableApp = app.rawInstance;
const debouncedAppStateChanged = debounce(() => {
applications$.next(getAllApps());
}, 100);
_appStateSubscribers.push(
ShareableContent.onAppStateChanged(tappableApp, () => {
debouncedAppStateChanged();
})
);
if (tappableApp) {
_appStateSubscribers.push(
ShareableContent.onAppStateChanged(tappableApp, () => {
setTimeout(() => {
const apps = applications$.getValue();
applications$.next(
apps.map(_app => {
if (_app.processId === app.processId) {
return { ..._app, isRunning: tappableApp.isRunning };
}
return _app;
})
);
}, 10);
})
);
}
} catch (error) {
logger.error(
`Failed to convert app ${app.name} to TappableApplication`,
@@ -217,15 +308,23 @@ function setupMediaListeners() {
appStateSubscribers = _appStateSubscribers;
return () => {
_appStateSubscribers.forEach(subscriber => {
subscriber.unsubscribe();
try {
subscriber.unsubscribe();
} catch {
// ignore unsubscribe error
}
});
};
})
);
}
export function getShareableContent() {
if (!shareableContent && isMacOS()) {
export function setupRecording() {
if (!isMacOS()) {
return;
}
if (!shareableContent) {
try {
shareableContent = new ShareableContent();
setupMediaListeners();
@@ -233,5 +332,99 @@ export function getShareableContent() {
logger.error('failed to get shareable content', error);
}
}
return shareableContent;
setupAppGroups();
setupNewRunningAppGroup();
setupRecordingListeners();
}
let recordingId = 0;
export function startRecording(
appGroup?: AppGroupInfo
): RecordingStatus | undefined {
if (!shareableContent) {
return; // likely called on unsupported platform
}
// hmm, is it possible that there are multiple apps running (listening) in the same group?
const appInfo = appGroup?.apps.find(app => app.isRunning);
const recordingStatus: RecordingStatus = {
id: recordingId++,
status: 'recording',
startTime: Date.now(),
app: appInfo,
appGroup,
};
recordingStatus$.next(recordingStatus);
return recordingStatus;
}
export function pauseRecording() {
const recordingStatus = recordingStatus$.value;
if (!recordingStatus) {
return;
}
recordingStatus$.next({
...recordingStatus,
status: 'paused',
});
}
export function resumeRecording() {
const recordingStatus = recordingStatus$.value;
if (!recordingStatus) {
return;
}
recordingStatus$.next({
...recordingStatus,
status: 'recording',
});
}
export function stopRecording() {
const recordingStatus = recordingStatus$.value;
if (!recordingStatus) {
return;
}
// do not remove the last recordingStatus from recordingStatus$
recordingStatus$.next({
...recordingStatus,
status: 'stopped',
});
// bring up the window
getMainWindow()
.then(mainWindow => {
if (mainWindow) {
mainWindow.show();
}
})
.catch(err => {
logger.error('failed to bring up the window', err);
});
}
export const recordingHandlers = {
saveRecording: async (_, id: number) => {
return saveRecording(id);
},
} satisfies NamespaceHandlers;
export const recordingEvents = {
onRecordingStatusChanged: (fn: (status: RecordingStatus | null) => void) => {
const sub = recordingStatus$.subscribe(fn);
return () => {
try {
sub.unsubscribe();
} catch {
// ignore unsubscribe error
}
};
},
};

View File

@@ -0,0 +1,38 @@
import type { AudioTapStream, TappableApplication } from '@affine/native';
export interface TappableAppInfo {
rawInstance: TappableApplication;
isRunning: boolean;
processId: number;
processGroupId: number;
bundleIdentifier: string;
name: string;
}
export interface AppGroupInfo {
processGroupId: number;
apps: TappableAppInfo[];
name: string;
bundleIdentifier: string;
icon: Buffer | undefined;
isRunning: boolean;
}
export interface Recording {
id: number;
// the app may not be available if the user choose to record system audio
app?: TappableAppInfo;
appGroup?: AppGroupInfo;
// the raw audio buffers that are already accumulated
buffers: Float32Array[];
stream: AudioTapStream;
startTime: number;
}
export interface RecordingStatus {
id: number; // corresponds to the recording id
status: 'recording' | 'paused' | 'stopped';
app?: TappableAppInfo;
appGroup?: AppGroupInfo;
startTime: number;
}

View File

@@ -0,0 +1,13 @@
import { join } from 'node:path';
import { resourcesPath } from '../../shared/utils';
export const icons = {
record: join(resourcesPath, 'icons/waveform.png'),
recording: join(resourcesPath, 'icons/waveform-recording.png'),
tray: join(resourcesPath, 'icons/tray-icon.png'),
journal: join(resourcesPath, 'icons/journal-today.png'),
page: join(resourcesPath, 'icons/doc-page.png'),
edgeless: join(resourcesPath, 'icons/doc-edgeless.png'),
monitor: join(resourcesPath, 'icons/monitor.png'),
};

View File

@@ -1,25 +1,34 @@
import { join } from 'node:path';
import {
app,
Menu,
MenuItem,
type MenuItemConstructorOptions,
type NativeImage,
nativeImage,
Tray,
} from 'electron';
import { isMacOS, resourcesPath } from '../../shared/utils';
import { isMacOS } from '../../shared/utils';
import { applicationMenuSubjects } from '../application-menu';
import { beforeAppQuit } from '../cleanup';
import { appGroups$ } from '../recording';
import { logger } from '../logger';
import {
appGroups$,
pauseRecording,
recordingStatus$,
resumeRecording,
startRecording,
stopRecording,
} from '../recording';
import { getMainWindow } from '../windows-manager';
import { icons } from './icons';
export interface TrayMenuConfigItem {
label: string;
click?: () => void;
icon?: NativeImage | string | Buffer;
disabled?: boolean;
submenu?: TrayMenuConfig;
}
export type TrayMenuConfig = Array<TrayMenuConfigItem | 'separator'>;
@@ -35,7 +44,35 @@ function showMainWindow() {
.then(w => {
w.show();
})
.catch(err => console.error(err));
.catch(err => logger.error('Failed to show main window:', err));
}
function buildMenuConfig(config: TrayMenuConfig): MenuItemConstructorOptions[] {
const menuConfig: MenuItemConstructorOptions[] = [];
config.forEach(item => {
if (item === 'separator') {
menuConfig.push({ type: 'separator' });
} else {
const { icon, disabled, submenu, ...rest } = item;
let nativeIcon: NativeImage | undefined;
if (typeof icon === 'string') {
nativeIcon = nativeImage.createFromPath(icon);
} else if (Buffer.isBuffer(icon)) {
nativeIcon = nativeImage.createFromBuffer(icon);
}
if (nativeIcon) {
nativeIcon = nativeIcon.resize({ width: 20, height: 20 });
}
const submenuConfig = submenu ? buildMenuConfig(submenu) : undefined;
menuConfig.push({
...rest,
enabled: !disabled,
icon: nativeIcon,
submenu: submenuConfig,
});
}
});
return menuConfig;
}
class TrayState {
@@ -43,7 +80,7 @@ class TrayState {
// tray's icon
icon: NativeImage = nativeImage
.createFromPath(join(resourcesPath, 'icons/tray-icon.png'))
.createFromPath(icons.tray)
.resize({ width: 16, height: 16 });
// tray's tooltip
@@ -60,24 +97,27 @@ class TrayState {
getConfig: () => [
{
label: 'Open Journal',
icon: join(resourcesPath, 'icons/journal-today.png'),
icon: icons.journal,
click: () => {
logger.info('User action: Open Journal');
showMainWindow();
applicationMenuSubjects.openJournal$.next();
},
},
{
label: 'New Page',
icon: join(resourcesPath, 'icons/doc-page.png'),
icon: icons.page,
click: () => {
logger.info('User action: New Page');
showMainWindow();
applicationMenuSubjects.newPageAction$.next('page');
},
},
{
label: 'New Edgeless',
icon: join(resourcesPath, 'icons/doc-edgeless.png'),
icon: icons.edgeless,
click: () => {
logger.info('User action: New Edgeless');
showMainWindow();
applicationMenuSubjects.newPageAction$.next('edgeless');
},
@@ -89,20 +129,78 @@ class TrayState {
getRecordingMenuProvider(): TrayMenuProvider {
const appGroups = appGroups$.value;
const runningAppGroups = appGroups.filter(appGroup => appGroup.isRunning);
const recordingStatus = recordingStatus$.value;
if (!recordingStatus || recordingStatus?.status === 'stopped') {
return {
key: 'recording',
getConfig: () => [
{
label: 'Start Recording Meeting',
icon: icons.record,
submenu: [
{
label: 'System audio (all audio will be recorded)',
icon: icons.monitor,
click: () => {
logger.info(
'User action: Start Recording Meeting (System audio)'
);
startRecording();
},
},
...runningAppGroups.map(appGroup => ({
label: appGroup.name,
icon: appGroup.icon || undefined,
click: () => {
logger.info(
`User action: Start Recording Meeting (${appGroup.name})`
);
startRecording(appGroup);
},
})),
],
},
],
};
}
const recordingLabel = recordingStatus.appGroup?.name
? `Recording (${recordingStatus.appGroup?.name})`
: 'Recording';
// recording is either started or paused
return {
key: 'recording',
getConfig: () => [
{
label: 'Start Recording Meeting',
label: recordingLabel,
icon: icons.recording,
disabled: true,
},
...runningAppGroups.map(appGroup => ({
label: appGroup.name,
icon: appGroup.icon || undefined,
recordingStatus.status === 'paused'
? {
label: 'Resume',
click: () => {
logger.info('User action: Resume Recording');
resumeRecording();
},
}
: {
label: 'Pause',
click: () => {
logger.info('User action: Pause Recording');
pauseRecording();
},
},
{
label: 'Stop',
click: () => {
console.log(appGroup);
logger.info('User action: Stop Recording');
stopRecording();
},
})),
},
],
};
}
@@ -114,12 +212,13 @@ class TrayState {
{
label: 'Open AFFiNE',
click: () => {
logger.info('User action: Open AFFiNE');
getMainWindow()
.then(w => {
w.show();
})
.catch(err => {
console.error(err);
logger.error('Failed to open AFFiNE:', err);
});
},
},
@@ -127,6 +226,7 @@ class TrayState {
{
label: 'Quit AFFiNE Completely...',
click: () => {
logger.info('User action: Quit AFFiNE Completely');
app.quit();
},
},
@@ -137,32 +237,9 @@ class TrayState {
buildMenu(providers: TrayMenuProvider[]) {
const menu = new Menu();
providers.forEach((provider, index) => {
provider.getConfig().forEach(item => {
if (item === 'separator') {
menu.append(new MenuItem({ type: 'separator' }));
} else {
const { icon, disabled, ...rest } = item;
let nativeIcon: NativeImage | undefined;
if (typeof icon === 'string') {
nativeIcon = nativeImage.createFromPath(icon);
} else if (Buffer.isBuffer(icon)) {
try {
nativeIcon = nativeImage.createFromBuffer(icon);
} catch (error) {
console.error('Failed to create icon from buffer', error);
}
}
if (nativeIcon) {
nativeIcon = nativeIcon.resize({ width: 20, height: 20 });
}
menu.append(
new MenuItem({
...rest,
enabled: !disabled,
icon: nativeIcon,
})
);
}
const config = provider.getConfig();
buildMenuConfig(config).forEach(item => {
menu.append(new MenuItem(item));
});
if (index !== providers.length - 1) {
menu.append(new MenuItem({ type: 'separator' }));
@@ -176,15 +253,22 @@ class TrayState {
this.tray = new Tray(this.icon);
this.tray.setToolTip(this.tooltip);
const clickHandler = () => {
logger.debug('User clicked on tray icon');
this.update();
if (!isMacOS()) {
this.tray?.popUpContextMenu();
}
};
this.tray.on('click', clickHandler);
const appGroupsSubscription = appGroups$.subscribe(() => {
logger.debug('App groups updated, refreshing tray menu');
this.update();
});
beforeAppQuit(() => {
logger.info('Cleaning up tray before app quit');
this.tray?.off('click', clickHandler);
this.tray?.destroy();
appGroupsSubscription.unsubscribe();
});
}
@@ -199,6 +283,7 @@ class TrayState {
}
init() {
logger.info('Initializing tray');
this.update();
}
}

View File

@@ -17,6 +17,13 @@ export interface HelperToRenderer {
// helper <-> main
export interface HelperToMain {
getMeta: () => ExposedMeta;
encodeToMp3: (
samples: Float32Array,
opts?: {
channels?: number;
sampleRate?: number;
}
) => Uint8Array;
}
export type MainToHelper = Pick<