feat(electron): recording popups (#11016)

Added a recording popup UI for the audio recording feature in the desktop app, improving the user experience when capturing audio from applications.

### What changed?

- Created a new popup window system for displaying recording controls
- Added a dedicated recording UI with start/stop controls and status indicators
- Moved audio encoding logic from the main app to a dedicated module
- Implemented smooth animations for popup appearance/disappearance
- Updated the recording workflow to show visual feedback during recording process
- Added internationalization support for recording-related text
- Modified the recording status flow to include new states: new, recording, stopped, ready

fix AF-2340
This commit is contained in:
pengx17
2025-03-26 04:53:43 +00:00
parent 96e83a2141
commit 61c0d01da3
38 changed files with 3611 additions and 383 deletions
@@ -5,134 +5,21 @@ import { AudioAttachmentService } from '@affine/core/modules/media/services/audi
import { WorkbenchService } from '@affine/core/modules/workbench';
import { DebugLogger } from '@affine/debug';
import { apis, events } from '@affine/electron-api';
import { i18nTime } from '@affine/i18n';
import type { AttachmentBlockModel } from '@blocksuite/affine/model';
import { Text } from '@blocksuite/affine/store';
import type { BlobEngine } from '@blocksuite/affine/sync';
import type { FrameworkProvider } from '@toeverything/infra';
import { ArrayBufferTarget, Muxer } from 'webm-muxer';
import { getCurrentWorkspace } from './utils';
const logger = new DebugLogger('electron-renderer:recording');
/**
* Encodes raw audio data to Opus in WebM container.
*/
async function encodeRawBufferToOpus({
filepath,
sampleRate,
numberOfChannels,
}: {
filepath: string;
sampleRate: number;
numberOfChannels: number;
}): Promise<Uint8Array> {
// Use streams to process audio data incrementally
const response = await fetch(new URL(filepath, location.origin));
if (!response.body) {
throw new Error('Response body is null');
}
// Setup Opus encoder
const encodedChunks: EncodedAudioChunk[] = [];
const encoder = new AudioEncoder({
output: chunk => {
encodedChunks.push(chunk);
},
error: err => {
throw new Error(`Encoding error: ${err}`);
},
});
// Configure Opus encoder
encoder.configure({
codec: 'opus',
sampleRate: sampleRate,
numberOfChannels: numberOfChannels,
bitrate: 96000, // 96 kbps is good for stereo audio
});
// Process the stream
const reader = response.body.getReader();
let offset = 0;
const CHUNK_SIZE = numberOfChannels * 1024; // Process 1024 samples per channel at a time
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
// Convert the chunk to Float32Array
const float32Data = new Float32Array(value.buffer);
// Process in smaller chunks to avoid large frames
for (let i = 0; i < float32Data.length; i += CHUNK_SIZE) {
const chunkSize = Math.min(CHUNK_SIZE, float32Data.length - i);
const chunk = float32Data.subarray(i, i + chunkSize);
// Create and encode frame
const frame = new AudioData({
format: 'f32',
sampleRate: sampleRate,
numberOfFrames: chunk.length / numberOfChannels,
numberOfChannels: numberOfChannels,
timestamp: (offset * 1000000) / sampleRate, // timestamp in microseconds
data: chunk,
});
encoder.encode(frame);
frame.close();
offset += chunk.length / numberOfChannels;
}
}
} finally {
await encoder.flush();
encoder.close();
}
if (encodedChunks.length === 0) {
throw new Error('No chunks were produced during encoding');
}
// Initialize WebM muxer
const target = new ArrayBufferTarget();
const muxer = new Muxer({
target,
audio: {
codec: 'A_OPUS',
sampleRate: sampleRate,
numberOfChannels: numberOfChannels,
},
});
// Add all chunks to the muxer
for (const chunk of encodedChunks) {
muxer.addAudioChunk(chunk, {});
}
// Finalize and get WebM container
muxer.finalize();
const { buffer: webmBuffer } = target;
return new Uint8Array(webmBuffer);
}
async function saveRecordingBlob(
blobEngine: BlobEngine,
recording: {
id: number;
filepath: string;
sampleRate: number;
numberOfChannels: number;
}
) {
logger.debug('Saving recording', recording.id);
const opusBuffer = await encodeRawBufferToOpus({
filepath: recording.filepath,
sampleRate: recording.sampleRate,
numberOfChannels: recording.numberOfChannels,
});
async function saveRecordingBlob(blobEngine: BlobEngine, filepath: string) {
logger.debug('Saving recording', filepath);
const opusBuffer = await fetch(new URL(filepath, location.origin)).then(res =>
res.arrayBuffer()
);
const blob = new Blob([opusBuffer], {
type: 'audio/webm',
});
@@ -144,7 +31,7 @@ async function saveRecordingBlob(
export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
events?.recording.onRecordingStatusChanged(status => {
(async () => {
if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') {
if ((await apis?.ui.isActiveTab()) && status?.status === 'ready') {
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
if (!currentWorkspace) {
return;
@@ -155,30 +42,28 @@ export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
const docsService = workspace.scope.get(DocsService);
const editorSetting = editorSettingService.editorSetting;
const timestamp = i18nTime(status.startTime, {
absolute: {
accuracy: 'minute',
noYear: true,
},
});
const docProps: DocProps = {
note: editorSetting.get('affine:note'),
page: {
title: new Text(
'Recording ' +
(status.appGroup?.name ?? 'System Audio') +
(status.appName ?? 'System Audio') +
' ' +
new Date(status.startTime).toISOString()
timestamp
),
},
onStoreLoad: (doc, { noteId }) => {
(async () => {
const recording = await apis?.recording.getRecording(status.id);
if (!recording) {
logger.error('Failed to save recording');
return;
}
// name + timestamp(readable) + extension
const attachmentName =
(status.appGroup?.name ?? 'System Audio') +
' ' +
new Date(status.startTime).toISOString() +
'.webm';
(status.appName ?? 'System Audio') + ' ' + timestamp + '.webm';
// add size and sourceId to the attachment later
const attachmentId = doc.addBlock(
@@ -193,11 +78,11 @@ export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
const model = doc.getBlock(attachmentId)
?.model as AttachmentBlockModel;
if (model) {
if (model && status.filepath) {
// it takes a while to save the blob, so we show the attachment first
const { blobId, blob } = await saveRecordingBlob(
doc.workspace.blobSync,
recording
status.filepath
);
model.props.size = blob.size;
@@ -0,0 +1,16 @@
import { globalStyle, style } from '@vanilla-extract/css';
globalStyle('html', {
backgroundColor: 'transparent',
userSelect: 'none',
});
globalStyle('body', {
backgroundColor: 'transparent',
});
export const root = style({
backgroundColor: 'transparent',
width: '100%',
height: '100%',
});
@@ -0,0 +1,37 @@
import { ThemeProvider } from '@affine/core/components/theme-provider';
import { configureDesktopApiModule } from '@affine/core/modules/desktop-api';
import { configureI18nModule, I18nProvider } from '@affine/core/modules/i18n';
import {
configureElectronStateStorageImpls,
configureStorageModule,
} from '@affine/core/modules/storage';
import { configureEssentialThemeModule } from '@affine/core/modules/theme';
import { appInfo } from '@affine/electron-api';
import { Framework, FrameworkRoot } from '@toeverything/infra';
import * as styles from './app.css';
import { Recording } from './recording';
const framework = new Framework();
configureI18nModule(framework);
configureEssentialThemeModule(framework);
configureStorageModule(framework);
configureElectronStateStorageImpls(framework);
configureDesktopApiModule(framework);
const frameworkProvider = framework.provider();
const mode = appInfo?.windowName as 'notification' | 'recording';
export function App() {
return (
<FrameworkRoot framework={frameworkProvider}>
<ThemeProvider>
<I18nProvider>
<div className={styles.root}>
{mode === 'recording' && <Recording />}
</div>
</I18nProvider>
</ThemeProvider>
</FrameworkRoot>
);
}
@@ -0,0 +1,24 @@
import './setup';
import { StrictMode } from 'react';
import { createRoot } from 'react-dom/client';
import { App } from './app';
function main() {
mountApp();
}
function mountApp() {
const root = document.getElementById('app');
if (!root) {
throw new Error('Root element not found');
}
createRoot(root).render(
<StrictMode>
<App />
</StrictMode>
);
}
main();
@@ -0,0 +1,100 @@
import { ArrayBufferTarget, Muxer } from 'webm-muxer';
/**
* Encodes raw audio data to Opus in WebM container.
*/
export async function encodeRawBufferToOpus({
filepath,
sampleRate,
numberOfChannels,
}: {
filepath: string;
sampleRate: number;
numberOfChannels: number;
}): Promise<Uint8Array> {
// Use streams to process audio data incrementally
const response = await fetch(new URL(filepath, location.origin));
if (!response.body) {
throw new Error('Response body is null');
}
// Setup Opus encoder
const encodedChunks: EncodedAudioChunk[] = [];
const encoder = new AudioEncoder({
output: chunk => {
encodedChunks.push(chunk);
},
error: err => {
throw new Error(`Encoding error: ${err}`);
},
});
// Configure Opus encoder
encoder.configure({
codec: 'opus',
sampleRate: sampleRate,
numberOfChannels: numberOfChannels,
bitrate: 128000,
});
// Process the stream
const reader = response.body.getReader();
let offset = 0;
const CHUNK_SIZE = numberOfChannels * 1024; // Process 1024 samples per channel at a time
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
// Convert the chunk to Float32Array
const float32Data = new Float32Array(value.buffer);
// Process in smaller chunks to avoid large frames
for (let i = 0; i < float32Data.length; i += CHUNK_SIZE) {
const chunkSize = Math.min(CHUNK_SIZE, float32Data.length - i);
const chunk = float32Data.subarray(i, i + chunkSize);
// Create and encode frame
const frame = new AudioData({
format: 'f32',
sampleRate: sampleRate,
numberOfFrames: chunk.length / numberOfChannels,
numberOfChannels: numberOfChannels,
timestamp: (offset * 1000000) / sampleRate, // timestamp in microseconds
data: chunk,
});
encoder.encode(frame);
frame.close();
offset += chunk.length / numberOfChannels;
}
}
} finally {
await encoder.flush();
encoder.close();
}
// Initialize WebM muxer
const target = new ArrayBufferTarget();
const muxer = new Muxer({
target,
audio: {
codec: 'A_OPUS',
sampleRate: sampleRate,
numberOfChannels: numberOfChannels,
},
});
// Add all chunks to the muxer
for (const chunk of encodedChunks) {
muxer.addAudioChunk(chunk, {});
}
// Finalize and get WebM container
muxer.finalize();
const { buffer: webmBuffer } = target;
return new Uint8Array(webmBuffer);
}
@@ -0,0 +1,183 @@
import { Button } from '@affine/component';
import { useAsyncCallback } from '@affine/core/components/hooks/affine-async-hooks';
import { appIconMap } from '@affine/core/utils';
import { apis, events } from '@affine/electron-api';
import { useI18n } from '@affine/i18n';
import { useEffect, useMemo, useState } from 'react';
import { encodeRawBufferToOpus } from './encode';
import * as styles from './styles.css';
type Status = {
id: number;
status: 'new' | 'recording' | 'paused' | 'stopped' | 'ready';
appName?: string;
appGroupId?: number;
icon?: Buffer;
};
export const useRecordingStatus = () => {
const [status, setStatus] = useState<Status | null>(null);
useEffect(() => {
// Get initial status
apis?.recording
.getCurrentRecording()
.then(status => setStatus(status as Status))
.catch(console.error);
// Subscribe to status changes
const unsubscribe = events?.recording.onRecordingStatusChanged(status =>
setStatus(status as Status)
);
return () => {
unsubscribe?.();
};
}, []);
return status;
};
const appIcon = appIconMap[BUILD_CONFIG.appBuildType];
export function Recording() {
const status = useRecordingStatus();
const t = useI18n();
const textElement = useMemo(() => {
if (!status) {
return null;
}
if (status.status === 'new') {
return t['com.affine.recording.new']();
} else if (status.status === 'ready') {
return t['com.affine.recording.ready']();
} else if (status.appName) {
return t['com.affine.recording.recording']({
appName: status.appName,
});
} else {
return t['com.affine.recording.recording.unnamed']();
}
}, [status, t]);
const handleDismiss = useAsyncCallback(async () => {
await apis?.popup?.dismissCurrentRecording();
}, []);
const handleStopRecording = useAsyncCallback(async () => {
if (!status) {
return;
}
await apis?.recording?.stopRecording(status.id);
}, [status]);
const handleProcessStoppedRecording = useAsyncCallback(async () => {
let id: number | undefined;
try {
const result = await apis?.recording?.getCurrentRecording();
if (!result) {
return;
}
id = result.id;
const { filepath, sampleRate, numberOfChannels } = result;
if (!filepath || !sampleRate || !numberOfChannels) {
return;
}
const [buffer] = await Promise.all([
encodeRawBufferToOpus({
filepath,
sampleRate,
numberOfChannels,
}),
new Promise<void>(resolve => {
setTimeout(() => {
resolve();
}, 1000); // wait at least 1 second for better user experience
}),
]);
await apis?.recording.readyRecording(result.id, buffer);
} catch (error) {
console.error('Failed to stop recording', error);
await apis?.popup?.dismissCurrentRecording();
if (id) {
await apis?.recording.removeRecording(id);
}
}
}, []);
useEffect(() => {
// allow processing stopped event in tray menu as well:
return events?.recording.onRecordingStatusChanged(status => {
if (status?.status === 'stopped') {
handleProcessStoppedRecording();
}
});
}, [handleProcessStoppedRecording]);
const handleStartRecording = useAsyncCallback(async () => {
if (!status) {
return;
}
await apis?.recording?.startRecording(status.appGroupId);
}, [status]);
const controlsElement = useMemo(() => {
if (!status) {
return null;
}
if (status.status === 'new') {
return (
<>
<Button variant="plain" onClick={handleDismiss}>
{t['com.affine.recording.dismiss']()}
</Button>
<Button
onClick={handleStartRecording}
variant="primary"
prefix={<div className={styles.recordingIcon} />}
>
{t['com.affine.recording.start']()}
</Button>
</>
);
} else if (status.status === 'recording') {
return (
<Button variant="error" onClick={handleStopRecording}>
{t['com.affine.recording.stop']()}
</Button>
);
} else if (status.status === 'stopped') {
return (
<Button
variant="error"
onClick={handleDismiss}
loading={true}
disabled
/>
);
} else if (status.status === 'ready') {
return (
<Button variant="primary" onClick={handleDismiss}>
{t['com.affine.recording.ready']()}
</Button>
);
}
return null;
}, [handleDismiss, handleStartRecording, handleStopRecording, status, t]);
if (!status) {
return null;
}
return (
<div className={styles.root}>
<img className={styles.affineIcon} src={appIcon} alt="AFFiNE" />
<div className={styles.text}>{textElement}</div>
<div className={styles.controls}>{controlsElement}</div>
</div>
);
}
@@ -0,0 +1,38 @@
import { cssVar } from '@toeverything/theme';
import { cssVarV2 } from '@toeverything/theme/v2';
import { style } from '@vanilla-extract/css';
export const root = style({
width: '100%',
height: '100%',
padding: 4,
display: 'flex',
gap: 4,
alignItems: 'center',
});
export const affineIcon = style({
width: 28,
height: 28,
});
export const recordingIcon = style({
width: 8,
height: 8,
borderRadius: '50%',
backgroundColor: cssVarV2('layer/pureWhite'),
});
export const text = style({
fontSize: cssVar('fontSm'),
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
fontWeight: 600,
flex: 1,
});
export const controls = style({
display: 'flex',
gap: 2,
});
@@ -0,0 +1,2 @@
import '@affine/core/bootstrap/electron';
import '@affine/component/theme';
@@ -0,0 +1,18 @@
import { useEffect, useState } from 'react';
export const useBlobUrl = (buffer?: Buffer) => {
const [blobUrl, setBlobUrl] = useState<string | null>(null);
useEffect(() => {
if (!buffer) {
return;
}
const url = URL.createObjectURL(new Blob([buffer]));
setBlobUrl(url);
return () => {
URL.revokeObjectURL(url);
};
}, [buffer]);
return blobUrl;
};
@@ -3,5 +3,6 @@ export const config = {
app: './src/app/index.tsx',
shell: './src/shell/index.tsx',
backgroundWorker: './src/background-worker/index.ts',
popup: './src/popup/index.tsx',
},
};
@@ -8,13 +8,14 @@ import {
addTab,
initAndShowMainWindow,
reloadView,
showDevTools,
showMainWindow,
switchTab,
switchToNextTab,
switchToPreviousTab,
undoCloseTab,
WebContentViewsManager,
} from '../windows-manager';
import { popupManager } from '../windows-manager/popup';
import { WorkerManager } from '../worker/pool';
import { applicationMenuSubjects } from './subject';
@@ -111,21 +112,55 @@ export function createApplicationMenu() {
label: 'Open devtools',
accelerator: isMac ? 'Cmd+Option+I' : 'Ctrl+Shift+I',
click: () => {
showDevTools();
},
},
{
label: 'Open worker devtools',
click: () => {
const workerContents = Array.from(
WorkerManager.instance.workers.values()
).map(
worker => [worker.key, worker.browserWindow.webContents] as const
);
const tabs = Array.from(
WebContentViewsManager.instance.tabViewsMap
).map(view => {
const isActive = WebContentViewsManager.instance.isActiveTab(
view[0]
);
return [
view[0] + (isActive ? ' (active)' : ''),
view[1].webContents,
] as const;
});
const popups = Array.from(popupManager.popupWindows$.value.values())
.filter(popup => popup.browserWindow)
.map(popup => {
// oxlint-disable-next-line no-non-null-assertion
return [popup.type, popup.browserWindow!.webContents] as const;
});
const allWebContents = [
['tabs', tabs],
['workers', workerContents],
['popups', popups],
] as const;
Menu.buildFromTemplate(
Array.from(WorkerManager.instance.workers.values()).map(item => ({
label: `${item.key}`,
click: () => {
item.browserWindow.webContents.openDevTools({
mode: 'undocked',
});
},
}))
allWebContents.flatMap(([type, contents]) => {
return [
{
label: type,
enabled: false,
},
...contents.map(([id, webContents]) => ({
label: id,
click: () => {
webContents.openDevTools({
mode: 'undocked',
});
},
})),
{ type: 'separator' },
];
})
).popup();
},
},
@@ -3,3 +3,4 @@ export const onboardingViewUrl = `${mainWindowOrigin}${mainWindowOrigin.endsWith
export const shellViewUrl = `${mainWindowOrigin}${mainWindowOrigin.endsWith('/') ? '' : '/'}shell.html`;
export const backgroundWorkerViewUrl = `${mainWindowOrigin}${mainWindowOrigin.endsWith('/') ? '' : '/'}background-worker.html`;
export const customThemeViewUrl = `${mainWindowOrigin}${mainWindowOrigin.endsWith('/') ? '' : '/'}theme-editor`;
export const popupViewUrl = `${mainWindowOrigin}${mainWindowOrigin.endsWith('/') ? '' : '/'}popup.html`;
@@ -8,6 +8,7 @@ import { recordingEvents } from './recording';
import { sharedStorageEvents } from './shared-storage';
import { uiEvents } from './ui/events';
import { updaterEvents } from './updater/event';
import { popupEvents } from './windows-manager/popup';
export const allEvents = {
applicationMenu: applicationMenuEvents,
@@ -15,6 +16,7 @@ export const allEvents = {
ui: uiEvents,
sharedStorage: sharedStorageEvents,
recording: recordingEvents,
popup: popupEvents,
};
function getActiveWindows() {
@@ -9,6 +9,7 @@ import { recordingHandlers } from './recording';
import { sharedStorageHandlers } from './shared-storage';
import { uiHandlers } from './ui/handlers';
import { updaterHandlers } from './updater';
import { popupHandlers } from './windows-manager/popup';
import { workerHandlers } from './worker/handlers';
export const debugHandlers = {
@@ -31,6 +32,7 @@ export const allHandlers = {
sharedStorage: sharedStorageHandlers,
worker: workerHandlers,
recording: recordingHandlers,
popup: popupHandlers,
};
export const registerHandlers = () => {
@@ -79,7 +79,11 @@ async function handleFileRequest(request: Request) {
filepath = decodeURIComponent(urlObject.pathname);
// security check if the filepath is within app.getPath('sessionData')
const sessionDataPath = app.getPath('sessionData');
if (!filepath.startsWith(sessionDataPath)) {
const tempPath = app.getPath('temp');
if (
!filepath.startsWith(sessionDataPath) &&
!filepath.startsWith(tempPath)
) {
throw new Error('Invalid filepath');
}
}
@@ -1,16 +1,25 @@
import path from 'node:path';
import { ShareableContent } from '@affine/native';
import { app, nativeImage, Notification } from 'electron';
import { app } from 'electron';
import fs from 'fs-extra';
import { debounce } from 'lodash-es';
import { BehaviorSubject, distinctUntilChanged, groupBy, mergeMap } from 'rxjs';
import {
BehaviorSubject,
distinctUntilChanged,
groupBy,
interval,
mergeMap,
Subject,
throttleTime,
} from 'rxjs';
import { isMacOS } from '../../shared/utils';
import { isMacOS, shallowEqual } from '../../shared/utils';
import { beforeAppQuit } from '../cleanup';
import { logger } from '../logger';
import type { NamespaceHandlers } from '../type';
import { getMainWindow } from '../windows-manager';
import { popupManager } from '../windows-manager/popup';
import { recordingStateMachine } from './state-machine';
import type {
AppGroupInfo,
Recording,
@@ -20,9 +29,10 @@ import type {
const subscribers: Subscriber[] = [];
// adhoc recordings are saved in the temp directory
const SAVED_RECORDINGS_DIR = path.join(
app.getPath('sessionData'),
'recordings'
app.getPath('temp'),
'affine-recordings'
);
beforeAppQuit(() => {
@@ -40,14 +50,15 @@ let shareableContent: ShareableContent | null = null;
export const applications$ = new BehaviorSubject<TappableAppInfo[]>([]);
export const appGroups$ = new BehaviorSubject<AppGroupInfo[]>([]);
export const updateApplicationsPing$ = new Subject<number>();
// recording id -> recording
// recordings will be saved in memory before consumed and created as an audio block to user's doc
const recordings = new Map<number, Recording>();
// there should be only one active recording at a time
export const recordingStatus$ = new BehaviorSubject<RecordingStatus | null>(
null
);
// We'll now use recordingStateMachine.status$ instead of our own BehaviorSubject
export const recordingStatus$ = recordingStateMachine.status$;
function createAppGroup(processGroupId: number): AppGroupInfo | undefined {
const groupProcess =
@@ -113,37 +124,45 @@ function setupNewRunningAppGroup() {
)
)
);
appGroups$.value.forEach(group => {
const recordingStatus = recordingStatus$.value;
if (
group.isRunning &&
(!recordingStatus || recordingStatus.status === 'new')
) {
newRecording(group);
}
});
subscribers.push(
appGroupRunningChanged$.subscribe(currentGroup => {
logger.info(
'appGroupRunningChanged',
currentGroup.bundleIdentifier,
currentGroup.isRunning
);
const recordingStatus = recordingStatus$.value;
if (currentGroup.isRunning) {
// TODO(@pengx17): stub impl. will be replaced with a real one later
const notification = new Notification({
icon: currentGroup.icon
? nativeImage.createFromBuffer(currentGroup.icon)
: undefined,
title: 'Recording Meeting',
body: `Recording meeting with ${currentGroup.name}`,
actions: [
{
type: 'button',
text: 'Start',
},
],
});
notification.on('action', () => {
startRecording(currentGroup);
});
notification.show();
} else {
// if the group is not running, we should stop the recording (if it is recording)
// when the app is running and there is no active recording popup
// we should show a new recording popup
if (
recordingStatus$.value?.status === 'recording' &&
recordingStatus$.value?.appGroup?.processGroupId ===
currentGroup.processGroupId
!recordingStatus ||
recordingStatus.status === 'new' ||
recordingStatus.status === 'ready'
) {
stopRecording().catch(err => {
logger.error('failed to stop recording', err);
});
newRecording(currentGroup);
}
} else {
// when displaying in "new" state but the app is not running any more
// we should remove the recording
if (
recordingStatus?.status === 'new' &&
currentGroup.bundleIdentifier ===
recordingStatus.appGroup?.bundleIdentifier
) {
removeRecording(recordingStatus.id);
}
}
})
@@ -212,23 +231,60 @@ export async function getRecording(id: number) {
};
}
// recording popup status
// new: recording is started, popup is shown
// recording: recording is started, popup is shown
// stopped: recording is stopped, popup showing processing status
// ready: recording is ready, show "open app" button
// null: hide popup
function setupRecordingListeners() {
subscribers.push(
recordingStatus$.pipe(distinctUntilChanged()).subscribe(status => {
if (status?.status === 'recording') {
let recording = recordings.get(status.id);
// create a recording if not exists
if (!recording) {
recording = createRecording(status);
recordings.set(status.id, recording);
recordingStatus$
.pipe(distinctUntilChanged(shallowEqual))
.subscribe(status => {
const popup = popupManager.get('recording');
if (status && !popup.showing) {
popup.show().catch(err => {
logger.error('failed to show recording popup', err);
});
}
} else if (status?.status === 'stopped') {
const recording = recordings.get(status.id);
if (recording) {
recording.stream.stop();
if (status?.status === 'recording') {
let recording = recordings.get(status.id);
// create a recording if not exists
if (!recording) {
recording = createRecording(status);
recordings.set(status.id, recording);
}
} else if (status?.status === 'stopped') {
const recording = recordings.get(status.id);
if (recording) {
recording.stream.stop();
}
} else if (status?.status === 'ready') {
// show the popup for 10s
setTimeout(() => {
// check again if current status is still ready
if (
recordingStatus$.value?.status === 'ready' &&
recordingStatus$.value.id === status.id
) {
popup.hide().catch(err => {
logger.error('failed to hide recording popup', err);
});
}
}, 10_000);
} else if (!status) {
// status is removed, we should hide the popup
popupManager
.get('recording')
.hide()
.catch(err => {
logger.error('failed to hide recording popup', err);
});
}
}
})
})
);
}
@@ -236,7 +292,6 @@ function getAllApps(): TappableAppInfo[] {
if (!shareableContent) {
return [];
}
const apps = shareableContent.applications().map(app => {
try {
return {
@@ -259,7 +314,6 @@ function getAllApps(): TappableAppInfo[] {
!v.bundleIdentifier.startsWith('com.apple') &&
v.processId !== process.pid
);
return filteredApps;
}
@@ -270,9 +324,17 @@ type Subscriber = {
function setupMediaListeners() {
applications$.next(getAllApps());
subscribers.push(
interval(3000).subscribe(() => {
updateApplicationsPing$.next(Date.now());
}),
ShareableContent.onApplicationListChanged(() => {
applications$.next(getAllApps());
})
updateApplicationsPing$.next(Date.now());
}),
updateApplicationsPing$
.pipe(distinctUntilChanged(), throttleTime(3000))
.subscribe(() => {
applications$.next(getAllApps());
})
);
let appStateSubscribers: Subscriber[] = [];
@@ -291,12 +353,9 @@ function setupMediaListeners() {
apps.forEach(app => {
try {
const tappableApp = app.rawInstance;
const debouncedAppStateChanged = debounce(() => {
applications$.next(getAllApps());
}, 100);
_appStateSubscribers.push(
ShareableContent.onAppStateChanged(tappableApp, () => {
debouncedAppStateChanged();
updateApplicationsPing$.next(Date.now());
})
);
} catch (error) {
@@ -339,82 +398,104 @@ export function setupRecording() {
setupRecordingListeners();
}
let recordingId = 0;
function normalizeAppGroupInfo(
appGroup?: AppGroupInfo | number
): AppGroupInfo | undefined {
return typeof appGroup === 'number'
? appGroups$.value.find(group => group.processGroupId === appGroup)
: appGroup;
}
export function newRecording(
appGroup?: AppGroupInfo | number
): RecordingStatus | null {
if (!shareableContent) {
return null; // likely called on unsupported platform
}
return recordingStateMachine.dispatch({
type: 'NEW_RECORDING',
appGroup: normalizeAppGroupInfo(appGroup),
});
}
export function startRecording(
appGroup?: AppGroupInfo
): RecordingStatus | undefined {
if (!shareableContent) {
return; // likely called on unsupported platform
}
// hmm, is it possible that there are multiple apps running (listening) in the same group?
const appInfo = appGroup?.apps.find(app => app.isRunning);
const recordingStatus: RecordingStatus = {
id: recordingId++,
status: 'recording',
startTime: Date.now(),
app: appInfo,
appGroup,
};
recordingStatus$.next(recordingStatus);
return recordingStatus;
}
export function pauseRecording() {
const recordingStatus = recordingStatus$.value;
if (!recordingStatus) {
return;
}
recordingStatus$.next({
...recordingStatus,
status: 'paused',
appGroup?: AppGroupInfo | number
): RecordingStatus | null {
return recordingStateMachine.dispatch({
type: 'START_RECORDING',
appGroup: normalizeAppGroupInfo(appGroup),
});
}
export function resumeRecording() {
const recordingStatus = recordingStatus$.value;
if (!recordingStatus) {
export function pauseRecording(id: number) {
return recordingStateMachine.dispatch({ type: 'PAUSE_RECORDING', id });
}
export function resumeRecording(id: number) {
return recordingStateMachine.dispatch({ type: 'RESUME_RECORDING', id });
}
export async function stopRecording(id: number) {
const recording = recordings.get(id);
if (!recording) {
logger.error(`Recording ${id} not found`);
return;
}
recordingStatus$.next({
...recordingStatus,
status: 'recording',
});
}
if (!recording.file.path) {
logger.error(`Recording ${id} has no file path`);
return;
}
const recordingStatus = recordingStateMachine.dispatch({
type: 'STOP_RECORDING',
id,
filepath: String(recording.file.path),
sampleRate: recording.stream.sampleRate,
numberOfChannels: recording.stream.channels,
});
export async function stopRecording() {
const recordingStatus = recordingStatus$.value;
if (!recordingStatus) {
logger.error('No recording status to stop');
return;
}
const recording = recordings.get(recordingStatus?.id);
if (!recording) {
logger.error(`Recording ${recordingStatus?.id} not found`);
return;
}
// do not remove the last recordingStatus from recordingStatus$
recordingStatus$.next({
...recordingStatus,
status: 'stopped',
});
const { file } = recording;
file.end();
// Wait for file to finish writing
await new Promise<void>(resolve => {
file.on('finish', () => {
resolve();
});
});
return serializeRecordingStatus(recordingStatus);
}
export async function readyRecording(id: number, buffer: Buffer) {
const recordingStatus = recordingStatus$.value;
const recording = recordings.get(id);
if (!recordingStatus || recordingStatus.id !== id || !recording) {
logger.error(`Recording ${id} not found`);
return;
}
const filepath = path.join(
SAVED_RECORDINGS_DIR,
`${recordingStatus.appGroup?.bundleIdentifier ?? 'unknown'}-${recordingStatus.id}-${recordingStatus.startTime}.webm`
);
await fs.writeFile(filepath, buffer);
// Update the status through the state machine
recordingStateMachine.dispatch({
type: 'SAVE_RECORDING',
id,
filepath,
});
// bring up the window
getMainWindow()
.then(mainWindow => {
@@ -427,24 +508,75 @@ export async function stopRecording() {
});
}
function removeRecording(id: number) {
recordings.delete(id);
recordingStateMachine.dispatch({ type: 'REMOVE_RECORDING', id });
}
export interface SerializedRecordingStatus {
id: number;
status: RecordingStatus['status'];
appName?: string;
// if there is no app group, it means the recording is for system audio
appGroupId?: number;
icon?: Buffer;
startTime: number;
filepath?: string;
sampleRate?: number;
numberOfChannels?: number;
}
function serializeRecordingStatus(
status: RecordingStatus
): SerializedRecordingStatus {
return {
id: status.id,
status: status.status,
appName: status.appGroup?.name,
appGroupId: status.appGroup?.processGroupId,
icon: status.appGroup?.icon,
startTime: status.startTime,
filepath: status.filepath,
sampleRate: status.sampleRate,
numberOfChannels: status.numberOfChannels,
};
}
export const recordingHandlers = {
getRecording: async (_, id: number) => {
return getRecording(id);
},
deleteCachedRecording: async (_, id: number) => {
const recording = recordings.get(id);
if (recording) {
recording.stream.stop();
recordings.delete(id);
await fs.unlink(recording.file.path);
}
return true;
getCurrentRecording: async () => {
// not all properties are serializable, so we need to return a subset of the status
return recordingStatus$.value
? serializeRecordingStatus(recordingStatus$.value)
: null;
},
startRecording: async (_, appGroup?: AppGroupInfo | number) => {
return startRecording(appGroup);
},
pauseRecording: async (_, id: number) => {
return pauseRecording(id);
},
stopRecording: async (_, id: number) => {
return stopRecording(id);
},
// save the encoded recording buffer to the file system
readyRecording: async (_, id: number, buffer: Uint8Array) => {
return readyRecording(id, Buffer.from(buffer));
},
removeRecording: async (_, id: number) => {
return removeRecording(id);
},
} satisfies NamespaceHandlers;
export const recordingEvents = {
onRecordingStatusChanged: (fn: (status: RecordingStatus | null) => void) => {
const sub = recordingStatus$.subscribe(fn);
onRecordingStatusChanged: (
fn: (status: SerializedRecordingStatus | null) => void
) => {
const sub = recordingStatus$.subscribe(status => {
fn(status ? serializeRecordingStatus(status) : null);
});
return () => {
try {
sub.unsubscribe();
@@ -0,0 +1,269 @@
import { BehaviorSubject } from 'rxjs';
import { shallowEqual } from '../../shared/utils';
import { logger } from '../logger';
import type { AppGroupInfo, RecordingStatus } from './types';
/**
* Possible states for a recording
*/
export type RecordingState =
| 'new'
| 'recording'
| 'paused'
| 'stopped'
| 'ready'
| 'inactive';
/**
* Recording state machine events
*/
export type RecordingEvent =
| { type: 'NEW_RECORDING'; appGroup?: AppGroupInfo }
| { type: 'START_RECORDING'; appGroup?: AppGroupInfo }
| { type: 'PAUSE_RECORDING'; id: number }
| { type: 'RESUME_RECORDING'; id: number }
| {
type: 'STOP_RECORDING';
id: number;
filepath: string;
sampleRate: number;
numberOfChannels: number;
}
| {
type: 'SAVE_RECORDING';
id: number;
filepath: string;
}
| { type: 'REMOVE_RECORDING'; id: number };
/**
* Recording State Machine
* Handles state transitions for the recording process
*/
export class RecordingStateMachine {
private recordingId = 0;
private readonly recordingStatus$ =
new BehaviorSubject<RecordingStatus | null>(null);
/**
* Get the current recording status
*/
get status(): RecordingStatus | null {
return this.recordingStatus$.value;
}
/**
* Get the BehaviorSubject for recording status
*/
get status$(): BehaviorSubject<RecordingStatus | null> {
return this.recordingStatus$;
}
/**
* Dispatch an event to the state machine
* @param event The event to dispatch
* @returns The new recording status after the event is processed
*/
dispatch(event: RecordingEvent): RecordingStatus | null {
const currentStatus = this.recordingStatus$.value;
let newStatus: RecordingStatus | null = null;
switch (event.type) {
case 'NEW_RECORDING':
newStatus = this.handleNewRecording(event.appGroup);
break;
case 'START_RECORDING':
newStatus = this.handleStartRecording(event.appGroup);
break;
case 'PAUSE_RECORDING':
newStatus = this.handlePauseRecording();
break;
case 'RESUME_RECORDING':
newStatus = this.handleResumeRecording();
break;
case 'STOP_RECORDING':
newStatus = this.handleStopRecording(
event.id,
event.filepath,
event.sampleRate,
event.numberOfChannels
);
break;
case 'SAVE_RECORDING':
newStatus = this.handleSaveRecording(event.id, event.filepath);
break;
case 'REMOVE_RECORDING':
this.handleRemoveRecording(event.id);
newStatus = currentStatus?.id === event.id ? null : currentStatus;
break;
default:
logger.error('Unknown recording event type');
return currentStatus;
}
if (shallowEqual(newStatus, currentStatus)) {
return currentStatus;
}
this.recordingStatus$.next(newStatus);
return newStatus;
}
/**
* Handle the NEW_RECORDING event
*/
private handleNewRecording(appGroup?: AppGroupInfo): RecordingStatus {
const recordingStatus: RecordingStatus = {
id: this.recordingId++,
status: 'new',
startTime: Date.now(),
app: appGroup?.apps.find(app => app.isRunning),
appGroup,
};
return recordingStatus;
}
/**
* Handle the START_RECORDING event
*/
private handleStartRecording(appGroup?: AppGroupInfo): RecordingStatus {
const currentStatus = this.recordingStatus$.value;
if (
currentStatus?.status === 'recording' ||
currentStatus?.status === 'stopped'
) {
logger.error(
'Cannot start a new recording if there is already a recording'
);
return currentStatus;
}
if (
appGroup &&
currentStatus?.appGroup?.processGroupId === appGroup.processGroupId
) {
return {
...currentStatus,
status: 'recording',
};
} else {
const newStatus = this.handleNewRecording(appGroup);
return {
...newStatus,
status: 'recording',
};
}
}
/**
* Handle the PAUSE_RECORDING event
*/
private handlePauseRecording(): RecordingStatus | null {
const currentStatus = this.recordingStatus$.value;
if (!currentStatus) {
logger.error('No active recording to pause');
return null;
}
if (currentStatus.status !== 'recording') {
logger.error(`Cannot pause recording in ${currentStatus.status} state`);
return currentStatus;
}
return {
...currentStatus,
status: 'paused',
};
}
/**
* Handle the RESUME_RECORDING event
*/
private handleResumeRecording(): RecordingStatus | null {
const currentStatus = this.recordingStatus$.value;
if (!currentStatus) {
logger.error('No active recording to resume');
return null;
}
if (currentStatus.status !== 'paused') {
logger.error(`Cannot resume recording in ${currentStatus.status} state`);
return currentStatus;
}
return {
...currentStatus,
status: 'recording',
};
}
/**
* Handle the STOP_RECORDING event
*/
private handleStopRecording(
id: number,
filepath: string,
sampleRate: number,
numberOfChannels: number
): RecordingStatus | null {
const currentStatus = this.recordingStatus$.value;
if (!currentStatus || currentStatus.id !== id) {
logger.error(`Recording ${id} not found for stopping`);
return currentStatus;
}
if (
currentStatus.status !== 'recording' &&
currentStatus.status !== 'paused'
) {
logger.error(`Cannot stop recording in ${currentStatus.status} state`);
return currentStatus;
}
return {
...currentStatus,
status: 'stopped',
filepath,
sampleRate,
numberOfChannels,
};
}
/**
* Handle the SAVE_RECORDING event
*/
private handleSaveRecording(
id: number,
filepath: string
): RecordingStatus | null {
const currentStatus = this.recordingStatus$.value;
if (!currentStatus || currentStatus.id !== id) {
logger.error(`Recording ${id} not found for saving`);
return currentStatus;
}
return {
...currentStatus,
status: 'ready',
filepath,
};
}
/**
* Handle the REMOVE_RECORDING event
*/
private handleRemoveRecording(id: number): void {
// Actual recording removal logic would be handled by the caller
// This just ensures the state is updated correctly
logger.info(`Recording ${id} removed from state machine`);
}
}
// Create and export a singleton instance
export const recordingStateMachine = new RecordingStateMachine();
@@ -0,0 +1,88 @@
# Recording State Transitions
This document visualizes the possible state transitions in the recording system.
## States
The recording system has the following states:
- **inactive**: No active recording (null state)
- **new**: A new recording has been detected but not yet started
- **recording**: Audio is being recorded
- **paused**: Recording is temporarily paused
- **stopped**: Recording has been stopped and is processing
- **ready**: Recording is processed and ready for use
## Transitions
```
┌───────────┐ ┌───────┐
│ │ │ │
│ inactive │◀───────────────│ ready │
│ │ │ │
└─────┬─────┘ └───┬───┘
│ │
│ NEW_RECORDING │
▼ │
┌───────────┐ │
│ │ │
│ new │ │
│ │ │
└─────┬─────┘ │
│ │
│ START_RECORDING │
▼ │
┌───────────┐ │
│ │ STOP_RECORDING│
│ recording │─────────────────┐ │
│ │◀────────────┐ │ │
└─────┬─────┘ │ │ │
│ │ │ │
│ PAUSE_RECORDING │ │ │
▼ │ │ │
┌───────────┐ │ │ │
│ │ │ │ │
│ paused │ │ │ │
│ │ │ │ │
└─────┬─────┘ │ │ │
│ │ │ │
│ RESUME_RECORDING │ │ │
└───────────────────┘ │ │
│ │
▼ │
┌───────────┐
│ │
│ stopped │
│ │
└─────┬─────┘
│ SAVE_RECORDING
┌───────────┐
│ │
│ ready │
│ │
└───────────┘
```
## Events
The following events trigger state transitions:
- `NEW_RECORDING`: Create a new recording when an app starts or is detected
- `START_RECORDING`: Start recording audio
- `PAUSE_RECORDING`: Pause the current recording
- `RESUME_RECORDING`: Resume a paused recording
- `STOP_RECORDING`: Stop the current recording
- `SAVE_RECORDING`: Save and finalize a recording
- `REMOVE_RECORDING`: Delete a recording
## Error Handling
Invalid state transitions are logged and prevented. For example:
- Cannot start a new recording when one is already in progress
- Cannot pause a recording that is not in the 'recording' state
- Cannot resume a recording that is not in the 'paused' state
Each transition function in the state machine validates the current state before allowing a transition.
@@ -33,8 +33,17 @@ export interface Recording {
export interface RecordingStatus {
id: number; // corresponds to the recording id
status: 'recording' | 'paused' | 'stopped';
// the status of the recording in a linear state machine
// new: an new app group is listening. note, if there are any active recording, the current recording will not change
// recording: the recording is ongoing
// paused: the recording is paused
// stopped: the recording is stopped (processing audio file for use in the editor)
// ready: the recording is ready to be used
status: 'new' | 'recording' | 'paused' | 'stopped' | 'ready';
app?: TappableAppInfo;
appGroup?: AppGroupInfo;
startTime: number;
startTime: number; // 0 means not started yet
filepath?: string; // the filepath of the recording (only available when status is ready)
sampleRate?: number;
numberOfChannels?: number;
}
@@ -14,11 +14,10 @@ import { beforeAppQuit } from '../cleanup';
import { logger } from '../logger';
import {
appGroups$,
pauseRecording,
recordingStatus$,
resumeRecording,
startRecording,
stopRecording,
updateApplicationsPing$,
} from '../recording';
import { getMainWindow } from '../windows-manager';
import { icons } from './icons';
@@ -132,7 +131,21 @@ class TrayState {
const recordingStatus = recordingStatus$.value;
if (!recordingStatus || recordingStatus?.status === 'stopped') {
if (
!recordingStatus ||
(recordingStatus?.status !== 'paused' &&
recordingStatus?.status !== 'recording')
) {
const appMenuItems = runningAppGroups.map(appGroup => ({
label: appGroup.name,
icon: appGroup.icon || undefined,
click: () => {
logger.info(
`User action: Start Recording Meeting (${appGroup.name})`
);
startRecording(appGroup);
},
}));
return {
key: 'recording',
getConfig: () => [
@@ -150,18 +163,10 @@ class TrayState {
startRecording();
},
},
...runningAppGroups.map(appGroup => ({
label: appGroup.name,
icon: appGroup.icon || undefined,
click: () => {
logger.info(
`User action: Start Recording Meeting (${appGroup.name})`
);
startRecording(appGroup);
},
})),
...appMenuItems,
],
},
...appMenuItems,
],
};
}
@@ -179,26 +184,11 @@ class TrayState {
icon: icons.recording,
disabled: true,
},
recordingStatus.status === 'paused'
? {
label: 'Resume',
click: () => {
logger.info('User action: Resume Recording');
resumeRecording();
},
}
: {
label: 'Pause',
click: () => {
logger.info('User action: Pause Recording');
pauseRecording();
},
},
{
label: 'Stop',
click: () => {
logger.info('User action: Stop Recording');
stopRecording().catch(err => {
stopRecording(recordingStatus.id).catch(err => {
logger.error('Failed to stop recording:', err);
});
},
@@ -260,6 +250,7 @@ class TrayState {
if (!isMacOS()) {
this.tray?.popUpContextMenu();
}
updateApplicationsPing$.next(Date.now());
};
this.tray.on('click', clickHandler);
const appGroupsSubscription = appGroups$.subscribe(() => {
@@ -1,18 +1,12 @@
import { join } from 'node:path';
import type { Display } from 'electron';
import { BrowserWindow, screen } from 'electron';
import { isMacOS } from '../../shared/utils';
import { isDev } from '../config';
import { onboardingViewUrl } from '../constants';
// import { getExposedMeta } from './exposed';
import { logger } from '../logger';
const getScreenSize = (display: Display) => {
const { width, height } = isMacOS() ? display.bounds : display.workArea;
return { width, height };
};
import { fullscreenAndCenter, getScreenSize } from './utils';
// todo: not all window need all of the exposed meta
const getWindowAdditionalArguments = async () => {
@@ -24,19 +18,6 @@ const getWindowAdditionalArguments = async () => {
];
};
function fullscreenAndCenter(browserWindow: BrowserWindow) {
const position = browserWindow.getPosition();
const size = browserWindow.getSize();
const currentDisplay = screen.getDisplayNearestPoint({
x: position[0] + size[0] / 2,
y: position[1] + size[1] / 2,
});
if (!currentDisplay) return;
const { width, height } = getScreenSize(currentDisplay);
browserWindow.setSize(width, height);
browserWindow.center();
}
async function createOnboardingWindow(additionalArguments: string[]) {
logger.info('creating onboarding window');
@@ -0,0 +1,277 @@
import { join } from 'node:path';
import { setTimeout } from 'node:timers/promises';
import { BrowserWindow, type BrowserWindowConstructorOptions } from 'electron';
import { BehaviorSubject } from 'rxjs';
import { popupViewUrl } from '../constants';
import { logger } from '../logger';
import type { MainEventRegister, NamespaceHandlers } from '../type';
import { getCurrentDisplay } from './utils';
type PopupWindowType = 'notification' | 'recording';
async function getAdditionalArguments(name: string) {
const { getExposedMeta } = await import('../exposed');
const mainExposedMeta = getExposedMeta();
return [
`--main-exposed-meta=` + JSON.stringify(mainExposedMeta),
`--window-name=${name}`,
];
}
const POPUP_PADDING = 20; // padding between the popup and the edge of the screen
const NOTIFICATION_SIZE = [300, 128];
const RECORDING_SIZE = [300, 36];
async function animate(
current: number,
target: number,
setter: (val: number) => void,
duration = 200,
delay = 0
): Promise<void> {
const fps = 60;
const steps = duration / (1000 / fps);
const delta = target - current;
const easing = (t: number) => -(Math.cos(Math.PI * t) - 1) / 2;
if (delay > 0) {
await setTimeout(delay);
}
for (let i = 0; i < steps; i++) {
const progress = easing(i / steps);
setter(current + delta * progress);
await setTimeout(1000 / fps);
}
// Ensure we hit the target exactly
setter(target);
}
abstract class PopupWindow {
abstract readonly type: PopupWindowType;
abstract readonly name: string;
browserWindow: BrowserWindow | undefined;
abstract windowOptions: Partial<BrowserWindowConstructorOptions>;
resolveReady: () => void = () => {};
ready = new Promise<void>(resolve => {
this.resolveReady = resolve;
});
private readonly showing$ = new BehaviorSubject<boolean>(false);
get showing() {
return this.showing$.value;
}
async build(): Promise<BrowserWindow> {
const browserWindow = new BrowserWindow({
...this.windowOptions,
resizable: false,
minimizable: false,
maximizable: false,
closable: false,
alwaysOnTop: true,
focusable: false,
hiddenInMissionControl: true,
movable: false,
titleBarStyle: 'hidden',
show: false, // hide by default,
backgroundColor: 'transparent',
visualEffectState: 'active',
vibrancy: 'under-window',
webPreferences: {
...this.windowOptions.webPreferences,
webgl: true,
contextIsolation: true,
sandbox: false,
transparent: true,
spellcheck: false,
preload: join(__dirname, './preload.js'), // this points to the bundled preload module
// serialize exposed meta that to be used in preload
additionalArguments: await getAdditionalArguments(this.name),
},
});
// required to make the window transparent
browserWindow.setBackgroundColor('#00000000');
browserWindow.loadURL(popupViewUrl).catch(err => logger.error(err));
browserWindow.on('ready-to-show', () => {
browserWindow.webContents.on('did-finish-load', () => {
this.resolveReady();
});
});
return browserWindow;
}
async show() {
if (!this.browserWindow) {
this.browserWindow = await this.build();
}
const browserWindow = this.browserWindow;
const workArea = getCurrentDisplay(browserWindow).workArea;
const popupSize = browserWindow.getSize();
await this.ready;
this.showing$.next(true);
browserWindow.showInactive(); // focus the notification is too distracting right?
browserWindow.setOpacity(0);
// Calculate start and end positions for x coordinate
const startX = workArea.x + workArea.width + popupSize[0] + POPUP_PADDING;
const endX = workArea.x + workArea.width - popupSize[0] - POPUP_PADDING;
const y = workArea.y + POPUP_PADDING;
// Set initial position
browserWindow.setPosition(startX, y);
// First fade in, then slide
await Promise.all([
// Slide in animation
animate(
startX,
endX,
x => {
browserWindow.setPosition(Math.round(x), y);
},
300
),
// Fade in animation
animate(
0,
1,
opacity => {
this.browserWindow?.setOpacity(opacity);
},
100,
100
),
]);
}
async hide() {
if (!this.browserWindow) {
return;
}
this.showing$.next(false);
await animate(this.browserWindow.getOpacity(), 0, opacity => {
this.browserWindow?.setOpacity(opacity);
});
this.browserWindow?.hide();
}
destroy() {
this.browserWindow?.destroy();
}
}
// leave for future use
type ElectronNotification = null;
class NotificationPopupWindow extends PopupWindow {
readonly type = 'notification' as const;
readonly name = `${this.type}`;
notification$ = new BehaviorSubject<ElectronNotification | null>(null);
windowOptions: Partial<BrowserWindowConstructorOptions> = {
width: NOTIFICATION_SIZE[0],
height: NOTIFICATION_SIZE[1],
};
async notify(notification: ElectronNotification) {
this.notification$.next(notification);
await super.show();
}
}
// recording popup window is singleton across the app
class RecordingPopupWindow extends PopupWindow {
readonly type = 'recording' as const;
readonly name = `${this.type}`;
windowOptions: Partial<BrowserWindowConstructorOptions> = {
width: RECORDING_SIZE[0],
height: RECORDING_SIZE[1],
};
}
// Type mapping from PopupWindowType to specific window class
type PopupWindowTypeMap = {
notification: NotificationPopupWindow;
recording: RecordingPopupWindow;
};
export class PopupManager {
static readonly instance = new PopupManager();
// there could be a single instance of each type of popup window
readonly popupWindows$ = new BehaviorSubject<Map<string, PopupWindow>>(
new Map()
);
get<T extends PopupWindowType>(type: T): PopupWindowTypeMap[T] {
// Check if popup of this type already exists
const existingPopup = Array.from(this.popupWindows$.value.values()).find(
popup => popup.type === type
) as PopupWindowTypeMap[T] | undefined;
// If exists, return it
if (existingPopup) {
return existingPopup;
}
// Otherwise create a new one
const popupWindow = (() => {
switch (type) {
case 'notification':
return new NotificationPopupWindow() as PopupWindowTypeMap[T];
case 'recording':
return new RecordingPopupWindow() as PopupWindowTypeMap[T];
}
})();
this.popupWindows$.next(
new Map(this.popupWindows$.value).set(popupWindow.type, popupWindow)
);
return popupWindow;
}
}
export const popupManager = PopupManager.instance;
// recording popup window events/handlers are in ../recording/index.ts
export const popupHandlers = {
getCurrentNotification: async () => {
const notification = popupManager.get('notification').notification$.value;
if (!notification) {
return null;
}
return notification;
},
dismissCurrentNotification: async () => {
return popupManager.get('notification').hide();
},
dismissCurrentRecording: async () => {
return popupManager.get('recording').hide();
},
} satisfies NamespaceHandlers;
export const popupEvents = {
onNotificationChanged: (
callback: (notification: ElectronNotification | null) => void
) => {
const notification = popupManager.get('notification');
const sub = notification.notification$.subscribe(notification => {
callback(notification);
});
return () => {
sub.unsubscribe();
};
},
} satisfies Record<string, MainEventRegister>;
@@ -745,7 +745,8 @@ export class WebContentViewsManager {
const focusActiveView = () => {
if (
!this.activeWorkbenchView ||
this.activeWorkbenchView.webContents.isFocused()
this.activeWorkbenchView.webContents.isFocused() ||
this.activeWorkbenchView.webContents.isDevToolsFocused()
) {
return;
}
@@ -0,0 +1,27 @@
import { BrowserWindow, type Display, type Rectangle, screen } from 'electron';
import { isMacOS } from '../../shared/utils';
export const getCurrentDisplay = (browserWindow: BrowserWindow) => {
const position = browserWindow.getPosition();
const size = browserWindow.getSize();
const currentDisplay = screen.getDisplayNearestPoint({
x: position[0] + size[0] / 2,
y: position[1] + size[1] / 2,
});
return currentDisplay;
};
export const getScreenSize = (display: Display | BrowserWindow): Rectangle => {
if (display instanceof BrowserWindow) {
return getScreenSize(getCurrentDisplay(display));
}
return isMacOS() ? display.bounds : display.workArea;
};
export const fullscreenAndCenter = (browserWindow: BrowserWindow) => {
const currentDisplay = getCurrentDisplay(browserWindow);
const { width, height } = getScreenSize(currentDisplay);
browserWindow.setSize(width, height);
browserWindow.center();
};
@@ -48,7 +48,7 @@ export class MessageEventChannel implements EventBasedChannel {
export const resourcesPath = join(__dirname, `../resources`);
// credit: https://github.com/facebook/fbjs/blob/main/packages/fbjs/src/core/shallowEqual.js
export function shallowEqual(objA: any, objB: any) {
export function shallowEqual<T>(objA: T, objB: T) {
if (Object.is(objA, objB)) {
return true;
}
@@ -73,7 +73,7 @@ export function shallowEqual(objA: any, objB: any) {
for (const key of keysA) {
if (
!Object.prototype.hasOwnProperty.call(objB, key) ||
!Object.is(objA[key], objB[key])
!Object.is(objA[key as keyof T], objB[key as keyof T])
) {
return false;
}
@@ -1,11 +1,11 @@
import { Button } from '@affine/component';
import { Button, Tooltip } from '@affine/component';
import { AudioPlayer } from '@affine/core/components/audio-player';
import { AnimatedTranscribeIcon } from '@affine/core/components/audio-player/lottie/animated-transcribe-icon';
import { useSeekTime } from '@affine/core/components/audio-player/use-seek-time';
import { useEnableAI } from '@affine/core/components/hooks/affine/use-enable-ai';
import type { AudioAttachmentBlock } from '@affine/core/modules/media/entities/audio-attachment-block';
import { useAttachmentMediaBlock } from '@affine/core/modules/media/views/use-attachment-media';
import { useI18n } from '@affine/i18n';
import { TranscriptWithAiIcon } from '@blocksuite/icons/rc';
import { useLiveData } from '@toeverything/infra';
import { useCallback, useMemo } from 'react';
@@ -54,11 +54,14 @@ const AttachmentAudioPlayer = ({ block }: { block: AudioAttachmentBlock }) => {
if (!enableAi) {
return null;
}
return (
const inner = (
<Button
variant="plain"
prefix={<TranscriptWithAiIcon />}
loading={transcribing}
prefix={
<AnimatedTranscribeIcon
state={transcribing ? 'transcribing' : 'idle'}
/>
}
disabled={transcribing}
size="large"
prefixClassName={styles.notesButtonIcon}
@@ -71,11 +74,19 @@ const AttachmentAudioPlayer = ({ block }: { block: AudioAttachmentBlock }) => {
}
}}
>
{transcribing
? t['com.affine.attachmentViewer.audio.transcribing']()
: t['com.affine.attachmentViewer.audio.notes']()}
{t['com.affine.attachmentViewer.audio.notes']()}
</Button>
);
if (transcribing) {
return (
<Tooltip
content={t['com.affine.attachmentViewer.audio.transcribing']()}
>
{inner}
</Tooltip>
);
}
return inner;
}, [enableAi, transcribing, t, transcribed, block, expanded]);
return (
@@ -167,7 +167,8 @@ export const miniNameLabel = style({
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
height: 20,
lineHeight: '20px',
marginBottom: 2,
});
export const miniPlayerContainer = style({
@@ -1,6 +1,12 @@
import { IconButton } from '@affine/component';
import { CloseIcon, VoiceIcon } from '@blocksuite/icons/rc';
import {
AddThirtySecondIcon,
CloseIcon,
ReduceFifteenSecondIcon,
VoiceIcon,
} from '@blocksuite/icons/rc';
import bytes from 'bytes';
import { clamp } from 'lodash-es';
import { type MouseEventHandler, type ReactNode, useCallback } from 'react';
import * as styles from './audio-player.css';
@@ -150,19 +156,19 @@ export const MiniAudioPlayer = ({
);
const handleRewind = useCallback(
(e: React.MouseEvent<HTMLDivElement>) => {
(e: React.MouseEvent<HTMLButtonElement>) => {
e.stopPropagation();
onSeek(seekTime - 15);
onSeek(clamp(seekTime - 15, 0, duration));
},
[seekTime, onSeek]
[seekTime, duration, onSeek]
);
const handleForward = useCallback(
(e: React.MouseEvent<HTMLDivElement>) => {
(e: React.MouseEvent<HTMLButtonElement>) => {
e.stopPropagation();
onSeek(seekTime + 15);
onSeek(clamp(seekTime + 30, 0, duration));
},
[seekTime, onSeek]
[seekTime, duration, onSeek]
);
const handleClose = useCallback(
@@ -186,13 +192,24 @@ export const MiniAudioPlayer = ({
<div className={styles.miniRoot} onClick={onClick}>
<div className={styles.miniNameLabel}>{name}</div>
<div className={styles.miniPlayerContainer}>
<div onClick={handleRewind}>-15s</div>
<IconButton
icon={<ReduceFifteenSecondIcon />}
size={18}
variant="plain"
onClick={handleRewind}
/>
<AnimatedPlayIcon
onClick={handlePlayToggle}
className={styles.controlButton}
state={iconState}
/>
<div onClick={handleForward}>+15s</div>
<IconButton
icon={<AddThirtySecondIcon />}
size={18}
variant="plain"
onClick={handleForward}
/>
</div>
<IconButton
className={styles.miniCloseButton}
@@ -1,3 +0,0 @@
import { style } from '@vanilla-extract/css';
export const root = style({});
@@ -4,9 +4,9 @@ import type { LottieRef } from 'lottie-react';
import Lottie from 'lottie-react';
import { useEffect, useRef } from 'react';
import * as styles from './animated-play-icon.css';
import pausetoplay from './pausetoplay.json';
import playtopause from './playtopause.json';
import * as styles from './styles.css';
export interface AnimatedPlayIconProps {
state: 'play' | 'pause' | 'loading';
@@ -0,0 +1,81 @@
import clsx from 'clsx';
import type { LottieRef } from 'lottie-react';
import Lottie from 'lottie-react';
import { useEffect, useRef } from 'react';
import * as styles from './styles.css';
import lottieData from './transcribe.json';
export interface AnimatedTranscribeIconProps {
state: 'idle' | 'transcribing';
className?: string;
onClick?: (e: React.MouseEvent) => void;
}
export const AnimatedTranscribeIcon = ({
state,
className,
onClick,
}: AnimatedTranscribeIconProps) => {
const lottieRef: LottieRef = useRef(null);
useEffect(() => {
if (!lottieRef.current) return;
let loopInterval: NodeJS.Timeout | null = null;
// Cleanup function to clear any existing intervals
const cleanup = () => {
const animating = !!loopInterval;
if (loopInterval) {
clearInterval(loopInterval);
loopInterval = null;
}
const lottie = lottieRef.current;
// Play the final segment when stopped
if (lottie && animating) {
lottie.goToAndPlay(lottie.animationItem?.currentFrame || 0, true);
}
};
if (state === 'transcribing') {
// First play the transition to playing state (0-35)
lottieRef.current.playSegments([0, 35], true);
// After transition, start the main loop
const startMainLoop = () => {
// Play the main animation segment (35-64)
lottieRef.current?.playSegments([35, 64], true);
// Set up interval to continue looping
loopInterval = setInterval(
() => {
if (loopInterval) {
lottieRef.current?.playSegments([35, 64], true);
}
},
(64 - 35) * (1000 / 60)
); // 60fps
};
// Start the main loop after the transition
setTimeout(startMainLoop, 10 * (1000 / 60)); // Wait for transition to complete
} else {
cleanup();
}
// Cleanup on unmount or when state changes
return cleanup;
}, [state]);
return (
<Lottie
onClick={onClick}
lottieRef={lottieRef}
className={clsx(styles.root, className)}
animationData={lottieData}
loop={false} // We're handling the loop manually
autoplay={false} // We're controlling playback manually
/>
);
};
@@ -0,0 +1,59 @@
import { cssVarV2 } from '@toeverything/theme/v2';
import { globalStyle, style } from '@vanilla-extract/css';
export const root = style({});
// replace primary colors to cssVarV2('icon/primary')
const iconPrimaryColors = [
// legacy "--affine-icon-color"
'rgb(119,117,125)',
// --affine-v2-icon-primary
'rgb(122,122,122)',
];
// todo: may need to replace secondary colors & background colors as well?
const backgroundPrimaryColors = [
// --affine-v2-background-primary
'rgb(255,255,255)',
'#ffffff',
];
const backgroundSecondaryColors = [
// --affine-v2-background-secondary
'rgb(245,245,245)',
];
globalStyle(
`${root} :is(${iconPrimaryColors.map(color => `path[fill="${color}"]`).join(',')})`,
{
fill: cssVarV2('icon/primary'),
}
);
globalStyle(
`${root} :is(${iconPrimaryColors.map(color => `path[stroke="${color}"]`).join(',')})`,
{
stroke: cssVarV2('icon/primary'),
}
);
globalStyle(
`${root} :is(${backgroundPrimaryColors.map(color => `rect[fill="${color}"]`).join(',')})`,
{
fill: 'transparent',
}
);
globalStyle(
`${root} :is(${backgroundPrimaryColors.map(color => `path[fill="${color}"]`).join(',')})`,
{
fill: 'transparent',
}
);
globalStyle(
`${root} :is(${backgroundSecondaryColors.map(color => `path[fill="${color}"]`).join(',')})`,
{
fill: cssVarV2('layer/background/secondary'),
}
);
File diff suppressed because it is too large Load Diff
@@ -15,3 +15,7 @@ export function configureAppThemeModule(framework: Framework) {
.scope(WorkspaceScope)
.service(EdgelessThemeService, [AppThemeService, EditorSettingService]);
}
export function configureEssentialThemeModule(framework: Framework) {
framework.service(AppThemeService).entity(AppTheme);
}
+42 -30
View File
@@ -1114,10 +1114,6 @@ export function useAFFiNEI18N(): {
["com.affine.auth.sign.auth.code.resend.hint"](options: {
readonly second: string;
}): string;
/**
* `Sent`
*/
["com.affine.auth.sent"](): string;
/**
* `The verification link failed to be sent, please try again later.`
*/
@@ -1707,6 +1703,10 @@ export function useAFFiNEI18N(): {
* `Cancel`
*/
["com.affine.confirmModal.button.cancel"](): string;
/**
* `Ok`
*/
["com.affine.confirmModal.button.ok"](): string;
/**
* `Current year`
*/
@@ -6928,7 +6928,7 @@ export function useAFFiNEI18N(): {
*/
["com.affine.editor.at-menu.date-picker"](): string;
/**
* `Mention Members`
* `Mention Member`
*/
["com.affine.editor.at-menu.mention-member"](): string;
/**
@@ -7428,15 +7428,45 @@ export function useAFFiNEI18N(): {
/**
* `Notes`
*/
["com.affine.attachmentViewer.audio.notes"](): string;
["com.affine.audio.notes"](): string;
/**
* `Transcribing`
*/
["com.affine.attachmentViewer.audio.transcribing"](): string;
["com.affine.audio.transcribing"](): string;
/**
* `Accept & Join`
* `Unable to retrieve AI results for others`
*/
["com.affine.notification.invitation.accept"](): string;
["com.affine.audio.transcribe.non-owner.confirm.title"](): string;
/**
* `Audio activity`
*/
["com.affine.recording.new"](): string;
/**
* `Open app`
*/
["com.affine.recording.ready"](): string;
/**
* `{{appName}}'s audio`
*/
["com.affine.recording.recording"](options: {
readonly appName: string;
}): string;
/**
* `Audio recording`
*/
["com.affine.recording.recording.unnamed"](): string;
/**
* `Start`
*/
["com.affine.recording.start"](): string;
/**
* `Dismiss`
*/
["com.affine.recording.dismiss"](): string;
/**
* `Stop`
*/
["com.affine.recording.stop"](): string;
/**
* `An internal error occurred.`
*/
@@ -8494,29 +8524,11 @@ export const TypedTrans: {
a: JSX.Element;
}>>;
/**
* `<1>{{username}}</1> has accept your invitation`
* `Please contact <1>{{user}}</1> to upgrade AI rights or resend the attachment.`
*/
["com.affine.notification.invitation-accepted"]: ComponentType<TypedTransProps<{
readonly username: string;
["com.affine.audio.transcribe.non-owner.confirm.message"]: ComponentType<TypedTransProps<{
readonly user: string;
}, {
["1"]: JSX.Element;
}>>;
/**
* `There is an issue regarding your invitation to <1>{{workspaceName}}</1> `
*/
["com.affine.notification.invitation-blocked"]: ComponentType<TypedTransProps<{
readonly workspaceName: string;
}, {
["1"]: JSX.Element;
}>>;
/**
* `<1>{{username}}</1> invited you to join <2>{{workspaceName}}</2>`
*/
["com.affine.notification.invitation"]: ComponentType<TypedTransProps<Readonly<{
username: string;
workspaceName: string;
}>, {
["1"]: JSX.Element;
["2"]: JSX.Element;
}>>;
} = /*#__PURE__*/ createProxy(createComponent);
+13 -8
View File
@@ -270,7 +270,6 @@
"com.affine.auth.sign.auth.code.continue": "Continue with code",
"com.affine.auth.sign.auth.code.resend": "Resend code",
"com.affine.auth.sign.auth.code.resend.hint": "Resend in {{second}}s",
"com.affine.auth.sent": "Sent",
"com.affine.auth.sent.change.email.fail": "The verification link failed to be sent, please try again later.",
"com.affine.auth.sent.change.email.hint": "Verification link has been sent.",
"com.affine.auth.sent.change.password.hint": "Reset password link has been sent.",
@@ -422,6 +421,7 @@
"com.affine.copy.asImage.success": "Image copied",
"com.affine.copy.asImage.failed": "Image copy failed",
"com.affine.confirmModal.button.cancel": "Cancel",
"com.affine.confirmModal.button.ok": "Ok",
"com.affine.currentYear": "Current year",
"com.affine.delete-tags.confirm.description": "Deleting <1>{{tag}}</1> cannot be undone, please proceed with caution.",
"com.affine.delete-tags.confirm.multi-tag-description": "Deleting {{count}} tags cannot be undone, please proceed with caution.",
@@ -1720,7 +1720,7 @@
"com.affine.editor.at-menu.more-docs-hint": "{{count}} more docs",
"com.affine.editor.at-menu.journal": "Journal",
"com.affine.editor.at-menu.date-picker": "Select a specific date",
"com.affine.editor.at-menu.mention-member": "Mention Members",
"com.affine.editor.at-menu.mention-member": "Mention Member",
"com.affine.editor.at-menu.mention-success": "Successfully mentioned the member",
"com.affine.editor.bi-directional-link-panel.show": "Show",
"com.affine.editor.bi-directional-link-panel.hide": "Hide",
@@ -1851,12 +1851,17 @@
"com.affine.integration.readwise-prop.created": "Created",
"com.affine.integration.readwise-prop.updated": "Updated",
"com.affine.integration.properties": "Integration properties",
"com.affine.attachmentViewer.audio.notes": "Notes",
"com.affine.attachmentViewer.audio.transcribing": "Transcribing",
"com.affine.notification.invitation-accepted": "<1>{{username}}</1> has accept your invitation",
"com.affine.notification.invitation-blocked": "There is an issue regarding your invitation to <1>{{workspaceName}}</1> ",
"com.affine.notification.invitation": "<1>{{username}}</1> invited you to join <2>{{workspaceName}}</2>",
"com.affine.notification.invitation.accept": "Accept & Join",
"com.affine.audio.notes": "Notes",
"com.affine.audio.transcribing": "Transcribing",
"com.affine.audio.transcribe.non-owner.confirm.title": "Unable to retrieve AI results for others",
"com.affine.audio.transcribe.non-owner.confirm.message": "Please contact <1>{{user}}</1> to upgrade AI rights or resend the attachment.",
"com.affine.recording.new": "Audio activity",
"com.affine.recording.ready": "Open app",
"com.affine.recording.recording": "{{appName}}'s audio",
"com.affine.recording.recording.unnamed": "Audio recording",
"com.affine.recording.start": "Start",
"com.affine.recording.dismiss": "Dismiss",
"com.affine.recording.stop": "Stop",
"error.INTERNAL_SERVER_ERROR": "An internal error occurred.",
"error.NETWORK_ERROR": "Network error.",
"error.TOO_MANY_REQUEST": "Too many requests.",
+13
View File
@@ -112,6 +112,19 @@ export function createBackgroundWorkerHTMLPlugin(
});
}
export function createPopupHTMLPlugin(
flags: BuildFlags,
BUILD_CONFIG: BUILD_CONFIG_TYPE
) {
const htmlPluginOptions = getHTMLPluginOptions(flags, BUILD_CONFIG);
return new HTMLPlugin({
...htmlPluginOptions,
chunks: ['popup'],
filename: `popup.html`,
});
}
export function createHTMLPlugins(
flags: BuildFlags,
BUILD_CONFIG: BUILD_CONFIG_TYPE
+2
View File
@@ -17,6 +17,7 @@ import { productionCacheGroups } from './cache-group.js';
import {
createBackgroundWorkerHTMLPlugin,
createHTMLPlugins,
createPopupHTMLPlugin,
createShellHTMLPlugin,
} from './html-plugin.js';
import { WebpackS3Plugin } from './s3-plugin.js';
@@ -425,6 +426,7 @@ export function createWebpackConfig(
if (buildConfig.isElectron) {
config.plugins.push(createShellHTMLPlugin(flags, buildConfig));
config.plugins.push(createBackgroundWorkerHTMLPlugin(flags, buildConfig));
config.plugins.push(createPopupHTMLPlugin(flags, buildConfig));
// sourcemap url like # sourceMappingURL=76-6370cd185962bc89.js.map wont load in electron
// this is because the default file:// protocol will be ignored by Chromium