Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-12 15:46:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(native): async recorder (#14700)

Browse Source 
#### PR Dependency Tree


* **PR #14700** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Durable, resumable import queue with explicit import lifecycle and
updated popup/tray status behavior.
* Async native recording APIs and ability to abort recordings; audio
quality metrics (degraded, overflow count).
  * Added "Importing..." translation.

* **Bug Fixes**
* More reliable single-claim import processing, retries and cleanup to
avoid duplicate imports.
  * Improved stop/abort teardown stability and safer shutdown behavior.

* **Tests**
* New/updated tests covering coordinator, import queue, native async
flows and teardown scenarios.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-03-23 22:38:52 +08:00
committed by GitHub
parent b0251c93cb
commit 9b56a05159
26 changed files with 3560 additions and 1277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+3 -2
View File
@@ -99,6 +99,7 @@ dependencies = [
"screencapturekit",
"symphonia",
"thiserror 2.0.18",
"tokio",
"uuid",
"windows 0.61.3",
"windows-core 0.61.2",
@@ -5060,9 +5061,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.103.9"
version = "0.103.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
dependencies = [
"ring",
"rustls-pki-types",
packages/frontend/apps/electron-renderer/src/app/effects/recording.ts
+244 -158
View File
@@ -7,22 +7,33 @@ import { apis, events } from '@affine/electron-api';
import { i18nTime } from '@affine/i18n';
import track from '@affine/track';
import type { AttachmentBlockModel } from '@blocksuite/affine/model';
import type { Store } from '@blocksuite/affine/store';
import type { BlobEngine } from '@blocksuite/affine/sync';
import type { FrameworkProvider } from '@toeverything/infra';
import { getCurrentWorkspace, isAiEnabled } from './utils';
const logger = new DebugLogger('electron-renderer:recording');
const RECORDING_PROCESS_RETRY_MS = 1000;
const RECORDING_IMPORT_RETRY_MS = 1000;
const NATIVE_RECORDING_MIME_TYPE = 'audio/ogg';
type ProcessingRecordingStatus = {
type RecordingImportStatus = {
id: number;
status: 'processing';
appName?: string;
blockCreationStatus?: undefined;
workspaceId?: string;
docId?: string;
filepath: string;
startTime: number;
sampleRate?: number;
numberOfChannels?: number;
durationMs?: number;
size?: number;
degraded?: boolean;
overflowCount?: number;
errorMessage?: string;
importStatus: 'pending_import' | 'importing' | 'imported' | 'import_failed';
createdAt: number;
updatedAt: number;
};
type WorkspaceHandle = NonNullable<ReturnType<typeof getCurrentWorkspace>>;
@@ -65,29 +76,7 @@ async function saveRecordingBlob(blobEngine: BlobEngine, filepath: string) {
return { blob, blobId };
}
function shouldProcessRecording(
status: unknown
): status is ProcessingRecordingStatus {
return (
!!status &&
typeof status === 'object' &&
'status' in status &&
status.status === 'processing' &&
'filepath' in status &&
typeof status.filepath === 'string' &&
!('blockCreationStatus' in status && status.blockCreationStatus)
);
}
async function createRecordingDoc(
frameworkProvider: FrameworkProvider,
workspace: WorkspaceHandle['workspace'],
status: ProcessingRecordingStatus
) {
const docsService = workspace.scope.get(DocsService);
const aiEnabled = isAiEnabled(frameworkProvider);
const recordingFilepath = status.filepath;
function getAttachmentName(status: RecordingImportStatus) {
const timestamp = i18nTime(status.startTime, {
absolute: {
accuracy: 'minute',
@@ -95,77 +84,145 @@ async function createRecordingDoc(
},
});
await new Promise<void>((resolve, reject) => {
const docProps: DocProps = {
onStoreLoad: (doc, { noteId }) => {
void (async () => {
// it takes a while to save the blob, so we show the attachment first
const { blobId, blob } = await saveRecordingBlob(
doc.workspace.blobSync,
recordingFilepath
);
return {
timestamp,
attachmentName:
(status.appName ?? 'System Audio') + ' ' + timestamp + '.opus',
};
}
// name + timestamp(readable) + extension
const attachmentName =
(status.appName ?? 'System Audio') + ' ' + timestamp + '.opus';
function ensureNoteId(docStore: Store) {
const [existingNote] = docStore.getModelsByFlavour('affine:note');
if (existingNote) {
return existingNote.id;
}
const attachmentId = doc.addBlock(
'affine:attachment',
{
name: attachmentName,
type: NATIVE_RECORDING_MIME_TYPE,
size: blob.size,
sourceId: blobId,
embed: true,
},
noteId
);
const [page] = docStore.getModelsByFlavour('affine:page');
if (!page) {
throw new Error('Recording doc is missing the page block');
}
const model = doc.getBlock(attachmentId)
?.model as AttachmentBlockModel;
return docStore.addBlock('affine:note', {}, page.id);
}
if (!aiEnabled) {
return;
}
function findExistingAttachment(docStore: Store, attachmentName: string) {
return (
docStore.getModelsByFlavour('affine:attachment') as AttachmentBlockModel[]
).find(
model =>
model.props.name === attachmentName &&
model.props.type === NATIVE_RECORDING_MIME_TYPE
);
}
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
if (!currentWorkspace) {
return;
}
const { workspace } = currentWorkspace;
using audioAttachment = workspace.scope
.get(AudioAttachmentService)
.get(model);
audioAttachment?.obj
.transcribe()
.then(() => {
track.doc.editor.audioBlock.transcribeRecording({
type: 'Meeting record',
method: 'success',
option: 'Auto transcribing',
});
})
.catch(err => {
logger.error('Failed to transcribe recording', err);
});
})().then(resolve, reject);
},
};
async function createRecordingDoc(
frameworkProvider: FrameworkProvider,
workspace: WorkspaceHandle['workspace'],
status: RecordingImportStatus
) {
if (!status.docId) {
throw new Error('Recording import is missing docId');
}
const page = docsService.createDoc({
const docsService = workspace.scope.get(DocsService);
const aiEnabled = isAiEnabled(frameworkProvider);
const { attachmentName, timestamp } = getAttachmentName(status);
const targetDocId = status.docId;
const docExists = !!docsService.list.doc$(targetDocId).value;
if (!docExists) {
const docProps: DocProps = {};
docsService.createDoc({
id: targetDocId,
docProps,
title:
'Recording ' + (status.appName ?? 'System Audio') + ' ' + timestamp,
primaryMode: 'page',
});
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
});
}
const { doc, release } = docsService.open(targetDocId);
const disposePriorityLoad = doc.addPriorityLoad(10);
try {
try {
await doc.waitForSyncReady();
} finally {
disposePriorityLoad();
}
const noteId = ensureNoteId(doc.blockSuiteDoc);
const existingAttachment = findExistingAttachment(
doc.blockSuiteDoc,
attachmentName
);
let model = existingAttachment;
let attachmentCreated = false;
if (!model) {
const { blobId, blob } = await saveRecordingBlob(
workspace.docCollection.blobSync,
status.filepath
);
const attachmentId = doc.blockSuiteDoc.addBlock(
'affine:attachment',
{
name: attachmentName,
type: NATIVE_RECORDING_MIME_TYPE,
size: blob.size,
sourceId: blobId,
embed: true,
},
noteId
);
model = doc.blockSuiteDoc.getBlock(attachmentId)?.model as
| AttachmentBlockModel
| undefined;
if (!model) {
throw new Error('Failed to create recording attachment');
}
attachmentCreated = true;
}
workspace.scope.get(WorkbenchService).workbench.openDoc(targetDocId);
if (!aiEnabled || !attachmentCreated) {
return;
}
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
if (!currentWorkspace) {
return;
}
const { workspace: currentWorkspaceEntity } = currentWorkspace;
using audioAttachment = currentWorkspaceEntity.scope
.get(AudioAttachmentService)
.get(model);
audioAttachment?.obj
.transcribe()
.then(() => {
track.doc.editor.audioBlock.transcribeRecording({
type: 'Meeting record',
method: 'success',
option: 'Auto transcribing',
});
})
.catch(err => {
logger.error('Failed to transcribe recording', err);
});
} finally {
release();
}
}
export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
let pendingStatus: ProcessingRecordingStatus | null = null;
let importQueue: RecordingImportStatus[] = [];
let retryTimer: ReturnType<typeof setTimeout> | null = null;
let processingStatusId: number | null = null;
let isProcessingImport = false;
let hasSeenLiveQueueUpdate = false;
const clearRetry = () => {
if (retryTimer !== null) {
@@ -174,101 +231,130 @@ export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
}
};
const clearPending = (id?: number) => {
if (id === undefined || pendingStatus?.id === id) {
pendingStatus = null;
clearRetry();
}
if (id === undefined || processingStatusId === id) {
processingStatusId = null;
}
const updateQueue = (nextQueue: RecordingImportStatus[]) => {
importQueue = nextQueue;
};
const scheduleRetry = () => {
if (!pendingStatus || retryTimer !== null) {
if (importQueue.length === 0 || retryTimer !== null) {
return;
}
retryTimer = setTimeout(() => {
retryTimer = null;
void processPendingStatus().catch(console.error);
}, RECORDING_PROCESS_RETRY_MS);
void processNextImport().catch(console.error);
}, RECORDING_IMPORT_RETRY_MS);
};
const processPendingStatus = async () => {
const status = pendingStatus;
if (!status || processingStatusId === status.id) {
const processNextImport = async () => {
if (isProcessingImport) {
return;
}
let isActiveTab = false;
isProcessingImport = true;
try {
isActiveTab = !!(await apis?.ui.isActiveTab());
} catch (error) {
logger.error('Failed to probe active recording tab', error);
scheduleRetry();
return;
}
if (!isActiveTab) {
scheduleRetry();
return;
}
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
if (!currentWorkspace) {
// Workspace can lag behind the post-recording status update for a short
// time; keep retrying instead of permanently failing the import.
scheduleRetry();
return;
}
processingStatusId = status.id;
try {
await createRecordingDoc(
frameworkProvider,
currentWorkspace.workspace,
status
);
await apis?.recording.setRecordingBlockCreationStatus(
status.id,
'success'
);
clearPending(status.id);
} catch (error) {
logger.error('Failed to create recording block', error);
let isActiveTab = false;
try {
await apis?.recording.setRecordingBlockCreationStatus(
status.id,
'failed',
error instanceof Error ? error.message : undefined
isActiveTab = !!(await apis?.ui.isActiveTab());
} catch (error) {
logger.error('Failed to probe active recording tab', error);
return;
}
if (!isActiveTab) {
return;
}
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
if (!currentWorkspace) {
return;
}
const workspaceId = currentWorkspace.workspace.id;
const nextStatus =
importQueue.find(
status =>
status.importStatus === 'pending_import' &&
(!status.workspaceId || status.workspaceId === workspaceId)
) ??
importQueue.find(
status =>
status.importStatus === 'importing' &&
status.workspaceId === workspaceId
) ??
null;
if (!nextStatus) {
return;
}
const claimed = await apis?.recording.claimRecordingImport(
nextStatus.id,
workspaceId
);
if (!claimed) {
return;
}
try {
await createRecordingDoc(
frameworkProvider,
currentWorkspace.workspace,
claimed
);
} catch (error) {
const importError =
error instanceof Error
? error
: new Error('Failed to import recording artifact');
logger.error('Failed to import recording artifact', importError);
await apis?.recording.failRecordingImport(
nextStatus.id,
importError.message
);
return;
}
try {
await apis?.recording.completeRecordingImport(nextStatus.id);
} catch (error) {
const completionError =
error instanceof Error
? error
: new Error('Failed to persist recording import completion');
logger.error(
'Failed to persist recording import completion',
completionError
);
await apis?.recording.failRecordingImport(
nextStatus.id,
completionError.message
);
} finally {
clearPending(status.id);
}
} finally {
if (pendingStatus?.id === status.id) {
processingStatusId = null;
scheduleRetry();
}
isProcessingImport = false;
scheduleRetry();
}
};
events?.recording.onRecordingStatusChanged(status => {
if (shouldProcessRecording(status)) {
pendingStatus = status;
clearRetry();
void processPendingStatus().catch(console.error);
return;
}
if (apis?.recording) {
void apis.recording
.getRecordingImportQueue()
.then(queue => {
if (hasSeenLiveQueueUpdate) {
return;
}
updateQueue(queue ?? []);
void processNextImport().catch(console.error);
})
.catch(error => {
logger.error('Failed to load recording import queue', error);
});
}
if (!status) {
clearPending();
return;
}
if (pendingStatus?.id === status.id) {
clearPending(status.id);
}
events?.recording.onRecordingImportQueueChanged(queue => {
hasSeenLiveQueueUpdate = true;
updateQueue(queue);
clearRetry();
void processNextImport().catch(console.error);
});
}
packages/frontend/apps/electron-renderer/src/popup/recording/index.tsx
+38 -17
View File
@@ -10,8 +10,17 @@ import * as styles from './styles.css';
type Status = {
id: number;
status: 'new' | 'recording' | 'processing' | 'ready';
blockCreationStatus?: 'success' | 'failed';
status:
| 'new'
| 'starting'
| 'start_failed'
| 'recording'
| 'finalizing'
| 'pending_import'
| 'importing'
| 'imported'
| 'import_failed'
| 'finalize_failed';
appName?: string;
appGroupId?: number;
icon?: Buffer;
@@ -56,19 +65,18 @@ export function Recording() {
}
if (status.status === 'new') {
return t['com.affine.recording.new']();
} else if (
status.status === 'ready' &&
status.blockCreationStatus === 'success'
) {
} else if (status.status === 'imported') {
return t['com.affine.recording.success.prompt']();
} else if (
status.status === 'ready' &&
status.blockCreationStatus === 'failed'
status.status === 'import_failed' ||
status.status === 'start_failed' ||
status.status === 'finalize_failed'
) {
return t['com.affine.recording.failed.prompt']();
} else if (
status.status === 'starting' ||
status.status === 'recording' ||
status.status === 'processing'
status.status === 'finalizing'
) {
if (status.appName) {
return t['com.affine.recording.recording']({
@@ -77,11 +85,19 @@ export function Recording() {
} else {
return t['com.affine.recording.recording.unnamed']();
}
} else if (
status.status === 'pending_import' ||
status.status === 'importing'
) {
return t['com.affine.recording.importing.prompt']();
}
return null;
}, [status, t]);
const handleDismiss = useAsyncCallback(async () => {
if (status) {
await apis?.recording?.dismissRecordingStatus(status.id);
}
await apis?.popup?.dismissCurrentRecording();
track.popup.$.recordingBar.dismissRecording({
type: 'Meeting record',
@@ -155,8 +171,10 @@ export function Recording() {
</Button>
);
} else if (
status.status === 'processing' ||
(status.status === 'ready' && !status.blockCreationStatus)
status.status === 'starting' ||
status.status === 'finalizing' ||
status.status === 'pending_import' ||
status.status === 'importing'
) {
return (
<Button
@@ -166,18 +184,21 @@ export function Recording() {
disabled
/>
);
} else if (
status.status === 'ready' &&
status.blockCreationStatus === 'success'
) {
} else if (status.status === 'imported') {
return (
<Button variant="primary" onClick={handleDismiss}>
{t['com.affine.recording.success.button']()}
</Button>
);
} else if (status.status === 'start_failed') {
return (
<Button variant="plain" onClick={handleDismiss}>
{t['com.affine.recording.dismiss']()}
</Button>
);
} else if (
status.status === 'ready' &&
status.blockCreationStatus === 'failed'
status.status === 'import_failed' ||
status.status === 'finalize_failed'
) {
return (
<>
packages/frontend/apps/electron/src/main/recording/coordinator.ts
+864
View File
@@ -0,0 +1,864 @@
import { BehaviorSubject, type Observable } from 'rxjs';
import { logger } from '../logger';
import { globalStateStorage } from '../shared-storage/storage';
import type {
AppGroupInfo,
RecordingArtifactInfo,
RecordingDisplayState,
RecordingImportStatus,
RecordingJobStatus,
RecordingStatus,
} from './types';
const RECORDING_JOBS_KEY = 'recordingJobs:v2';
const IMPORT_LEASE_MS = 30_000;
interface NativeRecordingMeta {
id: string;
filepath: string;
sampleRate: number;
channels: number;
startedAt?: number;
}
interface NativeRecordingArtifact {
id: string;
filepath: string;
sampleRate: number;
channels: number;
durationMs: number;
size: number;
degraded?: boolean;
overflowCount?: number;
}
export interface NativeRecordingController {
startRecording(options: {
appProcessId?: number;
outputDir: string;
format: 'opus';
id: string;
}): Promise<NativeRecordingMeta>;
stopRecording(nativeId: string): Promise<NativeRecordingArtifact>;
abortRecording(nativeId: string): Promise<void>;
}
function isObject(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === 'object';
}
function isArtifactInfo(value: unknown): value is RecordingArtifactInfo {
if (!isObject(value) || typeof value.filepath !== 'string') {
return false;
}
return (
(value.sampleRate === undefined || typeof value.sampleRate === 'number') &&
(value.numberOfChannels === undefined ||
typeof value.numberOfChannels === 'number') &&
(value.durationMs === undefined || typeof value.durationMs === 'number') &&
(value.size === undefined || typeof value.size === 'number') &&
(value.degraded === undefined || typeof value.degraded === 'boolean') &&
(value.overflowCount === undefined ||
typeof value.overflowCount === 'number')
);
}
function isRecordingJob(value: unknown): value is RecordingJobStatus {
if (!isObject(value)) {
return false;
}
if (
typeof value.id !== 'number' ||
typeof value.phase !== 'string' ||
typeof value.startTime !== 'number' ||
typeof value.createdAt !== 'number' ||
typeof value.updatedAt !== 'number'
) {
return false;
}
if (value.appName !== undefined && typeof value.appName !== 'string') {
return false;
}
if (value.appGroupId !== undefined && typeof value.appGroupId !== 'number') {
return false;
}
if (
value.bundleIdentifier !== undefined &&
typeof value.bundleIdentifier !== 'string'
) {
return false;
}
if (
value.appProcessId !== undefined &&
typeof value.appProcessId !== 'number'
) {
return false;
}
if (value.nativeId !== undefined && typeof value.nativeId !== 'string') {
return false;
}
if (value.artifact !== undefined && !isArtifactInfo(value.artifact)) {
return false;
}
if (value.import !== undefined) {
if (!isObject(value.import)) {
return false;
}
if (
value.import.workspaceId !== undefined &&
typeof value.import.workspaceId !== 'string'
) {
return false;
}
if (
value.import.docId !== undefined &&
typeof value.import.docId !== 'string'
) {
return false;
}
if (
value.import.errorMessage !== undefined &&
typeof value.import.errorMessage !== 'string'
) {
return false;
}
if (
value.import.leaseExpiresAt !== undefined &&
typeof value.import.leaseExpiresAt !== 'number'
) {
return false;
}
if (
value.import.startedAt !== undefined &&
typeof value.import.startedAt !== 'number'
) {
return false;
}
if (
value.import.finishedAt !== undefined &&
typeof value.import.finishedAt !== 'number'
) {
return false;
}
}
if (
value.error !== undefined &&
(!isObject(value.error) ||
typeof value.error.stage !== 'string' ||
typeof value.error.message !== 'string')
) {
return false;
}
if (
value.dismissedAt !== undefined &&
typeof value.dismissedAt !== 'number'
) {
return false;
}
return true;
}
function loadPersistedJobs() {
const persisted = globalStateStorage.get(RECORDING_JOBS_KEY);
if (!Array.isArray(persisted)) {
return [] as RecordingJobStatus[];
}
const now = Date.now();
return persisted.flatMap(value => {
if (!isRecordingJob(value)) {
return [];
}
if (
value.phase === 'new' ||
value.phase === 'starting' ||
value.phase === 'recording' ||
value.phase === 'finalizing' ||
value.phase === 'aborted'
) {
return [];
}
if (value.phase === 'recorded' || value.phase === 'importing') {
return [
{
...value,
phase: 'recorded' as const,
import: {
...value.import,
errorMessage: undefined,
leaseExpiresAt: undefined,
},
updatedAt: now,
dismissedAt: value.dismissedAt ?? now,
},
];
}
if (value.phase === 'imported' || value.phase === 'failed') {
return [{ ...value, dismissedAt: value.dismissedAt ?? now }];
}
return [value];
});
}
function toImportStatus(job: RecordingJobStatus): RecordingImportStatus | null {
if (!job.artifact) {
return null;
}
let importStatus: RecordingImportStatus['importStatus'];
switch (job.phase) {
case 'recorded':
importStatus = 'pending_import';
break;
case 'importing':
importStatus = 'importing';
break;
case 'imported':
importStatus = 'imported';
break;
case 'failed':
if (job.error?.stage !== 'import') {
return null;
}
importStatus = 'import_failed';
break;
default:
return null;
}
return {
id: job.id,
appName: job.appName,
workspaceId: job.import?.workspaceId,
docId: job.import?.docId,
startTime: job.startTime,
filepath: job.artifact.filepath,
sampleRate: job.artifact.sampleRate,
numberOfChannels: job.artifact.numberOfChannels,
durationMs: job.artifact.durationMs,
size: job.artifact.size,
degraded: job.artifact.degraded,
overflowCount: job.artifact.overflowCount,
importStatus,
errorMessage: job.error?.stage === 'import' ? job.error.message : undefined,
createdAt: job.createdAt,
updatedAt: job.updatedAt,
};
}
function toDisplayStatus(
job: RecordingJobStatus | undefined
): RecordingStatus | null {
if (!job || job.dismissedAt) {
return null;
}
let status: RecordingDisplayState | null = null;
switch (job.phase) {
case 'new':
case 'starting':
case 'recording':
case 'finalizing':
status = job.phase;
break;
case 'recorded':
status = 'pending_import';
break;
case 'importing':
status = 'importing';
break;
case 'imported':
status = 'imported';
break;
case 'failed':
if (job.error?.stage === 'start') {
status = 'start_failed';
} else if (job.error?.stage === 'finalize') {
status = 'finalize_failed';
} else {
status = 'import_failed';
}
break;
case 'aborted':
return null;
default:
return null;
}
return {
id: job.id,
status,
appName: job.appName,
appGroupId: job.appGroupId,
startTime: job.startTime,
filepath: job.artifact?.filepath,
sampleRate: job.artifact?.sampleRate,
numberOfChannels: job.artifact?.numberOfChannels,
durationMs: job.artifact?.durationMs,
size: job.artifact?.size,
degraded: job.artifact?.degraded,
overflowCount: job.artifact?.overflowCount,
errorMessage: job.error?.message,
};
}
function buildDocId(jobId: number) {
return `recording-${jobId}`;
}
export class RecordingCoordinator {
private readonly jobsSubject$ = new BehaviorSubject<RecordingJobStatus[]>(
loadPersistedJobs()
);
private readonly statusSubject$ = new BehaviorSubject<RecordingStatus | null>(
null
);
private readonly importQueueSubject$ = new BehaviorSubject<
RecordingImportStatus[]
>([]);
private nextId =
this.jobsSubject$.value.reduce((max, job) => Math.max(max, job.id), -1) + 1;
constructor(
private readonly outputDir: string,
private readonly resolveFilepath: (filepath: string) => Promise<string>,
private readonly getNativeController: () => Promise<NativeRecordingController>
) {
this.emit();
}
get jobs$(): Observable<RecordingJobStatus[]> {
return this.jobsSubject$.asObservable();
}
get status$(): Observable<RecordingStatus | null> {
return this.statusSubject$.asObservable();
}
get importQueue$(): Observable<RecordingImportStatus[]> {
return this.importQueueSubject$.asObservable();
}
get jobs() {
return this.jobsSubject$.value;
}
currentStatus() {
return this.statusSubject$.value;
}
importQueue() {
return this.importQueueSubject$.value;
}
activeJob() {
return this.jobs.find(
job =>
job.phase === 'starting' ||
job.phase === 'recording' ||
job.phase === 'finalizing'
);
}
createPrompt(appGroup?: AppGroupInfo) {
const matchingPrompt = this.jobs.find(
job =>
job.phase === 'new' &&
job.dismissedAt === undefined &&
job.appGroupId === appGroup?.processGroupId
);
if (matchingPrompt) {
return matchingPrompt;
}
const now = Date.now();
const runningApp = appGroup?.apps.find(app => app.isRunning);
const job: RecordingJobStatus = {
id: this.nextId++,
phase: 'new',
appName: appGroup?.name,
appGroupId: appGroup?.processGroupId,
bundleIdentifier: appGroup?.bundleIdentifier,
appProcessId: runningApp?.processId,
startTime: 0,
createdAt: now,
updatedAt: now,
};
this.setJobs([...this.jobs, job]);
return job;
}
async start(appGroup?: AppGroupInfo) {
const currentActive = this.activeJob();
if (currentActive) {
logger.error(
'Cannot start a new recording while another session is active'
);
return currentActive;
}
const now = Date.now();
const runningApp = appGroup?.apps.find(app => app.isRunning);
const matchingPrompt = this.jobs.find(
job =>
job.phase === 'new' &&
job.dismissedAt === undefined &&
job.appGroupId === appGroup?.processGroupId
);
const startingJob: RecordingJobStatus = matchingPrompt
? {
...matchingPrompt,
phase: 'starting',
appName: appGroup?.name ?? matchingPrompt.appName,
appGroupId: appGroup?.processGroupId ?? matchingPrompt.appGroupId,
bundleIdentifier:
appGroup?.bundleIdentifier ?? matchingPrompt.bundleIdentifier,
appProcessId: runningApp?.processId,
updatedAt: now,
dismissedAt: undefined,
error: undefined,
}
: {
id: this.nextId++,
phase: 'starting',
appName: appGroup?.name,
appGroupId: appGroup?.processGroupId,
bundleIdentifier: appGroup?.bundleIdentifier,
appProcessId: runningApp?.processId,
startTime: 0,
createdAt: now,
updatedAt: now,
};
this.upsertJob(startingJob);
let nativeId: string | undefined;
try {
logger.info(`recording ${startingJob.id} starting`);
const nativeController = await this.getNativeController();
const meta = await nativeController.startRecording({
appProcessId: startingJob.appProcessId,
outputDir: this.outputDir,
format: 'opus',
id: String(startingJob.id),
});
nativeId = meta.id;
const filepath = await this.resolveFilepath(meta.filepath);
const currentJob = this.findJob(startingJob.id);
if (!currentJob || currentJob.phase !== 'starting') {
if (nativeId) {
await nativeController.abortRecording(nativeId).catch(error => {
logger.error('failed to cleanup abandoned native recording', error);
});
}
return this.findJob(startingJob.id) ?? currentJob ?? null;
}
const nextJob: RecordingJobStatus = {
...currentJob,
phase: 'recording',
nativeId: meta.id,
startTime: meta.startedAt ?? Date.now(),
updatedAt: Date.now(),
artifact: {
filepath,
sampleRate: meta.sampleRate,
numberOfChannels: meta.channels,
},
};
this.upsertJob(nextJob);
logger.info(`recording ${startingJob.id} started`, {
nativeId: meta.id,
sampleRate: meta.sampleRate,
channels: meta.channels,
});
return nextJob;
} catch (error) {
if (nativeId) {
const nativeController = await this.getNativeController();
await nativeController.abortRecording(nativeId).catch(cleanupError => {
logger.error(
'failed to cleanup abandoned native recording',
cleanupError
);
});
}
const currentJob = this.findJob(startingJob.id);
if (currentJob && currentJob.phase === 'starting') {
this.upsertJob({
...currentJob,
phase: 'failed',
updatedAt: Date.now(),
error: {
stage: 'start',
message: error instanceof Error ? error.message : 'failed to start',
},
});
}
logger.error('failed to start recording', error);
return this.findJob(startingJob.id) ?? null;
}
}
async stop(id: number) {
const job = this.findJob(id);
if (!job || job.phase !== 'recording' || !job.nativeId) {
logger.error(`stopRecording: Recording ${id} not found`);
return job ?? null;
}
const finalizingJob: RecordingJobStatus = {
...job,
phase: 'finalizing',
updatedAt: Date.now(),
error: undefined,
};
this.upsertJob(finalizingJob);
try {
logger.info(`recording ${id} finalizing`, {
nativeId: job.nativeId,
});
const nativeController = await this.getNativeController();
const artifact = await nativeController.stopRecording(job.nativeId);
const filepath = await this.resolveFilepath(artifact.filepath);
const currentJob = this.findJob(id);
if (!currentJob || currentJob.phase !== 'finalizing') {
return currentJob ?? null;
}
const nextJob: RecordingJobStatus = {
...currentJob,
phase: 'recorded',
nativeId: undefined,
updatedAt: Date.now(),
artifact: {
filepath,
sampleRate: artifact.sampleRate,
numberOfChannels: artifact.channels,
durationMs: artifact.durationMs,
size: artifact.size,
degraded: artifact.degraded,
overflowCount: artifact.overflowCount,
},
import: {
...currentJob.import,
errorMessage: undefined,
leaseExpiresAt: undefined,
},
};
this.upsertJob(nextJob);
logger.info(`recording ${id} finalized`, {
filepath,
degraded: artifact.degraded,
overflowCount: artifact.overflowCount,
});
return nextJob;
} catch (error) {
logger.error('Failed to stop recording', error);
const currentJob = this.findJob(id);
if (currentJob && currentJob.phase === 'finalizing') {
this.upsertJob({
...currentJob,
phase: 'failed',
updatedAt: Date.now(),
error: {
stage: 'finalize',
message:
error instanceof Error ? error.message : 'failed to finalize',
},
});
}
return this.findJob(id) ?? null;
}
}
async abortActive() {
const job =
this.activeJob() ??
[...this.jobs]
.sort((left, right) => right.updatedAt - left.updatedAt)
.find(
entry =>
!!entry.nativeId &&
(entry.phase === 'starting' ||
entry.phase === 'recording' ||
entry.phase === 'finalizing')
);
if (!job) {
return;
}
if (!job.nativeId) {
this.upsertJob({
...job,
phase: 'aborted',
updatedAt: Date.now(),
dismissedAt: Date.now(),
});
return;
}
const nativeController = await this.getNativeController();
try {
await nativeController.abortRecording(job.nativeId);
} finally {
this.removeJob(job.id);
}
}
getRecording(id: number) {
const job = this.findJob(id);
if (!job) {
return;
}
return {
id,
startTime: job.startTime,
filepath: job.artifact?.filepath,
sampleRate: job.artifact?.sampleRate,
numberOfChannels: job.artifact?.numberOfChannels,
appGroup: job.appGroupId
? {
processGroupId: job.appGroupId,
apps: [],
name: job.appName ?? '',
bundleIdentifier: job.bundleIdentifier ?? '',
icon: undefined,
isRunning: false,
}
: undefined,
app:
job.appProcessId &&
job.appName &&
job.appGroupId &&
job.bundleIdentifier
? {
info: {} as never,
isRunning: false,
processId: job.appProcessId,
processGroupId: job.appGroupId,
bundleIdentifier: job.bundleIdentifier,
name: job.appName,
}
: undefined,
};
}
claimImport(id: number, workspaceId: string) {
const now = Date.now();
let claimed: RecordingJobStatus | null = null;
this.setJobs(
this.jobs.map(job => {
if (job.id !== id || !job.artifact) {
return job;
}
if (job.import?.workspaceId && job.import.workspaceId !== workspaceId) {
return job;
}
if (job.phase === 'recorded') {
claimed = {
...job,
phase: 'importing',
updatedAt: now,
import: {
...job.import,
workspaceId,
docId: job.import?.docId ?? buildDocId(job.id),
errorMessage: undefined,
startedAt: job.import?.startedAt ?? now,
leaseExpiresAt: now + IMPORT_LEASE_MS,
},
dismissedAt: undefined,
};
return claimed;
}
if (
job.phase === 'importing' &&
(!job.import?.leaseExpiresAt || job.import.leaseExpiresAt <= now)
) {
claimed = {
...job,
updatedAt: now,
import: {
...job.import,
workspaceId,
docId: job.import?.docId ?? buildDocId(job.id),
errorMessage: undefined,
startedAt: job.import?.startedAt ?? now,
leaseExpiresAt: now + IMPORT_LEASE_MS,
},
dismissedAt: undefined,
};
return claimed;
}
return job;
})
);
return claimed ? toImportStatus(claimed) : null;
}
completeImport(id: number) {
const job = this.findJob(id);
if (!job || (job.phase !== 'recorded' && job.phase !== 'importing')) {
logger.error(`Recording import ${id} not found`);
return null;
}
const nextJob: RecordingJobStatus = {
...job,
phase: 'imported',
updatedAt: Date.now(),
import: {
...job.import,
errorMessage: undefined,
leaseExpiresAt: undefined,
finishedAt: Date.now(),
},
error: undefined,
dismissedAt: undefined,
};
this.upsertJob(nextJob);
return toImportStatus(nextJob);
}
failImport(id: number, errorMessage?: string) {
const job = this.findJob(id);
if (!job || (job.phase !== 'recorded' && job.phase !== 'importing')) {
logger.error(`Recording import ${id} not found`);
return null;
}
const nextJob: RecordingJobStatus = {
...job,
phase: 'failed',
updatedAt: Date.now(),
import: {
...job.import,
errorMessage,
leaseExpiresAt: undefined,
},
error: {
stage: 'import',
message: errorMessage ?? 'failed to import recording',
},
dismissedAt: undefined,
};
this.upsertJob(nextJob);
return toImportStatus(nextJob);
}
dismiss(id: number) {
const job = this.findJob(id);
if (!job) {
return null;
}
if (job.phase === 'imported') {
this.removeJob(id);
return this.currentStatus();
}
if (
job.phase === 'new' ||
(job.phase === 'failed' && job.error?.stage !== 'import')
) {
this.removeJob(id);
return this.currentStatus();
}
this.upsertJob({
...job,
updatedAt: Date.now(),
dismissedAt: Date.now(),
});
return this.currentStatus();
}
remove(id: number) {
this.removeJob(id);
}
private findJob(id: number) {
return this.jobs.find(job => job.id === id) ?? null;
}
private upsertJob(job: RecordingJobStatus) {
const nextJobs = this.jobs.filter(entry => entry.id !== job.id);
nextJobs.push(job);
nextJobs.sort((left, right) => left.id - right.id);
this.setJobs(nextJobs);
}
private removeJob(id: number) {
this.setJobs(this.jobs.filter(job => job.id !== id));
}
private setJobs(jobs: RecordingJobStatus[]) {
this.jobsSubject$.next(jobs);
globalStateStorage.set(RECORDING_JOBS_KEY, jobs);
this.emit();
}
private emit() {
const visibleJobs = this.jobs.filter(job => !job.dismissedAt);
const current =
visibleJobs.find(
job =>
job.phase === 'new' ||
job.phase === 'starting' ||
job.phase === 'recording' ||
job.phase === 'finalizing'
) ??
[...visibleJobs]
.sort((left, right) => right.updatedAt - left.updatedAt)
.find(
job =>
job.phase === 'recorded' ||
job.phase === 'importing' ||
job.phase === 'imported' ||
job.phase === 'failed'
);
this.statusSubject$.next(toDisplayStatus(current));
this.importQueueSubject$.next(
this.jobs
.flatMap(job => {
const status = toImportStatus(job);
if (!status) {
return [];
}
return status.importStatus === 'pending_import' ||
status.importStatus === 'importing'
? [status]
: [];
})
.sort((left, right) => {
if (left.updatedAt !== right.updatedAt) {
return right.updatedAt - left.updatedAt;
}
return right.id - left.id;
})
);
}
}
packages/frontend/apps/electron/src/main/recording/feature.ts
+247 -269
View File
@@ -3,7 +3,6 @@ import { execSync } from 'node:child_process';
import fsp from 'node:fs/promises';
import path from 'node:path';
// Should not load @affine/native for unsupported platforms
import type * as NativeModuleType from '@affine/native';
import { app, systemPreferences } from 'electron';
import fs from 'fs-extra';
@@ -14,6 +13,7 @@ import {
groupBy,
interval,
mergeMap,
type Observable,
Subject,
throttleTime,
} from 'rxjs';
@@ -35,8 +35,14 @@ import { globalStateStorage } from '../shared-storage/storage';
import { getMainWindow } from '../windows-manager';
import { popupManager } from '../windows-manager/popup';
import { isAppNameAllowed } from './allow-list';
import { recordingStateMachine } from './state-machine';
import type { AppGroupInfo, RecordingStatus, TappableAppInfo } from './types';
import { RecordingCoordinator } from './coordinator';
import type {
AppGroupInfo,
RecordingImportStatus,
RecordingJobStatus,
RecordingStatus,
TappableAppInfo,
} from './types';
export const MeetingsSettingsState = {
$: globalStateStorage.watch<MeetingSettingsSchema>(MeetingSettingsKey).pipe(
@@ -62,8 +68,6 @@ type Subscriber = {
const subscribers: Subscriber[] = [];
let appStateSubscribers: Subscriber[] = [];
// recordings are saved in the app data directory
// may need a way to clean up old recordings
export const SAVED_RECORDINGS_DIR = path.join(
app.getPath('sessionData'),
'recordings'
@@ -74,16 +78,46 @@ type ShareableContentType = InstanceType<NativeModule['ShareableContent']>;
type ShareableContentStatic = NativeModule['ShareableContent'];
let shareableContent: ShareableContentType | null = null;
let nativeModuleOverride: NativeModule | null = null;
function getNativeModule(): NativeModule {
return require('@affine/native') as NativeModule;
return nativeModuleOverride ?? (require('@affine/native') as NativeModule);
}
function cleanup() {
const nativeId = recordingStateMachine.status?.nativeId;
if (nativeId) cleanupAbandonedNativeRecording(nativeId);
recordingStatus$.next(null);
shareableContent = null;
async function getNativeModuleAsync(): Promise<NativeModule> {
if (nativeModuleOverride) {
return nativeModuleOverride;
}
return (await import('@affine/native')) as NativeModule;
}
async function assertRecordingFilepath(filepath: string) {
return await resolveExistingPathInBase(SAVED_RECORDINGS_DIR, filepath, {
caseInsensitive: isWindows(),
label: 'recording filepath',
});
}
const recordingCoordinator = new RecordingCoordinator(
SAVED_RECORDINGS_DIR,
assertRecordingFilepath,
async () => {
const nativeModule = await getNativeModuleAsync();
return {
startRecording: nativeModule.startRecording,
stopRecording: nativeModule.stopRecording,
abortRecording: nativeModule.abortRecording,
};
}
);
export function setRecordingNativeModuleForTesting(
nativeModule: NativeModule | null
) {
nativeModuleOverride = nativeModule;
}
function resetFeatureSubscriptions() {
appStateSubscribers.forEach(subscriber => {
try {
subscriber.unsubscribe();
@@ -100,44 +134,69 @@ function cleanup() {
}
});
subscribers.length = 0;
shareableContent = null;
applications$.next([]);
appGroups$.next([]);
}
async function abortActiveRecording() {
try {
await recordingCoordinator.abortActive();
} catch (error) {
logger.error('failed to cleanup abandoned native recording', error);
}
}
beforeAppQuit(() => {
cleanup();
void abortActiveRecording().catch(() => undefined);
resetFeatureSubscriptions();
});
export const applications$ = new BehaviorSubject<TappableAppInfo[]>([]);
export const appGroups$ = new BehaviorSubject<AppGroupInfo[]>([]);
export const updateApplicationsPing$ = new Subject<number>();
// There should be only one active recording at a time; state is managed by the state machine
export const recordingStatus$ = recordingStateMachine.status$;
export const recordingStatus$: Observable<RecordingStatus | null> =
recordingCoordinator.status$;
export const recordingImportQueue$: Observable<RecordingImportStatus[]> =
recordingCoordinator.importQueue$;
function isRecordingSettled(
export function getCurrentRecordingStatus() {
return recordingCoordinator.currentStatus();
}
function hasActivePopupStatus(status: RecordingStatus | null | undefined) {
return (
status?.status === 'starting' ||
status?.status === 'recording' ||
status?.status === 'finalizing'
);
}
function isTerminalPopupStatus(
status: RecordingStatus | null | undefined
): status is RecordingStatus & {
status: 'ready';
blockCreationStatus: 'success' | 'failed';
status: 'imported' | 'import_failed' | 'start_failed' | 'finalize_failed';
} {
return status?.status === 'ready' && status.blockCreationStatus !== undefined;
return (
status?.status === 'imported' ||
status?.status === 'import_failed' ||
status?.status === 'start_failed' ||
status?.status === 'finalize_failed'
);
}
function createAppGroup(processGroupId: number): AppGroupInfo | undefined {
// MUST require dynamically to avoid loading @affine/native for unsupported platforms
const SC: ShareableContentStatic = getNativeModule().ShareableContent;
const groupProcess = SC?.applicationWithProcessId(processGroupId);
if (!groupProcess) {
return;
}
return {
processGroupId: processGroupId,
apps: [], // leave it empty for now.
processGroupId,
apps: [],
name: groupProcess.name,
bundleIdentifier: groupProcess.bundleIdentifier,
// icon should be lazy loaded
get icon() {
try {
return groupProcess.icon;
@@ -150,32 +209,30 @@ function createAppGroup(processGroupId: number): AppGroupInfo | undefined {
};
}
// pipe applications$ to appGroups$
function setupAppGroups() {
subscribers.push(
applications$.pipe(distinctUntilChanged()).subscribe(apps => {
const appGroups: AppGroupInfo[] = [];
apps.forEach(app => {
let appGroup = appGroups.find(
group => group.processGroupId === app.processGroupId
const groups: AppGroupInfo[] = [];
apps.forEach(appInfo => {
let group = groups.find(
entry => entry.processGroupId === appInfo.processGroupId
);
if (!appGroup) {
appGroup = createAppGroup(app.processGroupId);
if (appGroup) {
appGroups.push(appGroup);
if (!group) {
group = createAppGroup(appInfo.processGroupId);
if (group) {
groups.push(group);
}
}
if (appGroup) {
appGroup.apps.push(app);
if (group) {
group.apps.push(appInfo);
}
});
appGroups.forEach(appGroup => {
appGroup.isRunning = appGroup.apps.some(app => app.isRunning);
groups.forEach(group => {
group.isRunning = group.apps.some(app => app.isRunning);
});
appGroups$.next(appGroups);
appGroups$.next(groups);
})
);
}
@@ -193,7 +250,7 @@ function setupNewRunningAppGroup() {
);
appGroups$.value.forEach(group => {
const recordingStatus = recordingStatus$.value;
const recordingStatus = getCurrentRecordingStatus();
if (
group.isRunning &&
(!recordingStatus || recordingStatus.status === 'new')
@@ -207,7 +264,7 @@ function setupNewRunningAppGroup() {
group => group.processGroupId === appGroup.processGroupId
);
if (currentGroup?.isRunning) {
startRecording(currentGroup).catch(err => {
void startRecording(currentGroup).catch(err => {
logger.error('failed to start recording', err);
});
}
@@ -225,47 +282,35 @@ function setupNewRunningAppGroup() {
return;
}
const recordingStatus = recordingStatus$.value;
const recordingStatus = getCurrentRecordingStatus();
if (currentGroup.isRunning) {
// when the app is running and there is no active recording popup
// we should show a new recording popup
if (
!recordingStatus ||
recordingStatus.status === 'new' ||
isRecordingSettled(recordingStatus)
!hasActivePopupStatus(recordingStatus)
) {
if (MeetingsSettingsState.value.recordingMode === 'prompt') {
newRecording(currentGroup);
} else if (
MeetingsSettingsState.value.recordingMode === 'auto-start'
) {
// there is a case that the watched app's running state changed rapidly
// we will schedule the start recording to avoid that
debounceStartRecording(currentGroup);
} else {
// do nothing, skip
}
}
} else {
// when displaying in "new" state but the app is not running any more
// we should remove the recording
if (
recordingStatus?.status === 'new' &&
currentGroup.bundleIdentifier ===
recordingStatus.appGroup?.bundleIdentifier
recordingStatus.appGroupId === currentGroup.processGroupId
) {
removeRecording(recordingStatus.id);
}
// if the watched app stops while we are recording it,
// we should stop the recording
if (
recordingStatus?.status === 'recording' &&
recordingStatus.appGroup?.bundleIdentifier ===
currentGroup.bundleIdentifier
recordingStatus.appGroupId === currentGroup.processGroupId
) {
stopRecording(recordingStatus.id).catch(err => {
void stopRecording(recordingStatus.id).catch(err => {
logger.error('failed to stop recording', err);
});
}
@@ -275,28 +320,9 @@ function setupNewRunningAppGroup() {
}
export async function getRecording(id: number) {
const recording = recordingStateMachine.status;
if (!recording || recording.id !== id) {
logger.error(`Recording ${id} not found`);
return;
}
return {
id,
appGroup: recording.appGroup,
app: recording.app,
startTime: recording.startTime,
filepath: recording.filepath,
sampleRate: recording.sampleRate,
numberOfChannels: recording.numberOfChannels,
};
return recordingCoordinator.getRecording(id);
}
// recording popup status
// new: waiting for user confirmation
// recording: native recording is ongoing
// processing: native stop or renderer import/transcription is ongoing
// ready + blockCreationStatus: post-processing finished
// null: hide popup
function setupRecordingListeners() {
subscribers.push(
recordingStatus$
@@ -305,30 +331,33 @@ function setupRecordingListeners() {
const popup = popupManager.get('recording');
if (status && !popup.showing) {
popup.show().catch(err => {
void popup.show().catch(err => {
logger.error('failed to show recording popup', err);
});
}
if (isRecordingSettled(status)) {
// show the popup for 10s
if (isTerminalPopupStatus(status)) {
setTimeout(
() => {
const currentStatus = recordingStatus$.value;
const currentStatus = getCurrentRecordingStatus();
if (
isRecordingSettled(currentStatus) &&
isTerminalPopupStatus(currentStatus) &&
currentStatus.id === status.id
) {
popup.hide().catch(err => {
void popup.hide().catch(err => {
logger.error('failed to hide recording popup', err);
});
dismissRecordingStatus(status.id);
}
},
status.blockCreationStatus === 'failed' ? 30_000 : 10_000
status.status === 'import_failed' ||
status.status === 'start_failed' ||
status.status === 'finalize_failed'
? 30_000
: 10_000
);
} else if (!status) {
// status is removed, we should hide the popup
popupManager
void popupManager
.get('recording')
.hide()
.catch(err => {
@@ -344,20 +373,16 @@ function getAllApps(): TappableAppInfo[] {
return [];
}
// MUST require dynamically to avoid loading @affine/native for unsupported platforms
const { ShareableContent } = getNativeModule();
const apps = ShareableContent.applications().map(app => {
const apps = ShareableContent.applications().map(appInfo => {
try {
// Check if this process is actively using microphone/audio
const isRunning = ShareableContent.isUsingMicrophone(app.processId);
const isRunning = ShareableContent.isUsingMicrophone(appInfo.processId);
return {
info: app,
processId: app.processId,
processGroupId: app.processGroupId,
bundleIdentifier: app.bundleIdentifier,
name: app.name,
info: appInfo,
processId: appInfo.processId,
processGroupId: appInfo.processGroupId,
bundleIdentifier: appInfo.bundleIdentifier,
name: appInfo.name,
isRunning,
};
} catch (error) {
@@ -366,14 +391,13 @@ function getAllApps(): TappableAppInfo[] {
}
});
const filteredApps = apps.filter(
(v): v is TappableAppInfo =>
v !== null &&
!v.bundleIdentifier.startsWith('com.apple') &&
!v.bundleIdentifier.startsWith('pro.affine') &&
v.processId !== process.pid
return apps.filter(
(value): value is TappableAppInfo =>
value !== null &&
!value.bundleIdentifier.startsWith('com.apple') &&
!value.bundleIdentifier.startsWith('pro.affine') &&
value.processId !== process.pid
);
return filteredApps;
}
function setupMediaListeners() {
@@ -402,25 +426,22 @@ function setupMediaListeners() {
// ignore unsubscribe error
}
});
const _appStateSubscribers: Subscriber[] = [];
apps.forEach(app => {
appStateSubscribers = apps.flatMap(appInfo => {
try {
const applicationInfo = app.info;
_appStateSubscribers.push(
ShareableContent.onAppStateChanged(applicationInfo, () => {
return [
ShareableContent.onAppStateChanged(appInfo.info, () => {
updateApplicationsPing$.next(Date.now());
})
);
}),
];
} catch (error) {
logger.error(
`Failed to set up app state listener for ${app.name}`,
`Failed to set up app state listener for ${appInfo.name}`,
error
);
return [];
}
});
appStateSubscribers = _appStateSubscribers;
})
);
}
@@ -431,7 +452,6 @@ function askForScreenRecordingPermission() {
}
try {
const ShareableContent = getNativeModule().ShareableContent;
// this will trigger the permission prompt
new ShareableContent();
return true;
} catch (error) {
@@ -440,7 +460,6 @@ function askForScreenRecordingPermission() {
return false;
}
// will be called when the app is ready or when the user has enabled the recording feature in settings
export function setupRecordingFeature() {
if (!MeetingsSettingsState.value.enabled || !checkCanRecordMeeting()) {
return;
@@ -452,8 +471,6 @@ export function setupRecordingFeature() {
shareableContent = new ShareableContent();
setupMediaListeners();
}
// reset all states
recordingStatus$.next(null);
setupAppGroups();
setupNewRunningAppGroup();
setupRecordingListeners();
@@ -464,8 +481,9 @@ export function setupRecordingFeature() {
}
}
export function disableRecordingFeature() {
cleanup();
export async function disableRecordingFeature() {
await abortActiveRecording();
resetFeatureSubscriptions();
}
function normalizeAppGroupInfo(
@@ -479,108 +497,22 @@ function normalizeAppGroupInfo(
export function newRecording(
appGroup?: AppGroupInfo | number
): RecordingStatus | null {
return recordingStateMachine.dispatch({
type: 'NEW_RECORDING',
appGroup: normalizeAppGroupInfo(appGroup),
});
recordingCoordinator.createPrompt(normalizeAppGroupInfo(appGroup));
return serializeRecordingStatus(getCurrentRecordingStatus());
}
export async function startRecording(
appGroup?: AppGroupInfo | number
): Promise<RecordingStatus | null> {
const previousState = recordingStateMachine.status;
const state = recordingStateMachine.dispatch({
type: 'START_RECORDING',
appGroup: normalizeAppGroupInfo(appGroup),
});
if (!state || state.status !== 'recording' || state === previousState) {
return state;
}
let nativeId: string | undefined;
try {
fs.ensureDirSync(SAVED_RECORDINGS_DIR);
const meta = getNativeModule().startRecording({
appProcessId: state.app?.processId,
outputDir: SAVED_RECORDINGS_DIR,
format: 'opus',
id: String(state.id),
});
nativeId = meta.id;
const filepath = await assertRecordingFilepath(meta.filepath);
const nextState = recordingStateMachine.dispatch({
type: 'ATTACH_NATIVE_RECORDING',
id: state.id,
nativeId: meta.id,
startTime: meta.startedAt ?? state.startTime,
filepath,
sampleRate: meta.sampleRate,
numberOfChannels: meta.channels,
});
if (!nextState || nextState.nativeId !== meta.id) {
throw new Error('Failed to attach native recording metadata');
}
return nextState;
} catch (error) {
if (nativeId) {
cleanupAbandonedNativeRecording(nativeId);
}
logger.error('failed to start recording', error);
return setRecordingBlockCreationStatus(
state.id,
'failed',
error instanceof Error ? error.message : undefined
);
}
fs.ensureDirSync(SAVED_RECORDINGS_DIR);
const job = await recordingCoordinator.start(normalizeAppGroupInfo(appGroup));
return serializeJob(job);
}
export async function stopRecording(id: number) {
const recording = recordingStateMachine.status;
if (!recording || recording.id !== id) {
logger.error(`stopRecording: Recording ${id} not found`);
return;
}
if (!recording.nativeId) {
logger.error(`stopRecording: Recording ${id} missing native id`);
return;
}
const processingState = recordingStateMachine.dispatch({
type: 'STOP_RECORDING',
id,
});
if (
!processingState ||
processingState.id !== id ||
processingState.status !== 'processing'
) {
return serializeRecordingStatus(processingState ?? recording);
}
try {
const artifact = getNativeModule().stopRecording(recording.nativeId);
const filepath = await assertRecordingFilepath(artifact.filepath);
const readyStatus = recordingStateMachine.dispatch({
type: 'ATTACH_RECORDING_ARTIFACT',
id,
filepath,
sampleRate: artifact.sampleRate,
numberOfChannels: artifact.channels,
});
if (!readyStatus) {
logger.error('No recording status to save');
return;
}
getMainWindow()
const job = await recordingCoordinator.stop(id);
if (job?.phase === 'recorded') {
void getMainWindow()
.then(mainWindow => {
if (mainWindow) {
mainWindow.show();
@@ -589,28 +521,8 @@ export async function stopRecording(id: number) {
.catch(err => {
logger.error('failed to bring up the window', err);
});
return serializeRecordingStatus(readyStatus);
} catch (error: unknown) {
logger.error('Failed to stop recording', error);
const recordingStatus = await setRecordingBlockCreationStatus(
id,
'failed',
error instanceof Error ? error.message : undefined
);
if (!recordingStatus) {
logger.error('No recording status to stop');
return;
}
return serializeRecordingStatus(recordingStatus);
}
}
async function assertRecordingFilepath(filepath: string) {
return await resolveExistingPathInBase(SAVED_RECORDINGS_DIR, filepath, {
caseInsensitive: isWindows(),
label: 'recording filepath',
});
return serializeRecordingStatus(getCurrentRecordingStatus());
}
export async function readRecordingFile(filepath: string) {
@@ -618,66 +530,133 @@ export async function readRecordingFile(filepath: string) {
return fsp.readFile(normalizedPath);
}
function cleanupAbandonedNativeRecording(nativeId: string) {
try {
const artifact = getNativeModule().stopRecording(nativeId);
void assertRecordingFilepath(artifact.filepath)
.then(filepath => {
fs.removeSync(filepath);
})
.catch(error => {
logger.error('failed to validate abandoned recording filepath', error);
});
} catch (error) {
logger.error('failed to cleanup abandoned native recording', error);
}
}
export async function setRecordingBlockCreationStatus(
id: number,
status: 'success' | 'failed',
errorMessage?: string
) {
return recordingStateMachine.dispatch({
type: 'SET_BLOCK_CREATION_STATUS',
id,
status,
errorMessage,
export function getRecordingImportQueue() {
return recordingCoordinator.importQueue().flatMap(status => {
const serialized = serializeRecordingImportStatus(status);
return serialized ? [serialized] : [];
});
}
export function claimRecordingImport(id: number, workspaceId: string) {
return serializeRecordingImportStatus(
recordingCoordinator.claimImport(id, workspaceId)
);
}
export function completeRecordingImport(id: number) {
logger.info(`recording import ${id} completed`);
return serializeRecordingImportStatus(
recordingCoordinator.completeImport(id)
);
}
export function failRecordingImport(id: number, errorMessage?: string) {
logger.error(`recording import ${id} failed`, errorMessage);
return serializeRecordingImportStatus(
recordingCoordinator.failImport(id, errorMessage)
);
}
export function dismissRecordingStatus(id: number) {
return serializeRecordingStatus(recordingCoordinator.dismiss(id));
}
export function removeRecording(id: number) {
recordingStateMachine.dispatch({ type: 'REMOVE_RECORDING', id });
recordingCoordinator.remove(id);
}
export interface SerializedRecordingStatus {
id: number;
status: RecordingStatus['status'];
blockCreationStatus?: RecordingStatus['blockCreationStatus'];
appName?: string;
// if there is no app group, it means the recording is for system audio
appGroupId?: number;
icon?: Buffer;
startTime: number;
filepath?: string;
sampleRate?: number;
numberOfChannels?: number;
durationMs?: number;
size?: number;
degraded?: boolean;
overflowCount?: number;
errorMessage?: string;
}
function serializeJob(job: RecordingJobStatus | null | undefined) {
if (!job) {
return null;
}
return serializeRecordingStatus(recordingCoordinator.currentStatus());
}
export function serializeRecordingStatus(
status: RecordingStatus
status: RecordingStatus | null | undefined
): SerializedRecordingStatus | null {
if (!status) {
return null;
}
return {
id: status.id,
status: status.status,
blockCreationStatus: status.blockCreationStatus,
appName: status.appGroup?.name,
appGroupId: status.appGroup?.processGroupId,
icon: status.appGroup?.icon,
appName: status.appName,
appGroupId: status.appGroupId,
icon: status.icon,
startTime: status.startTime,
filepath: status.filepath,
sampleRate: status.sampleRate,
numberOfChannels: status.numberOfChannels,
durationMs: status.durationMs,
size: status.size,
degraded: status.degraded,
overflowCount: status.overflowCount,
errorMessage: status.errorMessage,
};
}
export interface SerializedRecordingImportStatus {
id: number;
appName?: string;
workspaceId?: string;
docId?: string;
startTime: number;
filepath: string;
sampleRate?: number;
numberOfChannels?: number;
durationMs?: number;
size?: number;
degraded?: boolean;
overflowCount?: number;
importStatus: RecordingImportStatus['importStatus'];
errorMessage?: string;
createdAt: number;
updatedAt: number;
}
export function serializeRecordingImportStatus(
status: RecordingImportStatus | null | undefined
): SerializedRecordingImportStatus | null {
if (!status) {
return null;
}
return {
id: status.id,
appName: status.appName,
workspaceId: status.workspaceId,
docId: status.docId,
startTime: status.startTime,
filepath: status.filepath,
sampleRate: status.sampleRate,
numberOfChannels: status.numberOfChannels,
durationMs: status.durationMs,
size: status.size,
degraded: status.degraded,
overflowCount: status.overflowCount,
importStatus: status.importStatus,
errorMessage: status.errorMessage,
createdAt: status.createdAt,
updatedAt: status.updatedAt,
};
}
@@ -692,7 +671,6 @@ export const getMacOSVersion = () => {
}
};
// check if the system is MacOS and the version is >= 14.2
export const checkRecordingAvailable = () => {
if (isMacOS()) {
const version = getMacOSVersion();
packages/frontend/apps/electron/src/main/recording/index.ts
+57 -11
View File
@@ -11,15 +11,22 @@ import {
askForMeetingPermission,
checkMeetingPermissions,
checkRecordingAvailable,
claimRecordingImport,
completeRecordingImport,
disableRecordingFeature,
dismissRecordingStatus,
failRecordingImport,
getCurrentRecordingStatus,
getRecording,
getRecordingImportQueue,
readRecordingFile,
recordingImportQueue$,
recordingStatus$,
removeRecording,
SAVED_RECORDINGS_DIR,
type SerializedRecordingImportStatus,
type SerializedRecordingStatus,
serializeRecordingStatus,
setRecordingBlockCreationStatus,
setupRecordingFeature,
startRecording,
stopRecording,
@@ -32,9 +39,8 @@ export const recordingHandlers = {
},
getCurrentRecording: async () => {
// not all properties are serializable, so we need to return a subset of the status
return recordingStatus$.value
? serializeRecordingStatus(recordingStatus$.value)
: null;
const status = getCurrentRecordingStatus();
return status ? serializeRecordingStatus(status) : null;
},
startRecording: async (_, appGroup?: AppGroupInfo | number) => {
return startRecording(appGroup);
@@ -45,13 +51,20 @@ export const recordingHandlers = {
readRecordingFile: async (_, filepath: string) => {
return readRecordingFile(filepath);
},
setRecordingBlockCreationStatus: async (
_,
id: number,
status: 'success' | 'failed',
errorMessage?: string
) => {
return setRecordingBlockCreationStatus(id, status, errorMessage);
getRecordingImportQueue: async () => {
return getRecordingImportQueue();
},
claimRecordingImport: async (_, id: number, workspaceId: string) => {
return claimRecordingImport(id, workspaceId);
},
completeRecordingImport: async (_, id: number) => {
return completeRecordingImport(id);
},
dismissRecordingStatus: async (_, id: number) => {
return dismissRecordingStatus(id);
},
failRecordingImport: async (_, id: number, errorMessage?: string) => {
return failRecordingImport(id, errorMessage);
},
removeRecording: async (_, id: number) => {
return removeRecording(id);
@@ -108,4 +121,37 @@ export const recordingEvents = {
}
};
},
onRecordingImportQueueChanged: (
fn: (queue: SerializedRecordingImportStatus[]) => void
) => {
const sub = recordingImportQueue$.subscribe(queue => {
fn(
queue.map(item => ({
id: item.id,
appName: item.appName,
workspaceId: item.workspaceId,
docId: item.docId,
startTime: item.startTime,
filepath: item.filepath,
sampleRate: item.sampleRate,
numberOfChannels: item.numberOfChannels,
durationMs: item.durationMs,
size: item.size,
degraded: item.degraded,
overflowCount: item.overflowCount,
importStatus: item.importStatus,
errorMessage: item.errorMessage,
createdAt: item.createdAt,
updatedAt: item.updatedAt,
}))
);
});
return () => {
try {
sub.unsubscribe();
} catch {
// ignore unsubscribe error
}
};
},
};
packages/frontend/apps/electron/src/main/recording/state-machine.ts
-300
View File
@@ -1,300 +0,0 @@
import { BehaviorSubject } from 'rxjs';
import { shallowEqual } from '../../shared/utils';
import { logger } from '../logger';
import type { AppGroupInfo, RecordingStatus } from './types';
/**
* Recording state machine events
*/
export type RecordingEvent =
| { type: 'NEW_RECORDING'; appGroup?: AppGroupInfo }
| {
type: 'START_RECORDING';
appGroup?: AppGroupInfo;
}
| {
type: 'ATTACH_NATIVE_RECORDING';
id: number;
nativeId: string;
startTime: number;
filepath: string;
sampleRate: number;
numberOfChannels: number;
}
| {
type: 'STOP_RECORDING';
id: number;
}
| {
type: 'ATTACH_RECORDING_ARTIFACT';
id: number;
filepath: string;
sampleRate?: number;
numberOfChannels?: number;
}
| {
type: 'SET_BLOCK_CREATION_STATUS';
id: number;
status: 'success' | 'failed';
errorMessage?: string;
}
| { type: 'REMOVE_RECORDING'; id: number };
/**
* Recording State Machine
* Handles state transitions for the recording process
*/
export class RecordingStateMachine {
private recordingId = 0;
private readonly recordingStatus$ =
new BehaviorSubject<RecordingStatus | null>(null);
/**
* Get the current recording status
*/
get status(): RecordingStatus | null {
return this.recordingStatus$.value;
}
/**
* Get the BehaviorSubject for recording status
*/
get status$(): BehaviorSubject<RecordingStatus | null> {
return this.recordingStatus$;
}
/**
* Dispatch an event to the state machine
* @param event The event to dispatch
* @returns The new recording status after the event is processed
*/
dispatch(event: RecordingEvent, emit = true): RecordingStatus | null {
const currentStatus = this.recordingStatus$.value;
let newStatus: RecordingStatus | null = null;
switch (event.type) {
case 'NEW_RECORDING':
newStatus = this.handleNewRecording(event.appGroup);
break;
case 'START_RECORDING':
newStatus = this.handleStartRecording(event.appGroup);
break;
case 'ATTACH_NATIVE_RECORDING':
newStatus = this.handleAttachNativeRecording(event);
break;
case 'STOP_RECORDING':
newStatus = this.handleStopRecording(event.id);
break;
case 'ATTACH_RECORDING_ARTIFACT':
newStatus = this.handleAttachRecordingArtifact(
event.id,
event.filepath,
event.sampleRate,
event.numberOfChannels
);
break;
case 'SET_BLOCK_CREATION_STATUS':
newStatus = this.handleSetBlockCreationStatus(
event.id,
event.status,
event.errorMessage
);
break;
case 'REMOVE_RECORDING':
this.handleRemoveRecording(event.id);
newStatus = currentStatus?.id === event.id ? null : currentStatus;
break;
default:
logger.error('Unknown recording event type');
return currentStatus;
}
if (shallowEqual(newStatus, currentStatus)) {
return currentStatus;
}
if (emit) {
this.recordingStatus$.next(newStatus);
}
return newStatus;
}
/**
* Handle the NEW_RECORDING event
*/
private handleNewRecording(appGroup?: AppGroupInfo): RecordingStatus {
const recordingStatus: RecordingStatus = {
id: this.recordingId++,
status: 'new',
startTime: Date.now(),
app: appGroup?.apps.find(app => app.isRunning),
appGroup,
};
return recordingStatus;
}
/**
* Handle the START_RECORDING event
*/
private handleStartRecording(appGroup?: AppGroupInfo): RecordingStatus {
const currentStatus = this.recordingStatus$.value;
if (
currentStatus?.status === 'recording' ||
currentStatus?.status === 'processing'
) {
logger.error(
'Cannot start a new recording if there is already a recording'
);
return currentStatus;
}
if (
appGroup &&
currentStatus?.appGroup?.processGroupId === appGroup.processGroupId &&
currentStatus.status === 'new'
) {
return {
...currentStatus,
status: 'recording',
};
} else {
const newStatus = this.handleNewRecording(appGroup);
return {
...newStatus,
status: 'recording',
};
}
}
/**
* Attach native recording metadata to the current recording
*/
private handleAttachNativeRecording(
event: Extract<RecordingEvent, { type: 'ATTACH_NATIVE_RECORDING' }>
): RecordingStatus | null {
const currentStatus = this.recordingStatus$.value;
if (!currentStatus || currentStatus.id !== event.id) {
logger.error(`Recording ${event.id} not found for native attachment`);
return currentStatus;
}
if (currentStatus.status !== 'recording') {
logger.error(
`Cannot attach native metadata when recording is in ${currentStatus.status} state`
);
return currentStatus;
}
return {
...currentStatus,
nativeId: event.nativeId,
startTime: event.startTime,
filepath: event.filepath,
sampleRate: event.sampleRate,
numberOfChannels: event.numberOfChannels,
};
}
/**
* Handle the STOP_RECORDING event
*/
private handleStopRecording(id: number): RecordingStatus | null {
const currentStatus = this.recordingStatus$.value;
if (!currentStatus || currentStatus.id !== id) {
logger.error(`Recording ${id} not found for stopping`);
return currentStatus;
}
if (currentStatus.status !== 'recording') {
logger.error(`Cannot stop recording in ${currentStatus.status} state`);
return currentStatus;
}
return {
...currentStatus,
status: 'processing',
};
}
/**
* Attach the encoded artifact once native stop completes
*/
private handleAttachRecordingArtifact(
id: number,
filepath: string,
sampleRate?: number,
numberOfChannels?: number
): RecordingStatus | null {
const currentStatus = this.recordingStatus$.value;
if (!currentStatus || currentStatus.id !== id) {
logger.error(`Recording ${id} not found for saving`);
return currentStatus;
}
if (currentStatus.status !== 'processing') {
logger.error(`Cannot attach artifact in ${currentStatus.status} state`);
return currentStatus;
}
return {
...currentStatus,
filepath,
sampleRate: sampleRate ?? currentStatus.sampleRate,
numberOfChannels: numberOfChannels ?? currentStatus.numberOfChannels,
};
}
/**
* Set the renderer-side block creation result
*/
private handleSetBlockCreationStatus(
id: number,
status: 'success' | 'failed',
errorMessage?: string
): RecordingStatus | null {
const currentStatus = this.recordingStatus$.value;
if (!currentStatus || currentStatus.id !== id) {
logger.error(`Recording ${id} not found for block creation status`);
return currentStatus;
}
if (currentStatus.status === 'new') {
logger.error(`Cannot settle recording ${id} before it starts`);
return currentStatus;
}
if (
currentStatus.status === 'ready' &&
currentStatus.blockCreationStatus !== undefined
) {
return currentStatus;
}
if (errorMessage) {
logger.error(`Recording ${id} create block failed: ${errorMessage}`);
}
return {
...currentStatus,
status: 'ready',
blockCreationStatus: status,
};
}
/**
* Handle the REMOVE_RECORDING event
*/
private handleRemoveRecording(id: number): void {
// Actual recording removal logic would be handled by the caller
// This just ensures the state is updated correctly
logger.info(`Recording ${id} removed from state machine`);
}
}
// Create and export a singleton instance
export const recordingStateMachine = new RecordingStateMachine();
packages/frontend/apps/electron/src/main/recording/state-transitions.md
+57 -22
View File
@@ -1,35 +1,70 @@
# Recording State Transitions
The desktop recording flow now has a single linear engine state and a separate post-process result.
The desktop recording flow now uses two independent lifecycle models:
## Engine states
1. recording session state in Electron main, which tracks native capture/finalize.
2. artifact import state in Electron main, which tracks renderer-side doc import.
- `inactive`: no active recording
- `new`: app detected, waiting for user confirmation
- `recording`: native capture is running
- `processing`: native capture has stopped and the artifact is being imported
- `ready`: post-processing has finished
## Recording Session State
## Post-process result
- `inactive`: no session has been created yet.
- `new`: app detected, waiting for user confirmation.
- `starting`: native session setup is in progress.
- `start_failed`: native session setup failed.
- `recording`: native capture is running.
- `finalizing`: native stop/finalize is in progress.
- `finalized`: native finalized an artifact successfully.
- `finalize_failed`: native finalize failed.
`ready` recordings may carry `blockCreationStatus`:
Only `starting`, `recording`, and `finalizing` occupy the active native slot.
`start_failed`, `finalized`, and `finalize_failed` no longer block the next recording.
- `success`: the recording block was created successfully
- `failed`: the artifact was saved, but block creation/import failed
## Recording Artifact Import State
## State flow
- `pending_import`: artifact is finalized and durable in main, waiting for a renderer to consume it.
- `importing`: a renderer has claimed the artifact and is importing it into a doc.
- `imported`: doc import finished successfully.
- `import_failed`: doc import failed after import work began; the saved artifact remains available, but automatic import stops to avoid duplicate docs.
Artifacts are persisted in main process storage so renderer reloads or missing workspace context do not drop them.
## Session Flow
```text
inactive -> new -> recording -> processing -> ready
^ |
| |
+------ start ---------+
inactive -> new -> starting -> recording -> finalizing -> finalized
\ \
\ -> finalize_failed
-> start_failed
```
- `START_RECORDING` creates or reuses a pending `new` recording.
- `ATTACH_NATIVE_RECORDING` fills in native metadata while staying in `recording`.
- `STOP_RECORDING` moves the flow to `processing`.
- `ATTACH_RECORDING_ARTIFACT` attaches the finalized `.opus` artifact while staying in `processing`.
- `SET_BLOCK_CREATION_STATUS` settles the flow as `ready`.
- `START_RECORDING` creates or reuses a pending `new` recording and moves it to `starting`.
- `ATTACH_NATIVE_RECORDING` attaches native session metadata and moves the session to `recording`.
- `START_RECORDING_FAILED` keeps the session terminal with `start_failed`.
- `STOP_RECORDING` moves the session to `finalizing`.
- `ATTACH_RECORDING_ARTIFACT` marks the session `finalized` with the native artifact metadata.
- `FINALIZE_RECORDING_FAILED` marks the session `finalize_failed`.
- after enqueueing the artifact for renderer import, main clears the finalized session and lets the import registry become the sole source of truth.
Only one recording can be active at a time. A new recording can start only after the previous one has been removed or its `ready` result has been settled.
## Import Flow
```text
pending_import -> importing -> imported
\
-> import_failed
```
- main enqueues `pending_import` after native finalize succeeds.
- renderer claims the artifact, moving it to `importing`.
- renderer marks the artifact `imported` or `import_failed`.
- `imported` is not kept in the durable queue; it is projected as a transient popup status and then dropped.
- automatic retry only covers missing UI preconditions before import work begins; once doc creation starts, or completion cannot be persisted afterward, the entry stays `import_failed` to avoid duplicate docs.
## Popup Projection
The popup still renders a single current status, but it is now a projection:
- active session states map to `new`, `starting`, `start_failed`, `recording`, `finalizing`, `finalize_failed`.
- otherwise active import queue entries map to `pending_import` or `importing`.
- terminal import results (`imported`, `import_failed`) are shown through a transient popup projection instead of the durable queue.
This keeps the UI simple without collapsing the underlying source-of-truth back into a single overloaded `processing` state.
packages/frontend/apps/electron/src/main/recording/types.ts
+97 -13
View File
@@ -18,19 +18,103 @@ export interface AppGroupInfo {
isRunning: boolean;
}
export interface RecordingStatus {
id: number; // corresponds to the recording id
// an app group is detected and waiting for user confirmation
// recording: the native recorder is running
// processing: recording has stopped and the artifact is being prepared/imported
// ready: the post-processing result has been settled
status: 'new' | 'recording' | 'processing' | 'ready';
app?: TappableAppInfo;
appGroup?: AppGroupInfo;
startTime: number; // 0 means not started yet
filepath?: string; // encoded file path
nativeId?: string;
export type RecordingJobPhase =
| 'new'
| 'starting'
| 'recording'
| 'finalizing'
| 'recorded'
| 'importing'
| 'imported'
| 'failed'
| 'aborted';
export type RecordingFailureStage = 'start' | 'finalize' | 'import';
export interface RecordingFailureInfo {
stage: RecordingFailureStage;
message: string;
}
export type RecordingImportState =
| 'pending_import'
| 'importing'
| 'imported'
| 'import_failed';
export interface RecordingArtifactInfo {
filepath: string;
sampleRate?: number;
numberOfChannels?: number;
blockCreationStatus?: 'success' | 'failed';
durationMs?: number;
size?: number;
degraded?: boolean;
overflowCount?: number;
}
export interface RecordingImportProgress {
workspaceId?: string;
docId?: string;
errorMessage?: string;
leaseExpiresAt?: number;
startedAt?: number;
finishedAt?: number;
}
export interface RecordingJobStatus {
id: number;
phase: RecordingJobPhase;
appName?: string;
appGroupId?: number;
bundleIdentifier?: string;
appProcessId?: number;
nativeId?: string;
startTime: number;
createdAt: number;
updatedAt: number;
artifact?: RecordingArtifactInfo;
import?: RecordingImportProgress;
error?: RecordingFailureInfo;
dismissedAt?: number;
}
export interface RecordingImportStatus extends RecordingArtifactInfo {
id: number;
appName?: string;
workspaceId?: string;
docId?: string;
startTime: number;
importStatus: RecordingImportState;
errorMessage?: string;
createdAt: number;
updatedAt: number;
}
export type RecordingDisplayState =
| 'new'
| 'starting'
| 'start_failed'
| 'recording'
| 'finalizing'
| 'pending_import'
| 'importing'
| 'imported'
| 'import_failed'
| 'finalize_failed';
export interface RecordingStatus {
id: number;
status: RecordingDisplayState;
appName?: string;
appGroupId?: number;
icon?: Buffer;
startTime: number;
filepath?: string;
sampleRate?: number;
numberOfChannels?: number;
durationMs?: number;
size?: number;
degraded?: boolean;
overflowCount?: number;
errorMessage?: string;
}
packages/frontend/apps/electron/src/main/tray/index.ts
+16 -8
View File
@@ -17,8 +17,8 @@ import {
appGroups$,
checkCanRecordMeeting,
checkRecordingAvailable,
getCurrentRecordingStatus,
MeetingsSettingsState,
recordingStatus$,
startRecording,
stopRecording,
updateApplicationsPing$,
@@ -158,9 +158,14 @@ class TrayState implements Disposable {
appGroup => appGroup.isRunning
);
const recordingStatus = recordingStatus$.value;
const recordingStatus = getCurrentRecordingStatus();
if (!recordingStatus || recordingStatus.status !== 'recording') {
if (
!recordingStatus ||
(recordingStatus.status !== 'starting' &&
recordingStatus.status !== 'recording' &&
recordingStatus.status !== 'finalizing')
) {
const appMenuItems = runningAppGroups.map(appGroup => ({
label: appGroup.name,
icon: appGroup.icon || undefined,
@@ -197,8 +202,8 @@ class TrayState implements Disposable {
...appMenuItems
);
} else {
const recordingLabel = recordingStatus.appGroup?.name
? `Recording (${recordingStatus.appGroup?.name})`
const recordingLabel = recordingStatus.appName
? `Recording (${recordingStatus.appName})`
: 'Recording';
// recording is active
@@ -210,11 +215,14 @@ class TrayState implements Disposable {
},
{
label: 'Stop',
disabled: recordingStatus.status !== 'recording',
click: () => {
logger.info('User action: Stop Recording');
stopRecording(recordingStatus.id).catch(err => {
logger.error('Failed to stop recording:', err);
});
if (recordingStatus.status === 'recording') {
stopRecording(recordingStatus.id).catch(err => {
logger.error('Failed to stop recording:', err);
});
}
},
}
);
packages/frontend/apps/electron/test/main/recording-coordinator.spec.ts
+417
View File
@@ -0,0 +1,417 @@
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
const storageState = new Map<string, unknown>();
vi.mock('../../src/main/logger', () => ({
logger: {
error: vi.fn(),
info: vi.fn(),
},
}));
beforeEach(() => {
vi.resetModules();
vi.clearAllMocks();
storageState.clear();
vi.doMock('../../src/main/shared-storage/storage', () => ({
globalStateStorage: {
get: (key: string) => storageState.get(key),
set: (key: string, value: unknown) => {
storageState.set(key, value);
},
},
}));
});
afterEach(() => {
vi.useRealTimers();
});
async function createCoordinator(controllerOverrides?: {
startRecording?: ReturnType<typeof vi.fn>;
stopRecording?: ReturnType<typeof vi.fn>;
abortRecording?: ReturnType<typeof vi.fn>;
}) {
const { RecordingCoordinator } =
await import('../../src/main/recording/coordinator');
const controller = {
startRecording: vi.fn(),
stopRecording: vi.fn(),
abortRecording: vi.fn(),
...controllerOverrides,
};
return {
coordinator: new RecordingCoordinator(
'/tmp',
async filepath => filepath,
async () => controller
),
controller,
};
}
describe('RecordingCoordinator', () => {
test('disabling can await native abort before allowing another start', async () => {
const startRecording = vi.fn().mockResolvedValue({
id: 'native-1',
filepath: '/tmp/0.opus',
sampleRate: 48_000,
channels: 2,
startedAt: 123,
});
const stopRecording = vi.fn();
let releaseAbort!: () => void;
const abortRecording = vi.fn(
() =>
new Promise<void>(resolve => {
releaseAbort = resolve;
})
);
const { coordinator } = await createCoordinator({
startRecording,
stopRecording,
abortRecording,
});
await coordinator.start();
const abortPromise = coordinator.abortActive();
await Promise.resolve();
expect(startRecording).toHaveBeenCalledTimes(1);
expect(abortRecording).toHaveBeenCalledWith('native-1');
releaseAbort();
await abortPromise;
await coordinator.start();
expect(startRecording).toHaveBeenCalledTimes(2);
});
test('claiming an import binds it to a workspace and stable doc id', async () => {
storageState.set('recordingJobs:v2', [
{
id: 7,
phase: 'recorded',
appName: 'Zoom',
startTime: 1000,
createdAt: 1,
updatedAt: 1,
artifact: {
filepath: '/tmp/meeting.opus',
sampleRate: 48_000,
numberOfChannels: 2,
},
},
]);
const { coordinator } = await createCoordinator();
const claimed = coordinator.claimImport(7, 'workspace-1');
expect(claimed).toMatchObject({
id: 7,
workspaceId: 'workspace-1',
docId: 'recording-7',
importStatus: 'importing',
});
expect(coordinator.importQueue()).toEqual([
expect.objectContaining({
id: 7,
workspaceId: 'workspace-1',
docId: 'recording-7',
importStatus: 'importing',
}),
]);
});
test('reopens interrupted imports as pending work while suppressing stale terminal popups', async () => {
storageState.set('recordingJobs:v2', [
{
id: 1,
phase: 'importing',
appName: 'Zoom',
startTime: 1000,
createdAt: 1,
updatedAt: 1,
artifact: {
filepath: '/tmp/interrupted.opus',
},
import: {
workspaceId: 'workspace-1',
docId: 'recording-1',
leaseExpiresAt: Date.now() + 10_000,
},
},
{
id: 2,
phase: 'imported',
appName: 'Meet',
startTime: 2000,
createdAt: 2,
updatedAt: 2,
artifact: {
filepath: '/tmp/imported.opus',
},
},
{
id: 3,
phase: 'failed',
appName: 'Teams',
startTime: 3000,
createdAt: 3,
updatedAt: 3,
artifact: {
filepath: '/tmp/failed.opus',
},
error: {
stage: 'import',
message: 'import failed',
},
},
]);
const { coordinator } = await createCoordinator();
expect(coordinator.importQueue()).toEqual([
expect.objectContaining({
id: 1,
importStatus: 'pending_import',
}),
]);
expect(coordinator.currentStatus()).toBeNull();
});
test('abortActive clears the local job even when native abort fails', async () => {
const startRecording = vi.fn().mockResolvedValue({
id: 'native-1',
filepath: '/tmp/0.opus',
sampleRate: 48_000,
channels: 2,
startedAt: 123,
});
const abortRecording = vi
.fn()
.mockRejectedValue(new Error('native abort failed'));
const { coordinator } = await createCoordinator({
startRecording,
abortRecording,
});
await coordinator.start();
await expect(coordinator.abortActive()).rejects.toThrow(
'native abort failed'
);
expect(coordinator.currentStatus()).toBeNull();
expect(coordinator.jobs).toEqual([]);
});
test('start failures project to start_failed and release the active slot', async () => {
const startRecording = vi
.fn()
.mockRejectedValueOnce(new Error('native start failed'))
.mockResolvedValueOnce({
id: 'native-2',
filepath: '/tmp/1.opus',
sampleRate: 48_000,
channels: 2,
startedAt: 456,
});
const { coordinator } = await createCoordinator({
startRecording,
});
const failed = await coordinator.start();
expect(failed).toMatchObject({
phase: 'failed',
error: {
stage: 'start',
message: 'native start failed',
},
});
expect(coordinator.currentStatus()).toMatchObject({
status: 'start_failed',
errorMessage: 'native start failed',
});
const next = await coordinator.start();
expect(next).toMatchObject({
id: 1,
phase: 'recording',
});
});
test('stop failures project to finalize_failed and release the active slot', async () => {
const startRecording = vi
.fn()
.mockResolvedValueOnce({
id: 'native-1',
filepath: '/tmp/0.opus',
sampleRate: 48_000,
channels: 2,
startedAt: 123,
})
.mockResolvedValueOnce({
id: 'native-2',
filepath: '/tmp/1.opus',
sampleRate: 48_000,
channels: 2,
startedAt: 456,
});
const stopRecording = vi
.fn()
.mockRejectedValueOnce(new Error('native stop failed'));
const { coordinator } = await createCoordinator({
startRecording,
stopRecording,
});
const first = await coordinator.start();
await coordinator.stop(first!.id);
expect(coordinator.currentStatus()).toMatchObject({
id: first!.id,
status: 'finalize_failed',
errorMessage: 'native stop failed',
});
const next = await coordinator.start();
expect(next).toMatchObject({
id: 1,
phase: 'recording',
});
});
test.each([
{
name: 'successful import completion',
settleImport: (coordinator: {
completeImport: (id: number) => unknown;
}) => coordinator.completeImport(7),
expectedStatus: 'imported',
},
{
name: 'failed import completion',
settleImport: (coordinator: {
failImport: (id: number, errorMessage?: string) => unknown;
}) => coordinator.failImport(7, 'import failed'),
expectedStatus: 'import_failed',
},
])(
'projects $name without leaking completed queue items into the import queue',
async ({ settleImport, expectedStatus }) => {
storageState.set('recordingJobs:v2', [
{
id: 7,
phase: 'recorded',
appName: 'Zoom',
startTime: 1000,
createdAt: 1,
updatedAt: 1,
artifact: {
filepath: '/tmp/meeting.opus',
sampleRate: 48_000,
numberOfChannels: 2,
},
},
]);
const { coordinator } = await createCoordinator();
settleImport(coordinator as never);
expect(coordinator.currentStatus()).toMatchObject({
id: 7,
status: expectedStatus,
});
expect(coordinator.importQueue()).toEqual([]);
}
);
test.each([
{
name: 'successful imports',
seed: {
id: 11,
phase: 'imported' as const,
appName: 'Zoom',
startTime: 1000,
createdAt: 1,
updatedAt: 1,
artifact: {
filepath: '/tmp/imported.opus',
},
},
},
{
name: 'failed imports',
seed: {
id: 12,
phase: 'failed' as const,
appName: 'Meet',
startTime: 1000,
createdAt: 1,
updatedAt: 1,
artifact: {
filepath: '/tmp/failed.opus',
},
error: {
stage: 'import' as const,
message: 'import failed',
},
},
},
])(
'suppresses persisted terminal $name from the current status',
async ({ seed }) => {
storageState.set('recordingJobs:v2', [seed]);
const { coordinator } = await createCoordinator();
expect(coordinator.currentStatus()).toBeNull();
expect(coordinator.importQueue()).toEqual([]);
}
);
test('dismissing an import failure clears the popup projection without dropping the saved artifact', async () => {
storageState.set('recordingJobs:v2', [
{
id: 7,
phase: 'failed',
appName: 'Zoom',
startTime: 1000,
createdAt: 1,
updatedAt: 1,
artifact: {
filepath: '/tmp/meeting.opus',
},
error: {
stage: 'import',
message: 'import failed',
},
},
]);
const { coordinator } = await createCoordinator();
coordinator.dismiss(7);
expect(coordinator.currentStatus()).toBeNull();
expect(coordinator.importQueue()).toEqual([]);
expect(storageState.get('recordingJobs:v2')).toEqual([
expect.objectContaining({
id: 7,
phase: 'failed',
dismissedAt: expect.any(Number),
artifact: expect.objectContaining({
filepath: '/tmp/meeting.opus',
}),
error: {
stage: 'import',
message: 'import failed',
},
}),
]);
});
});
packages/frontend/apps/electron/test/main/recording-effect.spec.ts
+372 -99
View File
@@ -2,22 +2,45 @@ import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
const isActiveTab = vi.fn();
const readRecordingFile = vi.fn();
const setRecordingBlockCreationStatus = vi.fn();
const claimRecordingImport = vi.fn();
const completeRecordingImport = vi.fn();
const failRecordingImport = vi.fn();
const getRecordingImportQueue = vi.fn();
const getCurrentWorkspace = vi.fn();
const isAiEnabled = vi.fn();
const transcribeRecording = vi.fn();
let onRecordingStatusChanged:
| ((
status: {
id: number;
status: 'processing';
appName?: string;
filepath?: string;
startTime: number;
blockCreationStatus?: 'success' | 'failed';
} | null
) => void)
type RecordingImportStatus = {
id: number;
appName?: string;
workspaceId?: string;
docId?: string;
filepath: string;
startTime: number;
sampleRate?: number;
numberOfChannels?: number;
durationMs?: number;
size?: number;
degraded?: boolean;
overflowCount?: number;
errorMessage?: string;
importStatus: 'pending_import' | 'importing' | 'imported' | 'import_failed';
createdAt: number;
updatedAt: number;
};
function withQueueMeta(
status: Omit<RecordingImportStatus, 'createdAt' | 'updatedAt'>
): RecordingImportStatus {
return {
createdAt: 1,
updatedAt: 1,
...status,
};
}
let onRecordingImportQueueChanged:
| ((queue: RecordingImportStatus[]) => void)
| undefined;
vi.mock('@affine/core/modules/doc', () => ({
@@ -46,16 +69,19 @@ vi.mock('@affine/electron-api', () => ({
},
recording: {
readRecordingFile,
setRecordingBlockCreationStatus,
claimRecordingImport,
completeRecordingImport,
failRecordingImport,
getRecordingImportQueue,
},
},
events: {
recording: {
onRecordingStatusChanged: vi.fn(
(handler: typeof onRecordingStatusChanged) => {
onRecordingStatusChanged = handler;
onRecordingImportQueueChanged: vi.fn(
(handler: typeof onRecordingImportQueueChanged) => {
onRecordingImportQueueChanged = handler;
return () => {
onRecordingStatusChanged = undefined;
onRecordingImportQueueChanged = undefined;
};
}
),
@@ -84,46 +110,97 @@ vi.mock('../../../electron-renderer/src/app/effects/utils', () => ({
isAiEnabled,
}));
function createDeferred<T>() {
let resolve!: (value: T) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((res, rej) => {
resolve = res;
reject = rej;
});
return { promise, resolve, reject };
}
function createWorkspaceRef() {
const blobSet = vi.fn(async () => 'blob-1');
const addBlock = vi.fn(() => 'attachment-1');
const getBlock = vi.fn(() => ({ model: { id: 'attachment-1' } }));
const openDoc = vi.fn();
const createdDocs = new Set<string>();
const attachments: Array<{
id: string;
props: { name: string; type: string };
}> = [];
type MockDoc = {
workspace: {
blobSync: {
set: typeof blobSet;
};
const blockSuiteDoc = {
getModelsByFlavour: vi.fn((flavour: string) => {
if (flavour === 'affine:page') {
return [{ id: 'page-1' }];
}
if (flavour === 'affine:note') {
return [{ id: 'note-1' }];
}
if (flavour === 'affine:attachment') {
return attachments;
}
return [];
}),
addBlock: vi.fn(
(
flavour: string,
props: { name?: string; type?: string },
_parentId?: string
) => {
if (flavour === 'affine:attachment') {
const id = `attachment-${attachments.length + 1}`;
attachments.push({
id,
props: {
name: props.name ?? '',
type: props.type ?? '',
},
});
return id;
}
return `${flavour}-1`;
}
),
getBlock: vi.fn((id: string) => {
const attachment = attachments.find(entry => entry.id === id);
return attachment ? { model: attachment } : null;
}),
};
const createDoc = vi.fn(({ id }: { id: string }) => {
createdDocs.add(id);
return { id };
});
const open = vi.fn((docId: string) => {
if (!createdDocs.has(docId)) {
throw new Error(`Doc ${docId} not found`);
}
return {
doc: {
workspace: { id: 'workspace-1' },
blockSuiteDoc,
addPriorityLoad: vi.fn(() => vi.fn()),
waitForSyncReady: vi.fn(async () => undefined),
},
release: vi.fn(),
};
addBlock: typeof addBlock;
getBlock: typeof getBlock;
};
type MockDocProps = {
onStoreLoad: (doc: MockDoc, meta: { noteId: string }) => void;
};
const createDoc = vi.fn(({ docProps }: { docProps: MockDocProps }) => {
queueMicrotask(() => {
docProps.onStoreLoad(
{
workspace: { blobSync: { set: blobSet } },
addBlock,
getBlock,
},
{ noteId: 'note-1' }
);
});
return { id: 'doc-1' };
});
const scope = {
get(token: { name?: string }) {
switch (token.name) {
case 'DocsService':
return { createDoc };
return {
createDoc,
open,
list: {
['doc$']: (docId: string) => ({
value: createdDocs.has(docId) ? { id: docId } : null,
}),
},
};
case 'WorkbenchService':
return { workbench: { openDoc } };
case 'AudioAttachmentService':
@@ -145,15 +222,20 @@ function createWorkspaceRef() {
return {
ref: {
workspace: { scope },
workspace: {
id: 'workspace-1',
scope,
docCollection: {
blobSync: { set: blobSet },
},
},
dispose,
[Symbol.dispose]: dispose,
},
createDoc,
openDoc,
blobSet,
addBlock,
getBlock,
attachments,
};
}
@@ -162,10 +244,12 @@ describe('recording effect', () => {
vi.useFakeTimers();
vi.clearAllMocks();
vi.resetModules();
onRecordingStatusChanged = undefined;
onRecordingImportQueueChanged = undefined;
readRecordingFile.mockResolvedValue(new Uint8Array([1, 2, 3]).buffer);
setRecordingBlockCreationStatus.mockResolvedValue(undefined);
completeRecordingImport.mockResolvedValue(undefined);
failRecordingImport.mockResolvedValue(undefined);
isAiEnabled.mockReturnValue(false);
getRecordingImportQueue.mockResolvedValue([]);
});
afterEach(() => {
@@ -173,84 +257,273 @@ describe('recording effect', () => {
vi.useRealTimers();
});
test('retries processing until the active tab has a workspace', async () => {
test('retries pending imports until the active tab has a workspace and claims with workspace binding', async () => {
const workspace = createWorkspaceRef();
const pendingImport = {
id: 7,
importStatus: 'pending_import' as const,
appName: 'Zoom',
filepath: '/tmp/meeting.opus',
startTime: 1000,
};
isActiveTab.mockResolvedValueOnce(false).mockResolvedValue(true);
getCurrentWorkspace
.mockReturnValueOnce(undefined)
.mockReturnValue(workspace.ref);
claimRecordingImport.mockResolvedValue({
...withQueueMeta(pendingImport),
workspaceId: 'workspace-1',
docId: 'recording-7',
importStatus: 'importing',
});
getRecordingImportQueue.mockResolvedValue([withQueueMeta(pendingImport)]);
const { setupRecordingEvents } =
await import('../../../electron-renderer/src/app/effects/recording');
setupRecordingEvents({} as never);
onRecordingStatusChanged?.({
id: 7,
status: 'processing',
appName: 'Zoom',
filepath: '/tmp/meeting.opus',
startTime: 1000,
});
await Promise.resolve();
expect(workspace.createDoc).not.toHaveBeenCalled();
expect(setRecordingBlockCreationStatus).not.toHaveBeenCalled();
expect(claimRecordingImport).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1000);
expect(workspace.createDoc).not.toHaveBeenCalled();
expect(setRecordingBlockCreationStatus).not.toHaveBeenCalled();
expect(claimRecordingImport).not.toHaveBeenCalled();
onRecordingImportQueueChanged?.([withQueueMeta(pendingImport)]);
await vi.advanceTimersByTimeAsync(1000);
expect(workspace.createDoc).toHaveBeenCalledTimes(1);
expect(workspace.openDoc).toHaveBeenCalledWith('doc-1');
expect(claimRecordingImport).toHaveBeenCalledWith(7, 'workspace-1');
expect(workspace.createDoc).toHaveBeenCalledWith(
expect.objectContaining({ id: 'recording-7' })
);
expect(workspace.openDoc).toHaveBeenCalledWith('recording-7');
expect(workspace.blobSet).toHaveBeenCalledTimes(1);
const [savedBlob] = workspace.blobSet.mock.calls[0] ?? [];
expect(savedBlob).toBeInstanceOf(Blob);
expect((savedBlob as Blob).type).toBe('audio/ogg');
expect(workspace.addBlock).toHaveBeenCalledWith(
'affine:attachment',
expect.objectContaining({ type: 'audio/ogg' }),
'note-1'
);
expect(setRecordingBlockCreationStatus).toHaveBeenCalledWith(7, 'success');
expect(setRecordingBlockCreationStatus).not.toHaveBeenCalledWith(
7,
'failed',
expect.anything()
);
expect(completeRecordingImport).toHaveBeenCalledWith(7);
expect(failRecordingImport).not.toHaveBeenCalled();
});
test('retries when the active-tab probe rejects', async () => {
test('reuses the same doc when a claimed import already carries docId', async () => {
const workspace = createWorkspaceRef();
isActiveTab
.mockRejectedValueOnce(new Error('probe failed'))
.mockResolvedValue(true);
getCurrentWorkspace.mockReturnValue(workspace.ref);
const { setupRecordingEvents } =
await import('../../../electron-renderer/src/app/effects/recording');
setupRecordingEvents({} as never);
onRecordingStatusChanged?.({
id: 9,
status: 'processing',
const pendingImport = {
id: 8,
importStatus: 'pending_import' as const,
appName: 'Meet',
filepath: '/tmp/meeting.opus',
startTime: 1000,
workspaceId: 'workspace-1',
docId: 'recording-8',
};
workspace.createDoc({ id: 'recording-8' });
isActiveTab.mockResolvedValue(true);
getCurrentWorkspace.mockReturnValue(workspace.ref);
claimRecordingImport.mockResolvedValue({
...withQueueMeta(pendingImport),
importStatus: 'importing',
});
getRecordingImportQueue.mockResolvedValue([withQueueMeta(pendingImport)]);
const { setupRecordingEvents } =
await import('../../../electron-renderer/src/app/effects/recording');
setupRecordingEvents({} as never);
await Promise.resolve();
expect(workspace.createDoc).not.toHaveBeenCalled();
expect(setRecordingBlockCreationStatus).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1000);
await Promise.resolve();
await vi.advanceTimersByTimeAsync(0);
expect(workspace.createDoc).toHaveBeenCalledTimes(1);
expect(setRecordingBlockCreationStatus).toHaveBeenCalledWith(9, 'success');
expect(workspace.blobSet).toHaveBeenCalledTimes(1);
expect(completeRecordingImport).toHaveBeenCalledWith(8);
});
test('marks imports as failed when blob import fails after claim', async () => {
const pendingImport = {
id: 9,
importStatus: 'pending_import' as const,
appName: 'Meet',
filepath: '/tmp/meeting.opus',
startTime: 1000,
};
const workspace = createWorkspaceRef();
workspace.blobSet.mockRejectedValueOnce(new Error('blob import failed'));
isActiveTab.mockResolvedValue(true);
getCurrentWorkspace.mockReturnValue(workspace.ref);
claimRecordingImport.mockResolvedValue({
...withQueueMeta(pendingImport),
workspaceId: 'workspace-1',
docId: 'recording-9',
importStatus: 'importing',
});
getRecordingImportQueue.mockResolvedValue([withQueueMeta(pendingImport)]);
const { setupRecordingEvents } =
await import('../../../electron-renderer/src/app/effects/recording');
setupRecordingEvents({} as never);
await Promise.resolve();
await Promise.resolve();
await vi.advanceTimersByTimeAsync(0);
expect(failRecordingImport).toHaveBeenCalledWith(9, 'blob import failed');
expect(completeRecordingImport).not.toHaveBeenCalled();
});
test('releases priority load when waiting for sync readiness fails', async () => {
const pendingImport = {
id: 10,
importStatus: 'pending_import' as const,
appName: 'Meet',
filepath: '/tmp/meeting.opus',
startTime: 1000,
};
const disposePriorityLoad = vi.fn();
const release = vi.fn();
const workspace = {
ref: {
workspace: {
id: 'workspace-1',
scope: {
get(token: { name?: string }) {
switch (token.name) {
case 'DocsService':
return {
createDoc: vi.fn(),
open: vi.fn(() => ({
doc: {
blockSuiteDoc: {},
addPriorityLoad: vi.fn(() => disposePriorityLoad),
waitForSyncReady: vi
.fn()
.mockRejectedValue(new Error('sync failed')),
},
release,
})),
list: {
['doc$']: vi.fn(() => ({
value: { id: 'recording-10' },
})),
},
};
case 'WorkbenchService':
case 'AudioAttachmentService':
return {};
default:
throw new Error(`Unexpected token: ${token.name}`);
}
},
},
docCollection: {
blobSync: { set: vi.fn() },
},
},
dispose: vi.fn(),
[Symbol.dispose]: vi.fn(),
},
};
isActiveTab.mockResolvedValue(true);
getCurrentWorkspace.mockReturnValue(workspace.ref);
claimRecordingImport.mockResolvedValue({
...withQueueMeta(pendingImport),
workspaceId: 'workspace-1',
docId: 'recording-10',
importStatus: 'importing',
});
getRecordingImportQueue.mockResolvedValue([withQueueMeta(pendingImport)]);
const { setupRecordingEvents } =
await import('../../../electron-renderer/src/app/effects/recording');
setupRecordingEvents({} as never);
await Promise.resolve();
await Promise.resolve();
await vi.advanceTimersByTimeAsync(0);
expect(disposePriorityLoad).toHaveBeenCalledTimes(1);
expect(release).toHaveBeenCalledTimes(1);
expect(failRecordingImport).toHaveBeenCalledWith(10, 'sync failed');
expect(completeRecordingImport).not.toHaveBeenCalled();
});
test('processes recording imports one at a time even when the queue changes mid-import', async () => {
const firstImport = {
id: 7,
importStatus: 'pending_import' as const,
appName: 'Zoom',
filepath: '/tmp/meeting-1.opus',
startTime: 1000,
};
const secondImport = {
id: 8,
importStatus: 'pending_import' as const,
appName: 'Meet',
filepath: '/tmp/meeting-2.opus',
startTime: 2000,
};
const firstRead = createDeferred<ArrayBuffer>();
const workspace = createWorkspaceRef();
isActiveTab.mockResolvedValue(true);
getCurrentWorkspace.mockReturnValue(workspace.ref);
readRecordingFile
.mockImplementationOnce(() => firstRead.promise)
.mockResolvedValueOnce(new Uint8Array([4, 5, 6]).buffer);
claimRecordingImport.mockImplementation(async (id: number) => ({
...(id === firstImport.id ? firstImport : secondImport),
workspaceId: 'workspace-1',
docId: `recording-${id}`,
importStatus: 'importing' as const,
createdAt: 1,
updatedAt: 1,
}));
getRecordingImportQueue.mockResolvedValue([
withQueueMeta(firstImport),
withQueueMeta(secondImport),
]);
const { setupRecordingEvents } =
await import('../../../electron-renderer/src/app/effects/recording');
setupRecordingEvents({} as never);
await Promise.resolve();
await Promise.resolve();
await vi.advanceTimersByTimeAsync(0);
expect(claimRecordingImport).toHaveBeenCalledTimes(1);
expect(claimRecordingImport).toHaveBeenCalledWith(
firstImport.id,
'workspace-1'
);
onRecordingImportQueueChanged?.([
withQueueMeta({
...firstImport,
workspaceId: 'workspace-1',
docId: 'recording-7',
importStatus: 'importing',
}),
withQueueMeta(secondImport),
]);
await Promise.resolve();
expect(claimRecordingImport).toHaveBeenCalledTimes(1);
firstRead.resolve(new Uint8Array([1, 2, 3]).buffer);
await Promise.resolve();
await Promise.resolve();
await vi.advanceTimersByTimeAsync(1000);
expect(claimRecordingImport).toHaveBeenCalledTimes(2);
expect(claimRecordingImport).toHaveBeenNthCalledWith(
2,
secondImport.id,
'workspace-1'
);
expect(completeRecordingImport).toHaveBeenNthCalledWith(1, firstImport.id);
expect(completeRecordingImport).toHaveBeenNthCalledWith(2, secondImport.id);
});
});
packages/frontend/apps/electron/test/main/recording-feature.spec.ts
+341
View File
@@ -0,0 +1,341 @@
import { BehaviorSubject } from 'rxjs';
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
const nativeStartRecording = vi.fn();
const nativeStopRecording = vi.fn();
const nativeAbortRecording = vi.fn();
const ensureDirSync = vi.fn();
const resolveExistingPathInBase = vi.fn(
async (_base: string, filepath: string) => filepath
);
const getMainWindow = vi.fn(async () => ({
show: vi.fn(),
}));
const storageState = new Map<string, unknown>();
const watchSubjects = new Map<string, BehaviorSubject<unknown>>();
function createDeferred<T>() {
let resolve!: (value: T) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((res, rej) => {
resolve = res;
reject = rej;
});
return { promise, resolve, reject };
}
beforeEach(() => {
vi.resetModules();
vi.clearAllMocks();
storageState.clear();
watchSubjects.clear();
vi.doMock('@affine/native', () => ({
ShareableContent: class ShareableContent {
static applications() {
return [];
}
static applicationWithProcessId() {
return null;
}
static isUsingMicrophone() {
return false;
}
static onApplicationListChanged() {
return { unsubscribe: vi.fn() };
}
static onAppStateChanged() {
return { unsubscribe: vi.fn() };
}
},
startRecording: nativeStartRecording,
stopRecording: nativeStopRecording,
abortRecording: nativeAbortRecording,
}));
vi.doMock('electron', () => ({
app: {
getPath: vi.fn(() => '/tmp'),
on: vi.fn(),
},
systemPreferences: {
getMediaAccessStatus: vi.fn(() => 'granted'),
askForMediaAccess: vi.fn(async () => true),
},
}));
vi.doMock('fs-extra', () => ({
default: {
ensureDirSync,
removeSync: vi.fn(),
},
}));
vi.doMock('../../src/shared/utils', async () => {
const actual = await vi.importActual('../../src/shared/utils');
return {
...actual,
isMacOS: () => false,
isWindows: () => false,
resolveExistingPathInBase,
};
});
vi.doMock('../../src/main/shared-storage/storage', () => ({
globalStateStorage: {
get: (key: string) => storageState.get(key),
set: (key: string, value: unknown) => {
storageState.set(key, value);
watchSubjects.get(key)?.next(value);
},
watch: (key: string) => {
const watchSubject$ =
watchSubjects.get(key) ?? new BehaviorSubject(storageState.get(key));
watchSubjects.set(key, watchSubject$);
return watchSubject$.asObservable();
},
},
}));
vi.doMock('../../src/main/windows-manager', () => ({
getMainWindow,
}));
vi.doMock('../../src/main/windows-manager/popup', () => ({
popupManager: {
get: () => ({
showing: false,
show: vi.fn(async () => undefined),
hide: vi.fn(async () => undefined),
}),
},
}));
vi.doMock('lodash-es', () => ({
debounce: (fn: (...args: unknown[]) => void) => fn,
}));
});
async function loadRecordingFeature() {
const feature = await import('../../src/main/recording/feature');
feature.setRecordingNativeModuleForTesting({
ShareableContent: class ShareableContent {
static applications() {
return [];
}
static applicationWithProcessId() {
return null;
}
static isUsingMicrophone() {
return false;
}
static onApplicationListChanged() {
return { unsubscribe: vi.fn() };
}
static onAppStateChanged() {
return { unsubscribe: vi.fn() };
}
},
startRecording: nativeStartRecording,
stopRecording: nativeStopRecording,
abortRecording: nativeAbortRecording,
} as never);
return feature;
}
afterEach(() => {
vi.clearAllTimers();
});
describe('recording feature', () => {
test('slow start exposes starting state before native setup resolves', async () => {
const startDeferred = createDeferred<{
id: string;
filepath: string;
sampleRate: number;
channels: number;
startedAt: number;
}>();
nativeStartRecording.mockReturnValue(startDeferred.promise);
const { getCurrentRecordingStatus, startRecording } =
await loadRecordingFeature();
const startPromise = startRecording();
expect(getCurrentRecordingStatus()).toMatchObject({
status: 'starting',
});
startDeferred.resolve({
id: 'native-1',
filepath: '/tmp/0.opus',
sampleRate: 48_000,
channels: 2,
startedAt: 123,
});
await startPromise;
expect(getCurrentRecordingStatus()).toMatchObject({
status: 'recording',
});
expect(getCurrentRecordingStatus()?.filepath).toContain('0.opus');
});
test('stop handoff moves from finalizing to pending_import without clearing the popup state', async () => {
nativeStartRecording.mockResolvedValue({
id: 'native-1',
filepath: '/tmp/0.opus',
sampleRate: 48_000,
channels: 2,
startedAt: 123,
});
const stopDeferred = createDeferred<{
id: string;
filepath: string;
sampleRate: number;
channels: number;
durationMs: number;
size: number;
degraded: boolean;
overflowCount: number;
}>();
nativeStopRecording.mockReturnValue(stopDeferred.promise);
const {
getCurrentRecordingStatus,
getRecordingImportQueue,
recordingStatus$,
startRecording,
stopRecording,
} = await loadRecordingFeature();
const started = await startRecording();
const seenStatuses: Array<string | null> = [];
const subscription = recordingStatus$.subscribe(status => {
seenStatuses.push(status?.status ?? null);
});
const stopPromise = stopRecording(started!.id);
expect(getCurrentRecordingStatus()).toMatchObject({
id: started!.id,
status: 'finalizing',
});
stopDeferred.resolve({
id: 'native-1',
filepath: '/tmp/0.opus',
sampleRate: 48_000,
channels: 2,
durationMs: 2_000,
size: 256,
degraded: true,
overflowCount: 4,
});
await stopPromise;
subscription.unsubscribe();
expect(getCurrentRecordingStatus()).toMatchObject({
id: started!.id,
status: 'pending_import',
degraded: true,
overflowCount: 4,
});
expect(getRecordingImportQueue()).toEqual([
expect.objectContaining({
id: started!.id,
importStatus: 'pending_import',
filepath: '/tmp/0.opus',
degraded: true,
overflowCount: 4,
}),
]);
expect(seenStatuses).toContain('finalizing');
expect(seenStatuses).toContain('pending_import');
expect(seenStatuses).not.toContain(null);
});
test('system-audio start does not reuse a pending app-scoped prompt', async () => {
nativeStartRecording.mockResolvedValue({
id: 'native-1',
filepath: '/tmp/system.opus',
sampleRate: 48_000,
channels: 2,
startedAt: 123,
});
const { newRecording, startRecording } = await loadRecordingFeature();
newRecording({
processGroupId: 100,
name: 'Zoom',
bundleIdentifier: 'us.zoom.xos',
icon: undefined,
isRunning: true,
apps: [
{
info: {} as never,
isRunning: true,
processId: 42,
processGroupId: 100,
bundleIdentifier: 'us.zoom.xos',
name: 'Zoom',
},
],
});
await startRecording();
expect(nativeStartRecording).toHaveBeenCalledWith(
expect.objectContaining({
appProcessId: undefined,
})
);
});
test('disableRecordingFeature clears the active session so a new recording can start', async () => {
nativeStartRecording
.mockResolvedValueOnce({
id: 'native-1',
filepath: '/tmp/0.opus',
sampleRate: 48_000,
channels: 2,
startedAt: 123,
})
.mockResolvedValueOnce({
id: 'native-2',
filepath: '/tmp/1.opus',
sampleRate: 48_000,
channels: 2,
startedAt: 456,
});
const feature = await loadRecordingFeature();
const first = await feature.startRecording();
expect(feature.getCurrentRecordingStatus()).toMatchObject({
id: first!.id,
status: 'recording',
});
await feature.disableRecordingFeature();
expect(feature.getCurrentRecordingStatus()).toBeNull();
expect(nativeAbortRecording).toHaveBeenCalledWith('native-1');
const second = await feature.startRecording();
expect(second).toMatchObject({
id: expect.any(Number),
status: 'recording',
});
expect(second!.id).toBeGreaterThan(first!.id);
});
});
packages/frontend/apps/electron/test/main/recording-index.spec.ts
+75
View File
@@ -0,0 +1,75 @@
import { BehaviorSubject } from 'rxjs';
import { describe, expect, test, vi } from 'vitest';
const recordingStatus$ = new BehaviorSubject(null);
const recordingImportQueue$ = new BehaviorSubject([
{
id: 7,
appName: 'Zoom',
workspaceId: 'workspace-1',
docId: 'recording-7',
startTime: 1000,
filepath: '/tmp/meeting.opus',
importStatus: 'importing' as const,
createdAt: 1,
updatedAt: 1,
},
]);
vi.mock('electron', () => ({
shell: {
showItemInFolder: vi.fn(),
},
}));
vi.mock('../../src/shared/utils', () => ({
isMacOS: () => false,
resolvePathInBase: vi.fn((_base: string, subpath: string) => subpath),
}));
vi.mock('../../src/main/security/open-external', () => ({
openExternalSafely: vi.fn(),
}));
vi.mock('../../src/main/recording/feature', () => ({
askForMeetingPermission: vi.fn(),
checkMeetingPermissions: vi.fn(),
checkRecordingAvailable: vi.fn(),
claimRecordingImport: vi.fn(),
completeRecordingImport: vi.fn(),
disableRecordingFeature: vi.fn(),
dismissRecordingStatus: vi.fn(),
failRecordingImport: vi.fn(),
getCurrentRecordingStatus: vi.fn(),
getRecording: vi.fn(),
getRecordingImportQueue: vi.fn(),
readRecordingFile: vi.fn(),
recordingImportQueue$,
recordingStatus$,
removeRecording: vi.fn(),
SAVED_RECORDINGS_DIR: '/tmp',
serializeRecordingStatus: vi.fn(),
setupRecordingFeature: vi.fn(),
startRecording: vi.fn(),
stopRecording: vi.fn(),
}));
describe('recording index events', () => {
test('onRecordingImportQueueChanged preserves workspace binding metadata', async () => {
const { recordingEvents } = await import('../../src/main/recording');
const handler = vi.fn();
const unsubscribe = recordingEvents.onRecordingImportQueueChanged(handler);
expect(handler).toHaveBeenCalledWith([
expect.objectContaining({
id: 7,
workspaceId: 'workspace-1',
docId: 'recording-7',
importStatus: 'importing',
}),
]);
unsubscribe();
});
});
packages/frontend/apps/electron/test/main/recording-state.spec.ts
-116
View File
@@ -1,116 +0,0 @@
import { describe, expect, test, vi } from 'vitest';
vi.mock('../../src/main/logger', () => ({
logger: {
error: vi.fn(),
info: vi.fn(),
},
}));
import { RecordingStateMachine } from '../../src/main/recording/state-machine';
function createAttachedRecording(stateMachine: RecordingStateMachine) {
const pending = stateMachine.dispatch({
type: 'START_RECORDING',
});
stateMachine.dispatch({
type: 'ATTACH_NATIVE_RECORDING',
id: pending!.id,
nativeId: 'native-1',
startTime: 100,
filepath: '/tmp/recording.opus',
sampleRate: 48000,
numberOfChannels: 2,
});
return pending!;
}
describe('RecordingStateMachine', () => {
test('transitions from recording to ready after artifact import and block creation', () => {
const stateMachine = new RecordingStateMachine();
const pending = createAttachedRecording(stateMachine);
expect(pending?.status).toBe('recording');
const processing = stateMachine.dispatch({
type: 'STOP_RECORDING',
id: pending.id,
});
expect(processing?.status).toBe('processing');
const artifactAttached = stateMachine.dispatch({
type: 'ATTACH_RECORDING_ARTIFACT',
id: pending.id,
filepath: '/tmp/recording.opus',
sampleRate: 48000,
numberOfChannels: 2,
});
expect(artifactAttached).toMatchObject({
status: 'processing',
filepath: '/tmp/recording.opus',
});
const ready = stateMachine.dispatch({
type: 'SET_BLOCK_CREATION_STATUS',
id: pending.id,
status: 'success',
});
expect(ready).toMatchObject({
status: 'ready',
blockCreationStatus: 'success',
});
});
test('keeps native audio metadata when stop artifact omits it', () => {
const stateMachine = new RecordingStateMachine();
const pending = createAttachedRecording(stateMachine);
stateMachine.dispatch({ type: 'STOP_RECORDING', id: pending.id });
const artifactAttached = stateMachine.dispatch({
type: 'ATTACH_RECORDING_ARTIFACT',
id: pending.id,
filepath: '/tmp/recording.opus',
});
expect(artifactAttached).toMatchObject({
sampleRate: 48000,
numberOfChannels: 2,
});
});
test.each([
{ status: 'success' as const, errorMessage: undefined },
{ status: 'failed' as const, errorMessage: 'native start failed' },
])(
'settles recordings into ready state with blockCreationStatus=$status',
({ status, errorMessage }) => {
const stateMachine = new RecordingStateMachine();
const pending = stateMachine.dispatch({
type: 'START_RECORDING',
});
expect(pending?.status).toBe('recording');
const settled = stateMachine.dispatch({
type: 'SET_BLOCK_CREATION_STATUS',
id: pending!.id,
status,
errorMessage,
});
expect(settled).toMatchObject({
status: 'ready',
blockCreationStatus: status,
});
const next = stateMachine.dispatch({
type: 'START_RECORDING',
});
expect(next?.id).toBeGreaterThan(pending!.id);
expect(next?.status).toBe('recording');
expect(next?.blockCreationStatus).toBeUndefined();
}
);
});
packages/frontend/apps/electron/test/workspace/handlers.spec.ts
+37 -42
View File
@@ -132,51 +132,46 @@ describe('workspace db management', () => {
).toBe(false);
});
test('rejects unsafe ids when deleting a workspace', async () => {
const { deleteWorkspace } =
await import('@affine/electron/helper/workspace/handlers');
const outsideDir = path.join(tmpDir, 'outside-delete-target');
test.each([
{
name: 'deleting a workspace',
outsideDirName: 'outside-delete-target',
call: async () => {
const { deleteWorkspace } =
await import('@affine/electron/helper/workspace/handlers');
return deleteWorkspace(
universalId({
peer: 'local',
type: 'workspace',
id: '../../outside-delete-target',
})
);
},
},
{
name: 'deleting backup workspaces',
outsideDirName: 'outside-backup-target',
call: async () => {
const { deleteBackupWorkspace } =
await import('@affine/electron/helper/workspace/handlers');
return deleteBackupWorkspace('../../outside-backup-target');
},
},
{
name: 'recovering backup workspaces',
outsideDirName: 'outside-recover-target',
call: async () => {
const { recoverBackupWorkspace } =
await import('@affine/electron/helper/workspace/handlers');
return recoverBackupWorkspace('../../outside-recover-target');
},
},
])('rejects unsafe ids when $name', async ({ outsideDirName, call }) => {
const outsideDir = path.join(tmpDir, outsideDirName);
await fs.ensureDir(outsideDir);
await expect(
deleteWorkspace(
universalId({
peer: 'local',
type: 'workspace',
id: '../../outside-delete-target',
})
)
).rejects.toThrow('Invalid workspace id');
expect(await fs.pathExists(outsideDir)).toBe(true);
});
test('rejects unsafe ids when deleting backup workspaces', async () => {
const { deleteBackupWorkspace } =
await import('@affine/electron/helper/workspace/handlers');
const outsideDir = path.join(tmpDir, 'outside-backup-target');
await fs.ensureDir(outsideDir);
await expect(
deleteBackupWorkspace('../../outside-backup-target')
).rejects.toThrow('Invalid workspace id');
expect(await fs.pathExists(outsideDir)).toBe(true);
});
test('rejects unsafe ids when recovering backup workspaces', async () => {
const { recoverBackupWorkspace } =
await import('@affine/electron/helper/workspace/handlers');
const outsideDir = path.join(tmpDir, 'outside-recover-target');
await fs.ensureDir(outsideDir);
await expect(
recoverBackupWorkspace('../../outside-recover-target')
).rejects.toThrow('Invalid workspace id');
await expect(call()).rejects.toThrow('Invalid workspace id');
expect(await fs.pathExists(outsideDir)).toBe(true);
});
});
packages/frontend/core/src/modules/media/services/meeting-settings.ts
+3 -2
View File
@@ -83,8 +83,9 @@ export class MeetingSettingsService extends Service {
await this.desktopApiService?.handler.recording.getCurrentRecording();
if (
ongoingRecording &&
ongoingRecording.status !== 'new' &&
ongoingRecording.status !== 'ready'
['starting', 'recording', 'finalizing'].includes(
ongoingRecording.status
)
) {
throw new Error('There is an ongoing recording, please stop it first');
}
packages/frontend/i18n/src/i18n.gen.ts
+4
View File
@@ -8525,6 +8525,10 @@ export function useAFFiNEI18N(): {
* `Audio activity`
*/
["com.affine.recording.new"](): string;
/**
* `Importing...`
*/
["com.affine.recording.importing.prompt"](): string;
/**
* `Finished`
*/
packages/frontend/i18n/src/resources/en.json
+1
View File
@@ -2137,6 +2137,7 @@
"com.affine.audio.transcribe.non-owner.confirm.title": "Unable to retrieve AI results for others",
"com.affine.audio.transcribe.non-owner.confirm.message": "Please contact <1>{{user}}</1> to upgrade AI rights or resend the attachment.",
"com.affine.recording.new": "Audio activity",
"com.affine.recording.importing.prompt": "Importing...",
"com.affine.recording.success.prompt": "Finished",
"com.affine.recording.success.button": "Open app",
"com.affine.recording.failed.prompt": "Failed to save",
packages/frontend/i18n/src/resources/zh-Hans.json
+1
View File
@@ -2081,6 +2081,7 @@
"com.affine.audio.transcribe.non-owner.confirm.title": "无法检索他人的 AI 结果",
"com.affine.audio.transcribe.non-owner.confirm.message": "请联系 <1>{{user}}</1> 提升 AI 权限或重新发送附件。",
"com.affine.recording.new": "音频活动中",
"com.affine.recording.importing.prompt": "正在导入...",
"com.affine.recording.success.prompt": "已完成",
"com.affine.recording.success.button": "打开应用",
"com.affine.recording.failed.prompt": "保存失败",
packages/frontend/native/index.d.ts
Vendored
+6 -2
View File
@@ -36,6 +36,8 @@ export declare class ShareableContent {
static tapGlobalAudio(excludedProcesses: Array<ApplicationInfo> | undefined | null, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession
}
export declare function abortRecording(id: string): Promise<void>
export declare function decodeAudio(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null, signal?: AbortSignal | undefined | null): Promise<Float32Array>
/** Decode audio file into a Float32Array */
@@ -48,6 +50,8 @@ export interface RecordingArtifact {
channels: number
durationMs: number
size: number
degraded: boolean
overflowCount: number
}
export interface RecordingSessionMeta {
@@ -68,9 +72,9 @@ export interface RecordingStartOptions {
id?: string
}
export declare function startRecording(opts: RecordingStartOptions): RecordingSessionMeta
export declare function startRecording(opts: RecordingStartOptions): Promise<RecordingSessionMeta>
export declare function stopRecording(id: string): RecordingArtifact
export declare function stopRecording(id: string): Promise<RecordingArtifact>
export interface MermaidRenderOptions {
theme?: string
fontFamily?: string
packages/frontend/native/index.js
+1
View File
@@ -577,6 +577,7 @@ module.exports.ApplicationListChangedSubscriber = nativeBinding.ApplicationListC
module.exports.ApplicationStateChangedSubscriber = nativeBinding.ApplicationStateChangedSubscriber
module.exports.AudioCaptureSession = nativeBinding.AudioCaptureSession
module.exports.ShareableContent = nativeBinding.ShareableContent
module.exports.abortRecording = nativeBinding.abortRecording
module.exports.decodeAudio = nativeBinding.decodeAudio
module.exports.decodeAudioSync = nativeBinding.decodeAudioSync
module.exports.startRecording = nativeBinding.startRecording
packages/frontend/native/media_capture/Cargo.toml
+1
View File
@@ -21,6 +21,7 @@ rand = { workspace = true }
rubato = { workspace = true }
symphonia = { workspace = true, features = ["all", "opt-simd"] }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["rt", "sync"] }
[target.'cfg(target_os = "macos")'.dependencies]
block2 = { workspace = true }
packages/frontend/native/media_capture/src/audio_callback.rs
+41 -7
View File
@@ -1,6 +1,9 @@
use std::sync::Arc;
use std::sync::{
Arc,
atomic::{AtomicU64, Ordering},
};
use crossbeam_channel::Sender;
use crossbeam_channel::{Sender, TrySendError};
use napi::{
bindgen_prelude::Float32Array,
threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
@@ -11,7 +14,10 @@ use napi::{
#[derive(Clone)]
pub enum AudioCallback {
Js(Arc<ThreadsafeFunction<Float32Array, ()>>),
Channel(Sender<Vec<f32>>),
Channel {
sender: Sender<Vec<f32>>,
overflow_count: Arc<AtomicU64>,
},
}
impl AudioCallback {
@@ -22,10 +28,38 @@ impl AudioCallback {
// audio thread.
let _ = func.call(Ok(samples.into()), ThreadsafeFunctionCallMode::NonBlocking);
}
Self::Channel(sender) => {
// Drop the chunk if the channel is full to avoid blocking capture.
let _ = sender.try_send(samples);
}
Self::Channel { sender, overflow_count } => match sender.try_send(samples) {
Ok(()) => {}
Err(TrySendError::Full(_)) => {
overflow_count.fetch_add(1, Ordering::Relaxed);
}
Err(TrySendError::Disconnected(_)) => {}
},
}
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::Ordering;
use crossbeam_channel::bounded;
use super::AudioCallback;
#[test]
fn channel_overflow_only_increments_the_counter() {
let (sender, _rx) = bounded(1);
let overflow_count = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
let callback = AudioCallback::Channel {
sender: sender.clone(),
overflow_count: overflow_count.clone(),
};
sender.send(vec![0.0]).unwrap();
callback.call(vec![1.0]);
callback.call(vec![2.0]);
assert_eq!(overflow_count.load(Ordering::Relaxed), 2);
}
}
packages/frontend/native/media_capture/src/recording.rs
+421 -107
View File
@@ -2,7 +2,10 @@ use std::{
fs,
io::{BufWriter, Write},
path::PathBuf,
sync::{LazyLock, Mutex},
sync::{
Arc, LazyLock,
atomic::{AtomicU64, Ordering},
},
thread::{self, JoinHandle},
time::{SystemTime, UNIX_EPOCH},
};
@@ -13,6 +16,7 @@ use napi_derive::napi;
use ogg::writing::{PacketWriteEndInfo, PacketWriter};
use opus_codec::{Application, Channels, Encoder, FrameSize, SampleRate as OpusSampleRate};
use rubato::Resampler;
use tokio::sync::{Mutex as AsyncMutex, mpsc, oneshot};
#[cfg(any(target_os = "macos", target_os = "windows"))]
use crate::audio_callback::AudioCallback;
@@ -24,6 +28,7 @@ use crate::windows::screen_capture_kit::ShareableContent;
const ENCODE_SAMPLE_RATE: OpusSampleRate = OpusSampleRate::Hz48000;
const MAX_PACKET_SIZE: usize = 4096;
const RESAMPLER_INPUT_CHUNK: usize = 1024;
const AUDIO_CHUNK_QUEUE_CAPACITY: usize = 1024;
type RecordingResult<T> = std::result::Result<T, RecordingError>;
@@ -55,6 +60,8 @@ pub struct RecordingArtifact {
pub channels: u32,
pub duration_ms: i64,
pub size: i64,
pub degraded: bool,
pub overflow_count: u32,
}
#[derive(Debug, thiserror::Error)]
@@ -77,6 +84,8 @@ enum RecordingError {
Empty,
#[error("start failure: {0}")]
Start(String),
#[error("teardown failure: {0}")]
Teardown(String),
#[error("join failure")]
Join,
}
@@ -93,6 +102,7 @@ impl RecordingError {
RecordingError::NotFound => "not-found",
RecordingError::Empty => "empty-recording",
RecordingError::Start(_) => "start-failure",
RecordingError::Teardown(_) => "teardown-failure",
RecordingError::Join => "join-failure",
}
}
@@ -448,6 +458,8 @@ impl OggOpusWriter {
channels: self.channels.as_usize() as u32,
duration_ms,
size,
degraded: false,
overflow_count: 0,
})
}
}
@@ -507,17 +519,41 @@ impl PlatformCapture {
}
enum ControlMessage {
Stop(Sender<RecordingResult<RecordingArtifact>>),
Stop {
reply_tx: oneshot::Sender<RecordingResult<RecordingArtifact>>,
},
Abort {
reply_tx: oneshot::Sender<RecordingResult<()>>,
},
}
struct ActiveRecording {
id: String,
control_tx: Sender<ControlMessage>,
control_tx: mpsc::UnboundedSender<ControlMessage>,
controller: Option<JoinHandle<()>>,
}
static ACTIVE_RECORDING: LazyLock<Mutex<Option<ActiveRecording>>> = LazyLock::new(|| Mutex::new(None));
static START_RECORDING_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
#[derive(Default)]
struct RecordingQualityMetrics {
overflow_count: Arc<AtomicU64>,
}
impl RecordingQualityMetrics {
fn shared_counter(&self) -> Arc<AtomicU64> {
Arc::clone(&self.overflow_count)
}
fn overflow_count(&self) -> u32 {
self
.overflow_count
.load(Ordering::Relaxed)
.try_into()
.unwrap_or(u32::MAX)
}
}
static ACTIVE_RECORDING: LazyLock<AsyncMutex<Option<ActiveRecording>>> = LazyLock::new(|| AsyncMutex::new(None));
static START_RECORDING_LOCK: LazyLock<AsyncMutex<()>> = LazyLock::new(|| AsyncMutex::new(()));
fn now_millis() -> i64 {
SystemTime::now()
@@ -568,10 +604,17 @@ fn build_excluded_refs(ids: &[u32]) -> Result<Vec<ApplicationInfo>> {
Ok(excluded)
}
fn start_capture(opts: &RecordingStartOptions, tx: Sender<Vec<f32>>) -> Result<(PlatformCapture, u32, u32)> {
fn start_capture(
opts: &RecordingStartOptions,
tx: Sender<Vec<f32>>,
overflow_count: Arc<AtomicU64>,
) -> Result<(PlatformCapture, u32, u32)> {
#[cfg(target_os = "macos")]
{
let callback = AudioCallback::Channel(tx);
let callback = AudioCallback::Channel {
sender: tx,
overflow_count,
};
let session = if let Some(app_id) = opts.app_process_id {
ShareableContent::tap_audio_with_callback(app_id, callback)?
} else {
@@ -586,7 +629,10 @@ fn start_capture(opts: &RecordingStartOptions, tx: Sender<Vec<f32>>) -> Result<(
#[cfg(target_os = "windows")]
{
let callback = AudioCallback::Channel(tx);
let callback = AudioCallback::Channel {
sender: tx,
overflow_count,
};
let session =
ShareableContent::tap_audio_with_callback(opts.app_process_id.unwrap_or(0), callback, opts.sample_rate)?;
let sample_rate = session.get_sample_rate().round() as u32;
@@ -598,6 +644,7 @@ fn start_capture(opts: &RecordingStartOptions, tx: Sender<Vec<f32>>) -> Result<(
{
let _ = opts;
let _ = tx;
let _ = overflow_count;
Err(RecordingError::UnsupportedPlatform.into())
}
}
@@ -621,17 +668,65 @@ fn spawn_worker(
})
}
fn finalize_worker_artifact(
worker_result: RecordingResult<RecordingArtifact>,
metrics: &RecordingQualityMetrics,
) -> RecordingResult<RecordingArtifact> {
worker_result.map(|mut artifact| {
artifact.overflow_count = metrics.overflow_count();
artifact.degraded = artifact.overflow_count > 0;
artifact
})
}
fn resolve_stop_result(
stop_result: std::result::Result<(), Error>,
worker_result: RecordingResult<RecordingArtifact>,
metrics: &RecordingQualityMetrics,
) -> RecordingResult<RecordingArtifact> {
match finalize_worker_artifact(worker_result, metrics) {
Ok(artifact) => Ok(artifact),
Err(worker_error) => match stop_result {
Ok(()) => Err(worker_error),
Err(error) => Err(RecordingError::Teardown(error.to_string())),
},
}
}
fn resolve_abort_result(
stop_result: std::result::Result<(), Error>,
worker_result: RecordingResult<RecordingArtifact>,
) -> RecordingResult<()> {
match worker_result {
Ok(artifact) => {
fs::remove_file(&artifact.filepath).ok();
Ok(())
}
Err(RecordingError::Empty) => Ok(()),
Err(worker_error) => match stop_result {
Ok(()) => Err(worker_error),
Err(error) => Err(RecordingError::Teardown(error.to_string())),
},
}
}
fn spawn_recording_controller(
id: String,
filepath: PathBuf,
opts: RecordingStartOptions,
) -> (Receiver<RecordingResult<u32>>, Sender<ControlMessage>, JoinHandle<()>) {
let (started_tx, started_rx) = bounded(1);
let (control_tx, control_rx) = bounded(1);
) -> (
oneshot::Receiver<RecordingResult<u32>>,
mpsc::UnboundedSender<ControlMessage>,
JoinHandle<()>,
) {
let (started_tx, started_rx) = oneshot::channel();
let (control_tx, mut control_rx) = mpsc::unbounded_channel();
let controller = thread::spawn(move || {
let (tx, rx) = bounded::<Vec<f32>>(32);
let (mut capture, capture_rate, capture_channels) = match start_capture(&opts, tx.clone()) {
let (tx, rx) = bounded::<Vec<f32>>(AUDIO_CHUNK_QUEUE_CAPACITY);
let metrics = RecordingQualityMetrics::default();
let (mut capture, capture_rate, capture_channels) = match start_capture(&opts, tx.clone(), metrics.shared_counter())
{
Ok(capture) => capture,
Err(error) => {
let _ = started_tx.send(Err(RecordingError::Start(error.to_string())));
@@ -675,28 +770,35 @@ fn spawn_recording_controller(
return;
}
while let Ok(message) = control_rx.recv() {
if let Some(message) = control_rx.blocking_recv() {
match message {
ControlMessage::Stop(reply_tx) => {
let result = match capture.stop() {
Ok(()) => {
drop(audio_tx.take());
match worker.take() {
Some(handle) => match handle.join() {
Ok(result) => result,
Err(_) => Err(RecordingError::Join),
},
None => Err(RecordingError::Join),
}
}
Err(error) => Err(RecordingError::Start(error.to_string())),
ControlMessage::Stop { reply_tx } => {
let stop_result = capture.stop();
drop(audio_tx.take());
let worker_result = match worker.take() {
Some(handle) => match handle.join() {
Ok(result) => result,
Err(_) => Err(RecordingError::Join),
},
None => Err(RecordingError::Join),
};
let result = resolve_stop_result(stop_result, worker_result, &metrics);
let _ = reply_tx.send(result);
}
ControlMessage::Abort { reply_tx } => {
let stop_result = capture.stop();
drop(audio_tx.take());
let worker_result = match worker.take() {
Some(handle) => match handle.join() {
Ok(result) => result,
Err(_) => Err(RecordingError::Join),
},
None => Err(RecordingError::Join),
};
let result = resolve_abort_result(stop_result, worker_result);
if worker.is_none() {
break;
}
let _ = reply_tx.send(result);
}
}
}
@@ -711,17 +813,49 @@ fn spawn_recording_controller(
(started_rx, control_tx, controller)
}
fn cleanup_recording_controller(control_tx: &Sender<ControlMessage>, controller: JoinHandle<()>) {
let (reply_tx, reply_rx) = bounded(1);
let _ = control_tx.send(ControlMessage::Stop(reply_tx));
let _ = reply_rx.recv();
let _ = controller.join();
async fn join_controller_handle(controller: JoinHandle<()>) -> RecordingResult<()> {
tokio::task::spawn_blocking(move || controller.join().map_err(|_| RecordingError::Join))
.await
.map_err(|_| RecordingError::Join)?
}
fn take_active_recording(id: &str) -> RecordingResult<ActiveRecording> {
let mut active_recording = ACTIVE_RECORDING
.lock()
.map_err(|_| RecordingError::Start("lock poisoned".into()))?;
async fn cleanup_recording_controller(control_tx: &mpsc::UnboundedSender<ControlMessage>, controller: JoinHandle<()>) {
let (reply_tx, reply_rx) = oneshot::channel();
let _ = control_tx.send(ControlMessage::Abort { reply_tx });
let _ = reply_rx.await;
let _ = join_controller_handle(controller).await;
}
fn map_recording_result<T>(result: RecordingResult<T>) -> Result<T> {
result.map_err(Into::into)
}
async fn send_control_message<T>(
id: &str,
message: ControlMessage,
reply_rx: oneshot::Receiver<RecordingResult<T>>,
) -> Result<T> {
let active_recording = take_active_recording(id).await?;
if active_recording.control_tx.send(message).is_err() {
let _ = join_active_recording(active_recording).await;
return Err(RecordingError::Join.into());
}
let response = match reply_rx.await {
Ok(response) => response,
Err(_) => {
let _ = join_active_recording(active_recording).await;
return Err(RecordingError::Join.into());
}
};
join_active_recording(active_recording).await?;
map_recording_result(response)
}
async fn take_active_recording(id: &str) -> RecordingResult<ActiveRecording> {
let mut active_recording = ACTIVE_RECORDING.lock().await;
let recording = active_recording.take().ok_or(RecordingError::NotFound)?;
if recording.id != id {
*active_recording = Some(recording);
@@ -730,15 +864,14 @@ fn take_active_recording(id: &str) -> RecordingResult<ActiveRecording> {
Ok(recording)
}
fn join_active_recording(mut recording: ActiveRecording) -> RecordingResult<()> {
async fn join_active_recording(mut recording: ActiveRecording) -> RecordingResult<()> {
if let Some(handle) = recording.controller.take() {
handle.join().map_err(|_| RecordingError::Join)?;
join_controller_handle(handle).await?;
}
Ok(())
}
#[napi]
pub fn start_recording(opts: RecordingStartOptions) -> Result<RecordingSessionMeta> {
async fn start_recording_inner(opts: RecordingStartOptions) -> Result<RecordingSessionMeta> {
if let Some(fmt) = opts.format.as_deref()
&& !fmt.eq_ignore_ascii_case("opus")
{
@@ -748,17 +881,12 @@ pub fn start_recording(opts: RecordingStartOptions) -> Result<RecordingSessionMe
normalize_channel_count(channels)?;
}
let _start_lock = START_RECORDING_LOCK
.lock()
.map_err(|_| RecordingError::Start("lock poisoned".into()))?;
let _start_lock = START_RECORDING_LOCK.lock().await;
let output_dir = validate_output_dir(&opts.output_dir)?;
let id = sanitize_id(opts.id.clone());
{
let recording = ACTIVE_RECORDING
.lock()
.map_err(|_| RecordingError::Start("lock poisoned".into()))?;
let recording = ACTIVE_RECORDING.lock().await;
if recording.is_some() {
return Err(RecordingError::Start("recording already active".into()).into());
}
@@ -770,9 +898,17 @@ pub fn start_recording(opts: RecordingStartOptions) -> Result<RecordingSessionMe
}
let (started_rx, control_tx, controller) = spawn_recording_controller(id.clone(), filepath.clone(), opts);
let encoding_channels = started_rx
.recv()
.map_err(|_| RecordingError::Start("failed to start recording controller".into()))??;
let encoding_channels = match started_rx.await {
Ok(Ok(channels)) => channels,
Ok(Err(error)) => {
let _ = join_controller_handle(controller).await;
return Err(error.into());
}
Err(_) => {
let _ = join_controller_handle(controller).await;
return Err(RecordingError::Start("failed to start recording controller".into()).into());
}
};
let meta = RecordingSessionMeta {
id: id.clone(),
@@ -782,16 +918,10 @@ pub fn start_recording(opts: RecordingStartOptions) -> Result<RecordingSessionMe
started_at: now_millis(),
};
let mut recording = match ACTIVE_RECORDING.lock() {
Ok(recording) => recording,
Err(_) => {
cleanup_recording_controller(&control_tx, controller);
return Err(RecordingError::Start("lock poisoned".into()).into());
}
};
let mut recording = ACTIVE_RECORDING.lock().await;
if recording.is_some() {
cleanup_recording_controller(&control_tx, controller);
cleanup_recording_controller(&control_tx, controller).await;
return Err(RecordingError::Start("recording already active".into()).into());
}
@@ -805,63 +935,51 @@ pub fn start_recording(opts: RecordingStartOptions) -> Result<RecordingSessionMe
}
#[napi]
pub fn stop_recording(id: String) -> Result<RecordingArtifact> {
let control_tx = {
let recording = ACTIVE_RECORDING
.lock()
.map_err(|_| RecordingError::Start("lock poisoned".into()))?;
pub async fn start_recording(opts: RecordingStartOptions) -> Result<RecordingSessionMeta> {
start_recording_inner(opts).await
}
let active = recording.as_ref().ok_or(RecordingError::NotFound)?;
if active.id != id {
return Err(RecordingError::NotFound.into());
}
active.control_tx.clone()
};
async fn stop_recording_inner(id: String) -> Result<RecordingArtifact> {
let (reply_tx, reply_rx) = oneshot::channel();
send_control_message(&id, ControlMessage::Stop { reply_tx }, reply_rx).await
}
let (reply_tx, reply_rx) = bounded(1);
if control_tx.send(ControlMessage::Stop(reply_tx)).is_err() {
if let Ok(recording) = take_active_recording(&id) {
let _ = join_active_recording(recording);
}
return Err(RecordingError::Join.into());
}
#[napi]
pub async fn stop_recording(id: String) -> Result<RecordingArtifact> {
stop_recording_inner(id).await
}
let response = match reply_rx.recv() {
Ok(response) => response,
Err(_) => {
if let Ok(recording) = take_active_recording(&id) {
let _ = join_active_recording(recording);
}
return Err(RecordingError::Join.into());
}
};
async fn abort_recording_inner(id: String) -> Result<()> {
let (reply_tx, reply_rx) = oneshot::channel();
send_control_message(&id, ControlMessage::Abort { reply_tx }, reply_rx).await
}
let artifact = match response {
Ok(artifact) => artifact,
Err(RecordingError::Start(message)) => {
return Err(RecordingError::Start(message).into());
}
Err(error) => {
if let Ok(recording) = take_active_recording(&id) {
let _ = join_active_recording(recording);
}
return Err(error.into());
}
};
let active_recording = take_active_recording(&id)?;
join_active_recording(active_recording)?;
Ok(artifact)
#[napi]
pub async fn abort_recording(id: String) -> Result<()> {
abort_recording_inner(id).await
}
#[cfg(test)]
mod tests {
use std::{env, fs::File, path::PathBuf};
use std::{env, fs::File, path::PathBuf, thread};
use napi::{Error, Status};
use ogg::PacketReader;
use tokio::runtime::Builder;
use super::{OggOpusWriter, convert_interleaved_channels};
use super::{
ACTIVE_RECORDING, ActiveRecording, ControlMessage, OggOpusWriter, RecordingArtifact, RecordingError,
RecordingQualityMetrics, START_RECORDING_LOCK, abort_recording_inner, bounded, convert_interleaved_channels, mpsc,
resolve_abort_result, resolve_stop_result, stop_recording_inner,
};
use crate::audio_callback::AudioCallback;
fn block_on<F: std::future::Future>(future: F) -> F::Output {
Builder::new_current_thread()
.build()
.expect("create runtime")
.block_on(future)
}
fn temp_recording_path() -> PathBuf {
env::temp_dir().join(format!("affine-recording-test-{}.opus", rand::random::<u64>()))
@@ -939,4 +1057,200 @@ mod tests {
vec![3.0, 3.0, 4.0, 4.0]
);
}
#[test]
fn stop_recording_clears_active_session_after_stop_error() {
let _lock = START_RECORDING_LOCK.blocking_lock();
let id = String::from("stop-error");
let (control_tx, mut control_rx) = mpsc::unbounded_channel();
let controller = thread::spawn(move || {
let Some(ControlMessage::Stop { reply_tx }) = control_rx.blocking_recv() else {
panic!("expected stop");
};
let _ = reply_tx.send(Err(RecordingError::Teardown(String::from("boom"))));
});
*ACTIVE_RECORDING.blocking_lock() = Some(ActiveRecording {
id: id.clone(),
control_tx,
controller: Some(controller),
});
let error = match block_on(stop_recording_inner(id)) {
Ok(_) => panic!("stop should fail"),
Err(error) => error,
};
assert!(
error.to_string().contains("teardown failure: boom"),
"unexpected error: {error}"
);
assert!(ACTIVE_RECORDING.blocking_lock().is_none());
}
#[test]
fn stop_recording_returns_artifact_and_clears_active_session() {
let _lock = START_RECORDING_LOCK.blocking_lock();
let id = String::from("stop-success");
let (control_tx, mut control_rx) = mpsc::unbounded_channel();
let controller = thread::spawn(move || {
let Some(ControlMessage::Stop { reply_tx }) = control_rx.blocking_recv() else {
panic!("expected stop");
};
let _ = reply_tx.send(Ok(RecordingArtifact {
id: String::from("stop-success"),
filepath: String::from("/tmp/recording.opus"),
sample_rate: 48_000,
channels: 2,
duration_ms: 1_000,
size: 128,
degraded: true,
overflow_count: 3,
}));
});
*ACTIVE_RECORDING.blocking_lock() = Some(ActiveRecording {
id: id.clone(),
control_tx,
controller: Some(controller),
});
let artifact = block_on(stop_recording_inner(id)).expect("stop should succeed");
assert_eq!(artifact.filepath, "/tmp/recording.opus");
assert!(artifact.degraded);
assert_eq!(artifact.overflow_count, 3);
assert!(ACTIVE_RECORDING.blocking_lock().is_none());
}
#[test]
fn abort_recording_clears_active_session() {
let _lock = START_RECORDING_LOCK.blocking_lock();
let id = String::from("abort-success");
let (control_tx, mut control_rx) = mpsc::unbounded_channel();
let controller = thread::spawn(move || {
let Some(ControlMessage::Abort { reply_tx }) = control_rx.blocking_recv() else {
panic!("expected abort");
};
let _ = reply_tx.send(Ok(()));
});
*ACTIVE_RECORDING.blocking_lock() = Some(ActiveRecording {
id: id.clone(),
control_tx,
controller: Some(controller),
});
block_on(abort_recording_inner(id)).expect("abort should succeed");
assert!(ACTIVE_RECORDING.blocking_lock().is_none());
}
#[test]
fn concurrent_stop_and_abort_only_allow_one_owner() {
let _lock = START_RECORDING_LOCK.blocking_lock();
let id = String::from("concurrent-stop-abort");
let (control_tx, mut control_rx) = mpsc::unbounded_channel();
let (stop_started_tx, stop_started_rx) = std::sync::mpsc::channel();
let (finish_stop_tx, finish_stop_rx) = std::sync::mpsc::channel();
let controller = thread::spawn(move || {
let Some(ControlMessage::Stop { reply_tx }) = control_rx.blocking_recv() else {
panic!("expected stop");
};
stop_started_tx.send(()).expect("signal stop start");
finish_stop_rx.recv().expect("wait for stop completion");
let _ = reply_tx.send(Ok(RecordingArtifact {
id: String::from("concurrent-stop-abort"),
filepath: String::from("/tmp/recording.opus"),
sample_rate: 48_000,
channels: 2,
duration_ms: 1_000,
size: 128,
degraded: false,
overflow_count: 0,
}));
});
*ACTIVE_RECORDING.blocking_lock() = Some(ActiveRecording {
id: id.clone(),
control_tx,
controller: Some(controller),
});
let stop_id = id.clone();
let stop_thread = thread::spawn(move || block_on(stop_recording_inner(stop_id)));
stop_started_rx.recv().expect("stop should own the session");
let abort_error = block_on(abort_recording_inner(id)).expect_err("abort should lose ownership");
assert!(
abort_error.to_string().contains("not-found"),
"unexpected abort error: {abort_error}"
);
finish_stop_tx.send(()).expect("allow stop to finish");
let stop_result = stop_thread.join().expect("join stop thread");
assert!(stop_result.is_ok(), "stop should win ownership");
assert!(ACTIVE_RECORDING.blocking_lock().is_none());
}
#[test]
fn queue_overflow_marks_recording_as_degraded() {
let metrics = RecordingQualityMetrics::default();
let (sender, receiver) = bounded(1);
let callback = AudioCallback::Channel {
sender,
overflow_count: metrics.shared_counter(),
};
callback.call(vec![0.1, 0.2]);
callback.call(vec![0.3, 0.4]);
let _ = receiver.recv().expect("queued audio");
assert_eq!(metrics.overflow_count(), 1);
}
#[test]
fn stop_prefers_a_finished_artifact_over_teardown_errors() {
let metrics = RecordingQualityMetrics::default();
metrics.shared_counter().store(2, std::sync::atomic::Ordering::Relaxed);
let artifact = resolve_stop_result(
Err(Error::new(Status::GenericFailure, "pause failed")),
Ok(RecordingArtifact {
id: String::from("stop-success"),
filepath: String::from("/tmp/recording.opus"),
sample_rate: 48_000,
channels: 2,
duration_ms: 1_000,
size: 128,
degraded: false,
overflow_count: 0,
}),
&metrics,
)
.expect("artifact should be preserved");
assert_eq!(artifact.overflow_count, 2);
assert!(artifact.degraded);
}
#[test]
fn abort_cleans_artifacts_even_when_teardown_reports_an_error() {
let path = temp_recording_path();
std::fs::write(&path, b"artifact").expect("write temp artifact");
resolve_abort_result(
Err(Error::new(Status::GenericFailure, "pause failed")),
Ok(RecordingArtifact {
id: String::from("abort-success"),
filepath: path.to_string_lossy().into_owned(),
sample_rate: 48_000,
channels: 2,
duration_ms: 1_000,
size: 8,
degraded: false,
overflow_count: 0,
}),
)
.expect("abort should still clean up");
assert!(!path.exists());
}
}
packages/frontend/native/media_capture/src/windows/audio_capture.rs
+216 -102
View File
@@ -1,6 +1,4 @@
use std::{
cell::RefCell,
collections::HashMap,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
@@ -49,9 +47,33 @@ impl BufferedResampler {
}
}
fn append_output(&mut self, out_blocks: &[Vec<f32>], interleaved_out: &mut Vec<f32>, final_flush: bool) {
if out_blocks.is_empty() || out_blocks.len() != self.channels {
return;
}
if !self.initial_output_discarded && !final_flush {
self.initial_output_discarded = true;
return;
}
self.initial_output_discarded = true;
let out_len = out_blocks[0].len();
for i in 0..out_len {
for channel in out_blocks.iter().take(self.channels) {
interleaved_out.push(channel[i]);
}
}
}
fn process_chunk(&mut self, chunk: Vec<Vec<f32>>, interleaved_out: &mut Vec<f32>, final_flush: bool) {
if let Ok(out_blocks) = self.resampler.process(&chunk, None) {
self.append_output(&out_blocks, interleaved_out, final_flush);
}
}
// Feed planar samples; returns interleaved output (may be empty)
fn feed(&mut self, planar_in: &[Vec<f32>]) -> Vec<f32> {
// Push incoming samples into fifo buffers
for (ch, data) in planar_in.iter().enumerate() {
if ch < self.fifo.len() {
self.fifo[ch].extend_from_slice(data);
@@ -68,44 +90,27 @@ impl BufferedResampler {
chunk.push(tail);
}
if let Ok(out_blocks) = self.resampler.process(&chunk, None) {
if !out_blocks.is_empty() && out_blocks.len() == self.channels {
if !self.initial_output_discarded {
self.initial_output_discarded = true;
} else {
let out_len = out_blocks[0].len();
for i in 0..out_len {
for ch in 0..self.channels {
interleaved_out.push(out_blocks[ch][i]);
}
}
}
}
}
self.process_chunk(chunk, &mut interleaved_out, false);
}
interleaved_out
}
}
// Thread-local cache for resamplers keyed by (from, to, channels)
thread_local! {
static RESAMPLER_CACHE: RefCell<HashMap<(u32, u32, usize), BufferedResampler>> = RefCell::new(HashMap::new());
}
fn finish(&mut self) -> Vec<f32> {
let mut interleaved_out = Vec::new();
if self.fifo.first().is_none_or(|channel| channel.is_empty()) {
return interleaved_out;
}
fn process_audio_with_resampler(samples: Vec<f32>, from_sample_rate: u32, to_sample_rate: u32) -> Vec<f32> {
if from_sample_rate == to_sample_rate {
return samples;
let mut chunk: Vec<Vec<f32>> = Vec::with_capacity(self.channels);
for ch in 0..self.channels {
let mut tail = std::mem::take(&mut self.fifo[ch]);
tail.resize(RESAMPLER_INPUT_CHUNK, 0.0);
chunk.push(tail);
}
self.process_chunk(chunk, &mut interleaved_out, true);
interleaved_out
}
RESAMPLER_CACHE.with(|cache| {
let mut map = cache.borrow_mut();
let key = (from_sample_rate, to_sample_rate, 1usize); // mono resampler
let resampler = map
.entry(key)
.or_insert_with(|| BufferedResampler::new(from_sample_rate as f64, to_sample_rate as f64, 1));
resampler.feed(&[samples])
})
}
fn to_mono(frame: &[f32]) -> f32 {
@@ -159,16 +164,100 @@ struct AudioBuffer {
data: Vec<f32>,
}
fn extend_post_buffer(post_buffer: &mut Vec<f32>, mono_samples: Vec<f32>, resampler: &mut Option<BufferedResampler>) {
if let Some(resampler) = resampler.as_mut() {
post_buffer.extend(resampler.feed(&[mono_samples]));
} else {
post_buffer.extend(mono_samples);
}
}
fn emit_mixed_frames(
post_mic: &mut Vec<f32>,
post_lb: &mut Vec<f32>,
audio_buffer_callback: &AudioCallback,
final_flush: bool,
) {
while post_mic.len() >= TARGET_FRAME_SIZE && post_lb.len() >= TARGET_FRAME_SIZE {
let mic_chunk: Vec<f32> = post_mic.drain(..TARGET_FRAME_SIZE).collect();
let lb_chunk: Vec<f32> = post_lb.drain(..TARGET_FRAME_SIZE).collect();
let mixed = mix(&mic_chunk, &lb_chunk);
if !mixed.is_empty() {
audio_buffer_callback.call(mixed);
}
}
if final_flush && !post_mic.is_empty() && !post_lb.is_empty() {
let tail_len = post_mic.len().min(post_lb.len());
let mic_chunk: Vec<f32> = post_mic.drain(..tail_len).collect();
let lb_chunk: Vec<f32> = post_lb.drain(..tail_len).collect();
let mixed = mix(&mic_chunk, &lb_chunk);
if !mixed.is_empty() {
audio_buffer_callback.call(mixed);
}
post_mic.clear();
post_lb.clear();
}
}
#[napi]
pub struct AudioCaptureSession {
mic_stream: cpal::Stream,
lb_stream: cpal::Stream,
stopped: Arc<AtomicBool>,
mic_stream: Option<cpal::Stream>,
lb_stream: Option<cpal::Stream>,
stop_requested: Arc<AtomicBool>,
sample_rate: SampleRate,
channels: u32,
jh: Option<JoinHandle<()>>, // background mixing thread
}
fn teardown_audio_capture_resources<S, F>(
mic_stream: &mut Option<S>,
lb_stream: &mut Option<S>,
stop_requested: &Arc<AtomicBool>,
jh: &mut Option<JoinHandle<()>>,
pause_stream: F,
) -> Result<()>
where
F: Fn(&S) -> std::result::Result<(), String>,
{
if mic_stream.is_none() && lb_stream.is_none() && jh.is_none() {
return Ok(());
}
let mic_stream = mic_stream.take();
let lb_stream = lb_stream.take();
let jh = jh.take();
let mut pause_errors = Vec::new();
if let Some(stream) = mic_stream.as_ref() {
if let Err(error) = pause_stream(stream) {
pause_errors.push(format!("pause mic stream: {error}"));
}
}
if let Some(stream) = lb_stream.as_ref() {
if let Err(error) = pause_stream(stream) {
pause_errors.push(format!("pause loopback stream: {error}"));
}
}
stop_requested.store(true, Ordering::SeqCst);
drop(mic_stream);
drop(lb_stream);
if let Some(jh) = jh {
let _ = jh.join(); // ignore poison
}
if pause_errors.is_empty() {
Ok(())
} else {
Err(Error::new(Status::GenericFailure, pause_errors.join("; ")))
}
}
#[napi]
impl AudioCaptureSession {
#[napi(getter)]
@@ -189,22 +278,13 @@ impl AudioCaptureSession {
#[napi]
pub fn stop(&mut self) -> Result<()> {
if self.stopped.load(Ordering::SeqCst) {
return Ok(());
}
self
.mic_stream
.pause()
.map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?;
self
.lb_stream
.pause()
.map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?;
self.stopped.store(true, Ordering::SeqCst);
if let Some(jh) = self.jh.take() {
let _ = jh.join(); // ignore poison
}
Ok(())
teardown_audio_capture_resources(
&mut self.mic_stream,
&mut self.lb_stream,
&self.stop_requested,
&mut self.jh,
|stream| stream.pause().map_err(|error| error.to_string()),
)
}
}
@@ -250,7 +330,7 @@ pub fn start_recording(
let mic_stream_config: cpal::StreamConfig = mic_config.clone().into();
let lb_stream_config: cpal::StreamConfig = lb_config.clone().into();
let stopped = Arc::new(AtomicBool::new(false));
let stop_requested = Arc::new(AtomicBool::new(false));
// Channels for passing raw buffers between callback and mixer thread
let (tx_mic, rx_mic) = unbounded::<AudioBuffer>();
@@ -281,24 +361,25 @@ pub fn start_recording(
)
.map_err(|e| Error::new(Status::GenericFailure, format!("build_lb_stream: {e}")))?;
let stopped_flag = stopped.clone();
let stop_requested_flag = stop_requested.clone();
let jh = std::thread::spawn(move || {
// Accumulators before and after resampling
let mut pre_mic: Vec<f32> = Vec::new();
let mut pre_lb: Vec<f32> = Vec::new();
let mut post_mic: Vec<f32> = Vec::new();
let mut post_lb: Vec<f32> = Vec::new();
let mut mic_resampler = (mic_sample_rate != target_rate)
.then(|| BufferedResampler::new(mic_sample_rate.0 as f64, target_rate.0 as f64, 1));
let mut lb_resampler =
(lb_sample_rate != target_rate).then(|| BufferedResampler::new(lb_sample_rate.0 as f64, target_rate.0 as f64, 1));
let mut flushed_tail = false;
while !stopped_flag.load(Ordering::SeqCst) {
// Gather input from channels
loop {
while let Ok(buf) = rx_mic.try_recv() {
let mono_samples: Vec<f32> = if mic_channels == 1 {
buf.data
} else {
buf.data.chunks(mic_channels as usize).map(to_mono).collect()
};
pre_mic.extend_from_slice(&mono_samples);
extend_post_buffer(&mut post_mic, mono_samples, &mut mic_resampler);
}
while let Ok(buf) = rx_lb.try_recv() {
@@ -307,44 +388,10 @@ pub fn start_recording(
} else {
buf.data.chunks(lb_channels as usize).map(to_mono).collect()
};
pre_lb.extend_from_slice(&mono_samples);
extend_post_buffer(&mut post_lb, mono_samples, &mut lb_resampler);
}
// Resample when enough samples are available
while pre_mic.len() >= RESAMPLER_INPUT_CHUNK {
let to_resample: Vec<f32> = pre_mic.drain(..RESAMPLER_INPUT_CHUNK).collect();
let processed = process_audio_with_resampler(to_resample, mic_sample_rate.0, target_rate.0);
if !processed.is_empty() {
post_mic.extend_from_slice(&processed);
}
}
while pre_lb.len() >= RESAMPLER_INPUT_CHUNK {
let to_resample: Vec<f32> = pre_lb.drain(..RESAMPLER_INPUT_CHUNK).collect();
let processed = process_audio_with_resampler(to_resample, lb_sample_rate.0, target_rate.0);
if !processed.is_empty() {
post_lb.extend_from_slice(&processed);
}
}
// Mix when we have TARGET_FRAME_SIZE samples available from both
while post_mic.len() >= TARGET_FRAME_SIZE && post_lb.len() >= TARGET_FRAME_SIZE {
let mic_chunk: Vec<f32> = post_mic.drain(..TARGET_FRAME_SIZE).collect();
let lb_chunk: Vec<f32> = post_lb.drain(..TARGET_FRAME_SIZE).collect();
let mixed = mix(&mic_chunk, &lb_chunk);
if !mixed.is_empty() {
audio_buffer_callback.call(mixed);
}
}
// Prevent unbounded growth – keep some slack
const MAX_PRE: usize = RESAMPLER_INPUT_CHUNK * 10;
if pre_mic.len() > MAX_PRE {
pre_mic.drain(..pre_mic.len() - MAX_PRE);
}
if pre_lb.len() > MAX_PRE {
pre_lb.drain(..pre_lb.len() - MAX_PRE);
}
emit_mixed_frames(&mut post_mic, &mut post_lb, &audio_buffer_callback, false);
const MAX_POST: usize = TARGET_FRAME_SIZE * 10;
if post_mic.len() > MAX_POST {
@@ -354,8 +401,24 @@ pub fn start_recording(
post_lb.drain(..post_lb.len() - MAX_POST);
}
// Sleep if nothing to do
if rx_mic.is_empty()
let stop_requested = stop_requested_flag.load(Ordering::SeqCst);
if stop_requested && !flushed_tail && rx_mic.is_empty() && rx_lb.is_empty() {
if let Some(mut resampler) = mic_resampler.take() {
post_mic.extend(resampler.finish());
}
if let Some(mut resampler) = lb_resampler.take() {
post_lb.extend(resampler.finish());
}
emit_mixed_frames(&mut post_mic, &mut post_lb, &audio_buffer_callback, true);
flushed_tail = true;
}
if stop_requested && flushed_tail && rx_mic.is_empty() && rx_lb.is_empty() {
break;
}
if !stop_requested
&& rx_mic.is_empty()
&& rx_lb.is_empty()
&& post_mic.len() < TARGET_FRAME_SIZE
&& post_lb.len() < TARGET_FRAME_SIZE
@@ -373,11 +436,62 @@ pub fn start_recording(
.map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?;
Ok(AudioCaptureSession {
mic_stream,
lb_stream,
stopped,
mic_stream: Some(mic_stream),
lb_stream: Some(lb_stream),
stop_requested,
sample_rate: target_rate,
channels: 1, // mono output
jh: Some(jh),
})
}
#[cfg(test)]
mod tests {
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
use super::teardown_audio_capture_resources;
#[test]
fn teardown_consumes_resources_even_when_pause_fails() {
let mut mic_stream = Some("mic");
let mut lb_stream = Some("loopback");
let stop_requested = Arc::new(AtomicBool::new(false));
let joined = Arc::new(AtomicBool::new(false));
let joined_flag = joined.clone();
let mut jh = Some(std::thread::spawn(move || {
joined_flag.store(true, Ordering::SeqCst);
}));
let result = teardown_audio_capture_resources(&mut mic_stream, &mut lb_stream, &stop_requested, &mut jh, |_| {
Err("pause failed".to_owned())
});
assert!(result.is_err());
assert!(stop_requested.load(Ordering::SeqCst));
assert!(mic_stream.is_none());
assert!(lb_stream.is_none());
assert!(jh.is_none());
assert!(joined.load(Ordering::SeqCst));
}
#[test]
fn teardown_is_idempotent_after_resources_are_consumed() {
let mut mic_stream: Option<&'static str> = None;
let mut lb_stream: Option<&'static str> = None;
let stop_requested = Arc::new(AtomicBool::new(true));
let mut jh = None;
let result = teardown_audio_capture_resources(
&mut mic_stream,
&mut lb_stream,
&stop_requested,
&mut jh,
|_| -> std::result::Result<(), String> { panic!("pause should not be called when resources are already gone") },
);
assert!(result.is_ok());
}
}