mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-21 03:56:23 +08:00
feat(native): record encoding (#14188)
fix #13784 <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Start/stop system or meeting recordings with Ogg/Opus artifacts and native start/stop APIs; workspace backup recovery. * **Refactor** * Simplified recording lifecycle and UI flows; native runtime now orchestrates recording/processing and reporting. * **Bug Fixes** * Stronger path validation, safer import/export dialogs, consistent error handling/logging, and retry-safe recording processing. * **Chores** * Added cross-platform native audio capture and Ogg/Opus encoding support. * **Tests** * New unit, integration, and e2e tests for recording, path guards, dialogs, and workspace recovery. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
@@ -13,6 +13,19 @@ import type { FrameworkProvider } from '@toeverything/infra';
|
||||
import { getCurrentWorkspace, isAiEnabled } from './utils';
|
||||
|
||||
const logger = new DebugLogger('electron-renderer:recording');
|
||||
const RECORDING_PROCESS_RETRY_MS = 1000;
|
||||
const NATIVE_RECORDING_MIME_TYPE = 'audio/ogg';
|
||||
|
||||
type ProcessingRecordingStatus = {
|
||||
id: number;
|
||||
status: 'processing';
|
||||
appName?: string;
|
||||
blockCreationStatus?: undefined;
|
||||
filepath: string;
|
||||
startTime: number;
|
||||
};
|
||||
|
||||
type WorkspaceHandle = NonNullable<ReturnType<typeof getCurrentWorkspace>>;
|
||||
|
||||
async function readRecordingFile(filepath: string) {
|
||||
if (apis?.recording?.readRecordingFile) {
|
||||
@@ -45,118 +58,217 @@ async function saveRecordingBlob(blobEngine: BlobEngine, filepath: string) {
|
||||
logger.debug('Saving recording', filepath);
|
||||
const opusBuffer = await readRecordingFile(filepath);
|
||||
const blob = new Blob([opusBuffer], {
|
||||
type: 'audio/mp4',
|
||||
type: NATIVE_RECORDING_MIME_TYPE,
|
||||
});
|
||||
const blobId = await blobEngine.set(blob);
|
||||
logger.debug('Recording saved', blobId);
|
||||
return { blob, blobId };
|
||||
}
|
||||
|
||||
export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
|
||||
events?.recording.onRecordingStatusChanged(status => {
|
||||
(async () => {
|
||||
if ((await apis?.ui.isActiveTab()) && status?.status === 'ready') {
|
||||
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
|
||||
if (!currentWorkspace) {
|
||||
// maybe the workspace is not ready yet, eg. for shared workspace view
|
||||
await apis?.recording.handleBlockCreationFailed(status.id);
|
||||
return;
|
||||
}
|
||||
const { workspace } = currentWorkspace;
|
||||
const docsService = workspace.scope.get(DocsService);
|
||||
const aiEnabled = isAiEnabled(frameworkProvider);
|
||||
function shouldProcessRecording(
|
||||
status: unknown
|
||||
): status is ProcessingRecordingStatus {
|
||||
return (
|
||||
!!status &&
|
||||
typeof status === 'object' &&
|
||||
'status' in status &&
|
||||
status.status === 'processing' &&
|
||||
'filepath' in status &&
|
||||
typeof status.filepath === 'string' &&
|
||||
!('blockCreationStatus' in status && status.blockCreationStatus)
|
||||
);
|
||||
}
|
||||
|
||||
const timestamp = i18nTime(status.startTime, {
|
||||
absolute: {
|
||||
accuracy: 'minute',
|
||||
noYear: true,
|
||||
},
|
||||
});
|
||||
async function createRecordingDoc(
|
||||
frameworkProvider: FrameworkProvider,
|
||||
workspace: WorkspaceHandle['workspace'],
|
||||
status: ProcessingRecordingStatus
|
||||
) {
|
||||
const docsService = workspace.scope.get(DocsService);
|
||||
const aiEnabled = isAiEnabled(frameworkProvider);
|
||||
const recordingFilepath = status.filepath;
|
||||
|
||||
const docProps: DocProps = {
|
||||
onStoreLoad: (doc, { noteId }) => {
|
||||
(async () => {
|
||||
if (status.filepath) {
|
||||
// it takes a while to save the blob, so we show the attachment first
|
||||
const { blobId, blob } = await saveRecordingBlob(
|
||||
doc.workspace.blobSync,
|
||||
status.filepath
|
||||
);
|
||||
const timestamp = i18nTime(status.startTime, {
|
||||
absolute: {
|
||||
accuracy: 'minute',
|
||||
noYear: true,
|
||||
},
|
||||
});
|
||||
|
||||
// name + timestamp(readable) + extension
|
||||
const attachmentName =
|
||||
(status.appName ?? 'System Audio') +
|
||||
' ' +
|
||||
timestamp +
|
||||
'.opus';
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const docProps: DocProps = {
|
||||
onStoreLoad: (doc, { noteId }) => {
|
||||
void (async () => {
|
||||
// it takes a while to save the blob, so we show the attachment first
|
||||
const { blobId, blob } = await saveRecordingBlob(
|
||||
doc.workspace.blobSync,
|
||||
recordingFilepath
|
||||
);
|
||||
|
||||
// add size and sourceId to the attachment later
|
||||
const attachmentId = doc.addBlock(
|
||||
'affine:attachment',
|
||||
{
|
||||
name: attachmentName,
|
||||
type: 'audio/opus',
|
||||
size: blob.size,
|
||||
sourceId: blobId,
|
||||
embed: true,
|
||||
},
|
||||
noteId
|
||||
);
|
||||
// name + timestamp(readable) + extension
|
||||
const attachmentName =
|
||||
(status.appName ?? 'System Audio') + ' ' + timestamp + '.opus';
|
||||
|
||||
const model = doc.getBlock(attachmentId)
|
||||
?.model as AttachmentBlockModel;
|
||||
const attachmentId = doc.addBlock(
|
||||
'affine:attachment',
|
||||
{
|
||||
name: attachmentName,
|
||||
type: NATIVE_RECORDING_MIME_TYPE,
|
||||
size: blob.size,
|
||||
sourceId: blobId,
|
||||
embed: true,
|
||||
},
|
||||
noteId
|
||||
);
|
||||
|
||||
if (!aiEnabled) {
|
||||
return;
|
||||
}
|
||||
const model = doc.getBlock(attachmentId)
|
||||
?.model as AttachmentBlockModel;
|
||||
|
||||
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
const { workspace } = currentWorkspace;
|
||||
using audioAttachment = workspace.scope
|
||||
.get(AudioAttachmentService)
|
||||
.get(model);
|
||||
audioAttachment?.obj
|
||||
.transcribe()
|
||||
.then(() => {
|
||||
track.doc.editor.audioBlock.transcribeRecording({
|
||||
type: 'Meeting record',
|
||||
method: 'success',
|
||||
option: 'Auto transcribing',
|
||||
});
|
||||
})
|
||||
.catch(err => {
|
||||
logger.error('Failed to transcribe recording', err);
|
||||
});
|
||||
} else {
|
||||
throw new Error('No attachment model found');
|
||||
}
|
||||
})()
|
||||
.then(async () => {
|
||||
await apis?.recording.handleBlockCreationSuccess(status.id);
|
||||
})
|
||||
.catch(error => {
|
||||
logger.error('Failed to transcribe recording', error);
|
||||
return apis?.recording.handleBlockCreationFailed(
|
||||
status.id,
|
||||
error
|
||||
);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('unknown error', error);
|
||||
if (!aiEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
const { workspace } = currentWorkspace;
|
||||
using audioAttachment = workspace.scope
|
||||
.get(AudioAttachmentService)
|
||||
.get(model);
|
||||
audioAttachment?.obj
|
||||
.transcribe()
|
||||
.then(() => {
|
||||
track.doc.editor.audioBlock.transcribeRecording({
|
||||
type: 'Meeting record',
|
||||
method: 'success',
|
||||
option: 'Auto transcribing',
|
||||
});
|
||||
},
|
||||
};
|
||||
const page = docsService.createDoc({
|
||||
docProps,
|
||||
title:
|
||||
'Recording ' + (status.appName ?? 'System Audio') + ' ' + timestamp,
|
||||
primaryMode: 'page',
|
||||
});
|
||||
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
|
||||
}
|
||||
})().catch(console.error);
|
||||
})
|
||||
.catch(err => {
|
||||
logger.error('Failed to transcribe recording', err);
|
||||
});
|
||||
})().then(resolve, reject);
|
||||
},
|
||||
};
|
||||
|
||||
const page = docsService.createDoc({
|
||||
docProps,
|
||||
title:
|
||||
'Recording ' + (status.appName ?? 'System Audio') + ' ' + timestamp,
|
||||
primaryMode: 'page',
|
||||
});
|
||||
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
|
||||
});
|
||||
}
|
||||
|
||||
export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
|
||||
let pendingStatus: ProcessingRecordingStatus | null = null;
|
||||
let retryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let processingStatusId: number | null = null;
|
||||
|
||||
const clearRetry = () => {
|
||||
if (retryTimer !== null) {
|
||||
clearTimeout(retryTimer);
|
||||
retryTimer = null;
|
||||
}
|
||||
};
|
||||
|
||||
const clearPending = (id?: number) => {
|
||||
if (id === undefined || pendingStatus?.id === id) {
|
||||
pendingStatus = null;
|
||||
clearRetry();
|
||||
}
|
||||
if (id === undefined || processingStatusId === id) {
|
||||
processingStatusId = null;
|
||||
}
|
||||
};
|
||||
|
||||
const scheduleRetry = () => {
|
||||
if (!pendingStatus || retryTimer !== null) {
|
||||
return;
|
||||
}
|
||||
retryTimer = setTimeout(() => {
|
||||
retryTimer = null;
|
||||
void processPendingStatus().catch(console.error);
|
||||
}, RECORDING_PROCESS_RETRY_MS);
|
||||
};
|
||||
|
||||
const processPendingStatus = async () => {
|
||||
const status = pendingStatus;
|
||||
if (!status || processingStatusId === status.id) {
|
||||
return;
|
||||
}
|
||||
|
||||
let isActiveTab = false;
|
||||
try {
|
||||
isActiveTab = !!(await apis?.ui.isActiveTab());
|
||||
} catch (error) {
|
||||
logger.error('Failed to probe active recording tab', error);
|
||||
scheduleRetry();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isActiveTab) {
|
||||
scheduleRetry();
|
||||
return;
|
||||
}
|
||||
|
||||
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
|
||||
if (!currentWorkspace) {
|
||||
// Workspace can lag behind the post-recording status update for a short
|
||||
// time; keep retrying instead of permanently failing the import.
|
||||
scheduleRetry();
|
||||
return;
|
||||
}
|
||||
|
||||
processingStatusId = status.id;
|
||||
|
||||
try {
|
||||
await createRecordingDoc(
|
||||
frameworkProvider,
|
||||
currentWorkspace.workspace,
|
||||
status
|
||||
);
|
||||
await apis?.recording.setRecordingBlockCreationStatus(
|
||||
status.id,
|
||||
'success'
|
||||
);
|
||||
clearPending(status.id);
|
||||
} catch (error) {
|
||||
logger.error('Failed to create recording block', error);
|
||||
try {
|
||||
await apis?.recording.setRecordingBlockCreationStatus(
|
||||
status.id,
|
||||
'failed',
|
||||
error instanceof Error ? error.message : undefined
|
||||
);
|
||||
} finally {
|
||||
clearPending(status.id);
|
||||
}
|
||||
} finally {
|
||||
if (pendingStatus?.id === status.id) {
|
||||
processingStatusId = null;
|
||||
scheduleRetry();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
events?.recording.onRecordingStatusChanged(status => {
|
||||
if (shouldProcessRecording(status)) {
|
||||
pendingStatus = status;
|
||||
clearRetry();
|
||||
void processPendingStatus().catch(console.error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!status) {
|
||||
clearPending();
|
||||
return;
|
||||
}
|
||||
|
||||
if (pendingStatus?.id === status.id) {
|
||||
clearPending(status.id);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user