mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-02-27 02:42:25 +08:00
feat(electron): recording popups (#11016)
Added a recording popup UI for the audio recording feature in the desktop app, improving the user experience when capturing audio from applications. ### What changed? - Created a new popup window system for displaying recording controls - Added a dedicated recording UI with start/stop controls and status indicators - Moved audio encoding logic from the main app to a dedicated module - Implemented smooth animations for popup appearance/disappearance - Updated the recording workflow to show visual feedback during recording process - Added internationalization support for recording-related text - Modified the recording status flow to include new states: new, recording, stopped, ready fix AF-2340
This commit is contained in:
pengx17
@@ -5,134 +5,21 @@ import { AudioAttachmentService } from '@affine/core/modules/media/services/audi
|
||||
import { WorkbenchService } from '@affine/core/modules/workbench';
|
||||
import { DebugLogger } from '@affine/debug';
|
||||
import { apis, events } from '@affine/electron-api';
|
||||
import { i18nTime } from '@affine/i18n';
|
||||
import type { AttachmentBlockModel } from '@blocksuite/affine/model';
|
||||
import { Text } from '@blocksuite/affine/store';
|
||||
import type { BlobEngine } from '@blocksuite/affine/sync';
|
||||
import type { FrameworkProvider } from '@toeverything/infra';
|
||||
import { ArrayBufferTarget, Muxer } from 'webm-muxer';
|
||||
|
||||
import { getCurrentWorkspace } from './utils';
|
||||
|
||||
const logger = new DebugLogger('electron-renderer:recording');
|
||||
|
||||
/**
|
||||
* Encodes raw audio data to Opus in WebM container.
|
||||
*/
|
||||
async function encodeRawBufferToOpus({
|
||||
filepath,
|
||||
sampleRate,
|
||||
numberOfChannels,
|
||||
}: {
|
||||
filepath: string;
|
||||
sampleRate: number;
|
||||
numberOfChannels: number;
|
||||
}): Promise<Uint8Array> {
|
||||
// Use streams to process audio data incrementally
|
||||
const response = await fetch(new URL(filepath, location.origin));
|
||||
if (!response.body) {
|
||||
throw new Error('Response body is null');
|
||||
}
|
||||
|
||||
// Setup Opus encoder
|
||||
const encodedChunks: EncodedAudioChunk[] = [];
|
||||
const encoder = new AudioEncoder({
|
||||
output: chunk => {
|
||||
encodedChunks.push(chunk);
|
||||
},
|
||||
error: err => {
|
||||
throw new Error(`Encoding error: ${err}`);
|
||||
},
|
||||
});
|
||||
|
||||
// Configure Opus encoder
|
||||
encoder.configure({
|
||||
codec: 'opus',
|
||||
sampleRate: sampleRate,
|
||||
numberOfChannels: numberOfChannels,
|
||||
bitrate: 96000, // 96 kbps is good for stereo audio
|
||||
});
|
||||
|
||||
// Process the stream
|
||||
const reader = response.body.getReader();
|
||||
let offset = 0;
|
||||
const CHUNK_SIZE = numberOfChannels * 1024; // Process 1024 samples per channel at a time
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
// Convert the chunk to Float32Array
|
||||
const float32Data = new Float32Array(value.buffer);
|
||||
|
||||
// Process in smaller chunks to avoid large frames
|
||||
for (let i = 0; i < float32Data.length; i += CHUNK_SIZE) {
|
||||
const chunkSize = Math.min(CHUNK_SIZE, float32Data.length - i);
|
||||
const chunk = float32Data.subarray(i, i + chunkSize);
|
||||
|
||||
// Create and encode frame
|
||||
const frame = new AudioData({
|
||||
format: 'f32',
|
||||
sampleRate: sampleRate,
|
||||
numberOfFrames: chunk.length / numberOfChannels,
|
||||
numberOfChannels: numberOfChannels,
|
||||
timestamp: (offset * 1000000) / sampleRate, // timestamp in microseconds
|
||||
data: chunk,
|
||||
});
|
||||
|
||||
encoder.encode(frame);
|
||||
frame.close();
|
||||
|
||||
offset += chunk.length / numberOfChannels;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await encoder.flush();
|
||||
encoder.close();
|
||||
}
|
||||
|
||||
if (encodedChunks.length === 0) {
|
||||
throw new Error('No chunks were produced during encoding');
|
||||
}
|
||||
|
||||
// Initialize WebM muxer
|
||||
const target = new ArrayBufferTarget();
|
||||
const muxer = new Muxer({
|
||||
target,
|
||||
audio: {
|
||||
codec: 'A_OPUS',
|
||||
sampleRate: sampleRate,
|
||||
numberOfChannels: numberOfChannels,
|
||||
},
|
||||
});
|
||||
|
||||
// Add all chunks to the muxer
|
||||
for (const chunk of encodedChunks) {
|
||||
muxer.addAudioChunk(chunk, {});
|
||||
}
|
||||
|
||||
// Finalize and get WebM container
|
||||
muxer.finalize();
|
||||
const { buffer: webmBuffer } = target;
|
||||
|
||||
return new Uint8Array(webmBuffer);
|
||||
}
|
||||
|
||||
async function saveRecordingBlob(
|
||||
blobEngine: BlobEngine,
|
||||
recording: {
|
||||
id: number;
|
||||
filepath: string;
|
||||
sampleRate: number;
|
||||
numberOfChannels: number;
|
||||
}
|
||||
) {
|
||||
logger.debug('Saving recording', recording.id);
|
||||
const opusBuffer = await encodeRawBufferToOpus({
|
||||
filepath: recording.filepath,
|
||||
sampleRate: recording.sampleRate,
|
||||
numberOfChannels: recording.numberOfChannels,
|
||||
});
|
||||
async function saveRecordingBlob(blobEngine: BlobEngine, filepath: string) {
|
||||
logger.debug('Saving recording', filepath);
|
||||
const opusBuffer = await fetch(new URL(filepath, location.origin)).then(res =>
|
||||
res.arrayBuffer()
|
||||
);
|
||||
const blob = new Blob([opusBuffer], {
|
||||
type: 'audio/webm',
|
||||
});
|
||||
@@ -144,7 +31,7 @@ async function saveRecordingBlob(
|
||||
export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
|
||||
events?.recording.onRecordingStatusChanged(status => {
|
||||
(async () => {
|
||||
if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') {
|
||||
if ((await apis?.ui.isActiveTab()) && status?.status === 'ready') {
|
||||
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
@@ -155,30 +42,28 @@ export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
|
||||
const docsService = workspace.scope.get(DocsService);
|
||||
const editorSetting = editorSettingService.editorSetting;
|
||||
|
||||
const timestamp = i18nTime(status.startTime, {
|
||||
absolute: {
|
||||
accuracy: 'minute',
|
||||
noYear: true,
|
||||
},
|
||||
});
|
||||
|
||||
const docProps: DocProps = {
|
||||
note: editorSetting.get('affine:note'),
|
||||
page: {
|
||||
title: new Text(
|
||||
'Recording ' +
|
||||
(status.appGroup?.name ?? 'System Audio') +
|
||||
(status.appName ?? 'System Audio') +
|
||||
' ' +
|
||||
new Date(status.startTime).toISOString()
|
||||
timestamp
|
||||
),
|
||||
},
|
||||
onStoreLoad: (doc, { noteId }) => {
|
||||
(async () => {
|
||||
const recording = await apis?.recording.getRecording(status.id);
|
||||
if (!recording) {
|
||||
logger.error('Failed to save recording');
|
||||
return;
|
||||
}
|
||||
|
||||
// name + timestamp(readable) + extension
|
||||
const attachmentName =
|
||||
(status.appGroup?.name ?? 'System Audio') +
|
||||
' ' +
|
||||
new Date(status.startTime).toISOString() +
|
||||
'.webm';
|
||||
(status.appName ?? 'System Audio') + ' ' + timestamp + '.webm';
|
||||
|
||||
// add size and sourceId to the attachment later
|
||||
const attachmentId = doc.addBlock(
|
||||
@@ -193,11 +78,11 @@ export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
|
||||
const model = doc.getBlock(attachmentId)
|
||||
?.model as AttachmentBlockModel;
|
||||
|
||||
if (model) {
|
||||
if (model && status.filepath) {
|
||||
// it takes a while to save the blob, so we show the attachment first
|
||||
const { blobId, blob } = await saveRecordingBlob(
|
||||
doc.workspace.blobSync,
|
||||
recording
|
||||
status.filepath
|
||||
);
|
||||
|
||||
model.props.size = blob.size;
|
||||
|
||||
Reference in New Issue
Block a user