mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-02-13 21:05:19 +00:00
refactor(electron): encoding recording on the fly (#11457)
fix AF-2460, AF-2463
When recording is started, we start polling the pending raw buffers that are waiting for encoding. The buffers are determined by the cursor of the original raw buffer file. When recording is stopped, we will flush the pending buffers and wrap the encoded chunks into WebM.
```mermaid
sequenceDiagram
participant App as App/UI
participant RecordingFeature as Recording Feature
participant StateMachine as State Machine
participant FileSystem as File System
participant StreamEncoder as Stream Encoder
participant OpusEncoder as Opus Encoder
participant WebM as WebM Muxer
Note over App,WebM: Recording Start Flow
App->>RecordingFeature: startRecording()
RecordingFeature->>StateMachine: dispatch(START_RECORDING)
StateMachine-->>RecordingFeature: status: 'recording'
RecordingFeature->>StreamEncoder: createStreamEncoder(id, {sampleRate, channels})
Note over App,WebM: Streaming Flow
loop Audio Data Streaming
RecordingFeature->>FileSystem: Write raw audio chunks to .raw file
StreamEncoder->>FileSystem: Poll raw audio data
FileSystem-->>StreamEncoder: Raw audio chunks
StreamEncoder->>OpusEncoder: Encode chunks
OpusEncoder-->>StreamEncoder: Encoded Opus frames
end
Note over App,WebM: Recording Stop Flow
App->>RecordingFeature: stopRecording()
RecordingFeature->>StateMachine: dispatch(STOP_RECORDING)
StateMachine-->>RecordingFeature: status: 'stopped'
StreamEncoder->>OpusEncoder: flush()
StreamEncoder->>WebM: muxToWebM(encodedChunks)
WebM-->>RecordingFeature: WebM buffer
RecordingFeature->>FileSystem: Save as .opus file
RecordingFeature->>StateMachine: dispatch(SAVE_RECORDING)
```
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import { DebugLogger } from '@affine/debug';
|
||||
import { apis } from '@affine/electron-api';
|
||||
import { ArrayBufferTarget, Muxer } from 'webm-muxer';
|
||||
|
||||
interface AudioEncodingConfig {
|
||||
@@ -12,10 +13,10 @@ const logger = new DebugLogger('webm-encoding');
|
||||
/**
|
||||
* Creates and configures an Opus encoder with the given settings
|
||||
*/
|
||||
async function createOpusEncoder(config: AudioEncodingConfig): Promise<{
|
||||
export function createOpusEncoder(config: AudioEncodingConfig): {
|
||||
encoder: AudioEncoder;
|
||||
encodedChunks: EncodedAudioChunk[];
|
||||
}> {
|
||||
} {
|
||||
const encodedChunks: EncodedAudioChunk[] = [];
|
||||
const encoder = new AudioEncoder({
|
||||
output: chunk => {
|
||||
@@ -81,7 +82,7 @@ async function encodeAudioFrames({
|
||||
/**
|
||||
* Creates a WebM container with the encoded audio chunks
|
||||
*/
|
||||
function muxToWebM(
|
||||
export function muxToWebM(
|
||||
encodedChunks: EncodedAudioChunk[],
|
||||
config: AudioEncodingConfig
|
||||
): Uint8Array {
|
||||
@@ -121,7 +122,7 @@ export async function encodeRawBufferToOpus({
|
||||
throw new Error('Response body is null');
|
||||
}
|
||||
|
||||
const { encoder, encodedChunks } = await createOpusEncoder({
|
||||
const { encoder, encodedChunks } = createOpusEncoder({
|
||||
sampleRate,
|
||||
numberOfChannels,
|
||||
});
|
||||
@@ -193,7 +194,7 @@ export async function encodeAudioBlobToOpus(
|
||||
bitrate: targetBitrate,
|
||||
};
|
||||
|
||||
const { encoder, encodedChunks } = await createOpusEncoder(config);
|
||||
const { encoder, encodedChunks } = createOpusEncoder(config);
|
||||
|
||||
// Combine all channels into a single Float32Array
|
||||
const audioData = new Float32Array(
|
||||
@@ -220,3 +221,95 @@ export async function encodeAudioBlobToOpus(
|
||||
await audioContext.close();
|
||||
}
|
||||
}
|
||||
|
||||
export const createStreamEncoder = (
|
||||
recordingId: number,
|
||||
codecs: {
|
||||
sampleRate: number;
|
||||
numberOfChannels: number;
|
||||
targetBitrate?: number;
|
||||
}
|
||||
) => {
|
||||
const { encoder, encodedChunks } = createOpusEncoder({
|
||||
sampleRate: codecs.sampleRate,
|
||||
numberOfChannels: codecs.numberOfChannels,
|
||||
bitrate: codecs.targetBitrate,
|
||||
});
|
||||
|
||||
const toAudioData = (buffer: Uint8Array) => {
|
||||
// Each sample in f32 format is 4 bytes
|
||||
const BYTES_PER_SAMPLE = 4;
|
||||
return new AudioData({
|
||||
format: 'f32',
|
||||
sampleRate: codecs.sampleRate,
|
||||
numberOfChannels: codecs.numberOfChannels,
|
||||
numberOfFrames:
|
||||
buffer.length / BYTES_PER_SAMPLE / codecs.numberOfChannels,
|
||||
timestamp: 0,
|
||||
data: buffer,
|
||||
});
|
||||
};
|
||||
|
||||
let cursor = 0;
|
||||
let isClosed = false;
|
||||
|
||||
const next = async () => {
|
||||
if (!apis || isClosed) {
|
||||
throw new Error('Electron API is not available');
|
||||
}
|
||||
const { buffer, nextCursor } = await apis.recording.getRawAudioBuffers(
|
||||
recordingId,
|
||||
cursor
|
||||
);
|
||||
if (isClosed || cursor === nextCursor) {
|
||||
return;
|
||||
}
|
||||
cursor = nextCursor;
|
||||
logger.debug('Encoding next chunk', cursor, nextCursor);
|
||||
encoder.encode(toAudioData(buffer));
|
||||
};
|
||||
|
||||
const poll = async () => {
|
||||
if (isClosed) {
|
||||
return;
|
||||
}
|
||||
logger.debug('Polling next chunk');
|
||||
await next();
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
await poll();
|
||||
};
|
||||
|
||||
const close = () => {
|
||||
if (isClosed) {
|
||||
return;
|
||||
}
|
||||
isClosed = true;
|
||||
return encoder.close();
|
||||
};
|
||||
|
||||
return {
|
||||
id: recordingId,
|
||||
next,
|
||||
poll,
|
||||
flush: () => {
|
||||
return encoder.flush();
|
||||
},
|
||||
close,
|
||||
finish: async () => {
|
||||
logger.debug('Finishing encoding');
|
||||
await next();
|
||||
close();
|
||||
const buffer = muxToWebM(encodedChunks, {
|
||||
sampleRate: codecs.sampleRate,
|
||||
numberOfChannels: codecs.numberOfChannels,
|
||||
bitrate: codecs.targetBitrate,
|
||||
});
|
||||
return buffer;
|
||||
},
|
||||
[Symbol.dispose]: () => {
|
||||
close();
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export type OpusStreamEncoder = ReturnType<typeof createStreamEncoder>;
|
||||
|
||||
Reference in New Issue
Block a user