feat(native): decode audio and mp3 encoder (#10490)

packages/backend/native/package.json

@@ -33,7 +33,7 @@
     "build:debug": "napi build"
   },
   "devDependencies": {
-    "@napi-rs/cli": "3.0.0-alpha.70",
+    "@napi-rs/cli": "3.0.0-alpha.72",
     "lib0": "^0.2.99",
     "tiktoken": "^1.0.17",
     "tinybench": "^3.0.7",

packages/frontend/native/__tests__/audio.spec.mts

@@ -0,0 +1,23 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { tmpdir } from 'node:os';
+
+import test from 'ava';
+
+import { decodeAudio, Mp3Encoder } from '../index.js';
+
+const __dirname = join(fileURLToPath(import.meta.url), '..');
+
+const wav = await readFile(join(__dirname, 'fixtures', 'recording.wav'));
+
+test('convert wav to mp3', async t => {
+  const audio = await decodeAudio(wav);
+  const mp3 = new Mp3Encoder({
+    channels: 1,
+  });
+  await t.notThrowsAsync(async () => {
+    const mp3Data = mp3.encode(audio);
+    await writeFile(join(tmpdir(), 'recording.mp3'), mp3Data);
+  });
+});

packages/frontend/native/index.d.ts

@@ -61,6 +61,11 @@ export declare class DocStoragePool {
   clearClocks(universalId: string): Promise<void>
 }
 
+export declare class Mp3Encoder {
+  constructor(options: EncodeOptions)
+  encode(input: Float32Array): Uint8Array
+}
+
 export declare class RecordingPermissions {
   audio: boolean
   screen: boolean
@@ -113,6 +118,42 @@ export declare class SqliteConnection {
   checkpoint(): Promise<void>
 }
 
+/**Enumeration of valid values for `set_brate` */
+export declare enum Bitrate {
+  /**8_000 */
+  Kbps8 = 8,
+  /**16_000 */
+  Kbps16 = 16,
+  /**24_000 */
+  Kbps24 = 24,
+  /**32_000 */
+  Kbps32 = 32,
+  /**40_000 */
+  Kbps40 = 40,
+  /**48_000 */
+  Kbps48 = 48,
+  /**64_000 */
+  Kbps64 = 64,
+  /**80_000 */
+  Kbps80 = 80,
+  /**96_000 */
+  Kbps96 = 96,
+  /**112_000 */
+  Kbps112 = 112,
+  /**128_000 */
+  Kbps128 = 128,
+  /**160_000 */
+  Kbps160 = 160,
+  /**192_000 */
+  Kbps192 = 192,
+  /**224_000 */
+  Kbps224 = 224,
+  /**256_000 */
+  Kbps256 = 256,
+  /**320_000 */
+  Kbps320 = 320
+}
+
 export interface Blob {
   key: string
   data: Uint8Array
@@ -127,6 +168,11 @@ export interface BlobRow {
   timestamp: Date
 }
 
+export declare function decodeAudio(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null, signal?: AbortSignal | undefined | null): Promise<Float32Array>
+
+/** Decode audio file into a Float32Array */
+export declare function decodeAudioSync(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null): Float32Array
+
 export interface DocClock {
   docId: string
   timestamp: Date
@@ -149,6 +195,14 @@ export interface DocUpdate {
   bin: Uint8Array
 }
 
+export interface EncodeOptions {
+  channels: number
+  quality?: Quality
+  bitrate?: Bitrate
+  sampleRate?: number
+  mode?: Mode
+}
+
 export interface InsertRow {
   docId?: string
   data: Uint8Array
@@ -163,6 +217,42 @@ export interface ListedBlob {
 
 export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise<string>
 
+/** MPEG mode */
+export declare enum Mode {
+  Mono = 0,
+  Stereo = 1,
+  JointStereo = 2,
+  DualChannel = 3,
+  NotSet = 4
+}
+
+/**
+ *Possible quality parameter.
+ *From best(0) to worst(9)
+ */
+export declare enum Quality {
+  /**Best possible quality */
+  Best = 0,
+  /**Second best */
+  SecondBest = 1,
+  /**Close to best */
+  NearBest = 2,
+  /**Very nice */
+  VeryNice = 3,
+  /**Nice */
+  Nice = 4,
+  /**Good */
+  Good = 5,
+  /**Decent */
+  Decent = 6,
+  /**Okayish */
+  Ok = 7,
+  /**Almost worst */
+  SecondWorst = 8,
+  /**Worst */
+  Worst = 9
+}
+
 export interface SetBlob {
   key: string
   data: Uint8Array

packages/frontend/native/index.js

@@ -60,7 +60,13 @@ const isMuslFromChildProcess = () => {
 }
 
 function requireNative() {
-  if (process.platform === 'android') {
+  if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) {
+    try {
+      nativeBinding = require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH);
+    } catch (err) {
+      loadErrors.push(err);
+    }
+  } else if (process.platform === 'android') {
     if (process.arch === 'arm64') {
       try {
         return require('./affine.android-arm64.node')
@@ -370,9 +376,15 @@ module.exports.ApplicationStateChangedSubscriber = nativeBinding.ApplicationStat
 module.exports.AudioTapStream = nativeBinding.AudioTapStream
 module.exports.DocStorage = nativeBinding.DocStorage
 module.exports.DocStoragePool = nativeBinding.DocStoragePool
+module.exports.Mp3Encoder = nativeBinding.Mp3Encoder
 module.exports.RecordingPermissions = nativeBinding.RecordingPermissions
 module.exports.ShareableContent = nativeBinding.ShareableContent
 module.exports.SqliteConnection = nativeBinding.SqliteConnection
+module.exports.Bitrate = nativeBinding.Bitrate
+module.exports.decodeAudio = nativeBinding.decodeAudio
+module.exports.decodeAudioSync = nativeBinding.decodeAudioSync
 module.exports.mintChallengeResponse = nativeBinding.mintChallengeResponse
+module.exports.Mode = nativeBinding.Mode
+module.exports.Quality = nativeBinding.Quality
 module.exports.ValidationResult = nativeBinding.ValidationResult
 module.exports.verifyChallengeResponse = nativeBinding.verifyChallengeResponse

packages/frontend/native/media_capture/Cargo.toml

@@ -7,9 +7,12 @@ version = "0.0.0"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-napi        = { workspace = true, features = ["napi4"] }
-napi-derive = { workspace = true, features = ["type-def"] }
-rubato      = { workspace = true }
+mp3lame-encoder = { workspace = true, features = ["std"] }
+napi            = { workspace = true, features = ["napi4"] }
+napi-derive     = { workspace = true, features = ["type-def"] }
+rubato          = { workspace = true }
+symphonia       = { workspace = true, features = ["all", "opt-simd"] }
+thiserror       = { workspace = true }
 
 [target.'cfg(target_os = "macos")'.dependencies]
 block2           = { workspace = true }
@@ -20,7 +23,6 @@ libc             = { workspace = true }
 objc2            = { workspace = true }
 objc2-foundation = { workspace = true }
 screencapturekit = { workspace = true }
-thiserror        = { workspace = true }
 uuid             = { workspace = true, features = ["v4"] }
 
 [build-dependencies]

packages/frontend/native/media_capture/src/audio_decoder.rs

@@ -0,0 +1,179 @@
+use std::{io::Cursor, path::Path};
+
+use napi::{
+  bindgen_prelude::{AbortSignal, AsyncTask, Float32Array, Result, Status, Uint8Array},
+  Task,
+};
+use napi_derive::napi;
+use rubato::{Resampler, SincFixedIn, SincInterpolationParameters, SincInterpolationType};
+use symphonia::core::{
+  audio::{AudioBuffer, Signal},
+  codecs::DecoderOptions,
+  errors::Error,
+  formats::FormatOptions,
+  io::MediaSourceStream,
+  meta::MetadataOptions,
+  probe::Hint,
+};
+
+fn decode<B: AsRef<[u8]> + Send + Sync + 'static>(
+  buf: B,
+  dest_sample_rate: Option<u32>,
+  filename: Option<&str>,
+) -> std::result::Result<Vec<f32>, Error> {
+  // Create the media source
+  let mss = MediaSourceStream::new(Box::new(Cursor::new(buf)), Default::default());
+
+  // Create a probe hint using the file extension
+  let mut hint = Hint::new();
+  if let Some(ext) =
+    filename.and_then(|filename| Path::new(filename).extension().and_then(|ext| ext.to_str()))
+  {
+    hint.with_extension(ext);
+  }
+
+  let format_opts = FormatOptions {
+    enable_gapless: true,
+    ..Default::default()
+  };
+  let metadata_opts = MetadataOptions::default();
+  let decoder_opts = DecoderOptions::default();
+  let probed = symphonia::default::get_probe().format(&hint, mss, &format_opts, &metadata_opts)?;
+
+  let mut format = probed.format;
+
+  let track = format
+    .default_track()
+    .ok_or(Error::Unsupported("No default track found"))?;
+
+  let totol_samples = track
+    .codec_params
+    .n_frames
+    .ok_or(Error::Unsupported("No duration found"))?;
+  let sample_rate = track
+    .codec_params
+    .sample_rate
+    .ok_or(Error::Unsupported("No samplerate found"))?;
+
+  let mut decoder = symphonia::default::get_codecs().make(&track.codec_params, &decoder_opts)?;
+
+  let mut output: Vec<f32> = Vec::with_capacity(totol_samples as usize);
+  // Decode loop
+  while let Ok(packet) = format.next_packet() {
+    let decoded = decoder.decode(&packet)?;
+    let spec = decoded.spec();
+    let mut audio_buf: AudioBuffer<f32> = AudioBuffer::new(decoded.capacity() as u64, *spec);
+    decoded.convert(&mut audio_buf);
+
+    if spec.channels.count() > 1 {
+      // Mix all channels into mono
+      for i in 0..audio_buf.chan(0).len() {
+        let mut sample_sum = 0.0;
+        for ch in 0..spec.channels.count() {
+          sample_sum += audio_buf.chan(ch)[i];
+        }
+        output.push(sample_sum / spec.channels.count() as f32);
+      }
+    } else {
+      output.extend_from_slice(audio_buf.chan(0));
+    }
+  }
+
+  let Some(dest_sample_rate) = dest_sample_rate else {
+    return Ok(output);
+  };
+
+  if sample_rate != dest_sample_rate {
+    // Calculate parameters for resampling
+    let params = SincInterpolationParameters {
+      sinc_len: 256,
+      f_cutoff: 0.95,
+      interpolation: SincInterpolationType::Linear,
+      oversampling_factor: 256,
+      window: rubato::WindowFunction::BlackmanHarris2,
+    };
+
+    let mut resampler = SincFixedIn::<f32>::new(
+      dest_sample_rate as f64 / sample_rate as f64,
+      2.0,
+      params,
+      output.len(),
+      1,
+    )
+    .map_err(|_| Error::Unsupported("Failed to create resampler"))?;
+
+    let waves_in = vec![output];
+    let mut waves_out = resampler
+      .process(&waves_in, None)
+      .map_err(|_| Error::Unsupported("Failed to run resampler"))?;
+    output = waves_out
+      .pop()
+      .ok_or(Error::Unsupported("No resampled output found"))?;
+  }
+
+  Ok(output)
+}
+
+#[napi]
+/// Decode audio file into a Float32Array
+pub fn decode_audio_sync(
+  buf: Uint8Array,
+  dest_sample_rate: Option<u32>,
+  filename: Option<String>,
+) -> Result<Float32Array> {
+  decode(buf, dest_sample_rate, filename.as_deref())
+    .map(Float32Array::new)
+    .map_err(|e| {
+      napi::Error::new(
+        Status::InvalidArg,
+        format!("Decode audio into Float32Array failed: {e}"),
+      )
+    })
+}
+
+pub struct DecodeAudioTask {
+  buf: Uint8Array,
+  dest_sample_rate: Option<u32>,
+  filename: Option<String>,
+}
+
+#[napi]
+impl Task for DecodeAudioTask {
+  type Output = Vec<f32>;
+  type JsValue = Float32Array;
+
+  fn compute(&mut self) -> Result<Self::Output> {
+    decode(
+      std::mem::replace(&mut self.buf, Uint8Array::new(vec![])),
+      self.dest_sample_rate,
+      self.filename.as_deref(),
+    )
+    .map_err(|e| {
+      napi::Error::new(
+        Status::InvalidArg,
+        format!("Decode audio into Float32Array failed: {e}"),
+      )
+    })
+  }
+
+  fn resolve(&mut self, _: napi::Env, output: Self::Output) -> Result<Self::JsValue> {
+    Ok(Float32Array::new(output))
+  }
+}
+
+#[napi]
+pub fn decode_audio(
+  buf: Uint8Array,
+  dest_sample_rate: Option<u32>,
+  filename: Option<String>,
+  signal: Option<AbortSignal>,
+) -> AsyncTask<DecodeAudioTask> {
+  AsyncTask::with_optional_signal(
+    DecodeAudioTask {
+      buf,
+      dest_sample_rate,
+      filename,
+    },
+    signal,
+  )
+}

packages/frontend/native/media_capture/src/lib.rs

@@ -2,3 +2,5 @@
 pub mod macos;
 #[cfg(target_os = "macos")]
 pub(crate) use macos::*;
+pub mod audio_decoder;
+pub mod mp3;

packages/frontend/native/media_capture/src/mp3.rs

@@ -0,0 +1,215 @@
+use mp3lame_encoder::{Builder, Encoder, FlushNoGap, MonoPcm};
+use napi::bindgen_prelude::{Result, Uint8Array};
+use napi_derive::napi;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum LameError {
+  #[error("Create builder failed")]
+  CreateBuilderFailed,
+  #[error("Failed to create encoder")]
+  BuildError(#[from] mp3lame_encoder::BuildError),
+  #[error("Failed to encode")]
+  EncodeError(#[from] mp3lame_encoder::EncodeError),
+}
+
+impl From<LameError> for napi::Error {
+  fn from(value: LameError) -> Self {
+    napi::Error::new(napi::Status::GenericFailure, value.to_string())
+  }
+}
+
+#[napi]
+///Possible quality parameter.
+///From best(0) to worst(9)
+pub enum Quality {
+  ///Best possible quality
+  Best = 0,
+  ///Second best
+  SecondBest = 1,
+  ///Close to best
+  NearBest = 2,
+  ///Very nice
+  VeryNice = 3,
+  ///Nice
+  Nice = 4,
+  ///Good
+  Good = 5,
+  ///Decent
+  Decent = 6,
+  ///Okayish
+  Ok = 7,
+  ///Almost worst
+  SecondWorst = 8,
+  ///Worst
+  Worst = 9,
+}
+
+impl From<Quality> for mp3lame_encoder::Quality {
+  fn from(value: Quality) -> Self {
+    match value {
+      Quality::Best => mp3lame_encoder::Quality::Best,
+      Quality::SecondBest => mp3lame_encoder::Quality::SecondBest,
+      Quality::NearBest => mp3lame_encoder::Quality::NearBest,
+      Quality::VeryNice => mp3lame_encoder::Quality::VeryNice,
+      Quality::Nice => mp3lame_encoder::Quality::Nice,
+      Quality::Good => mp3lame_encoder::Quality::Good,
+      Quality::Decent => mp3lame_encoder::Quality::Decent,
+      Quality::Ok => mp3lame_encoder::Quality::Ok,
+      Quality::SecondWorst => mp3lame_encoder::Quality::SecondWorst,
+      Quality::Worst => mp3lame_encoder::Quality::Worst,
+    }
+  }
+}
+
+#[napi]
+#[repr(u16)]
+///Enumeration of valid values for `set_brate`
+pub enum Bitrate {
+  ///8_000
+  Kbps8 = 8,
+  ///16_000
+  Kbps16 = 16,
+  ///24_000
+  Kbps24 = 24,
+  ///32_000
+  Kbps32 = 32,
+  ///40_000
+  Kbps40 = 40,
+  ///48_000
+  Kbps48 = 48,
+  ///64_000
+  Kbps64 = 64,
+  ///80_000
+  Kbps80 = 80,
+  ///96_000
+  Kbps96 = 96,
+  ///112_000
+  Kbps112 = 112,
+  ///128_000
+  Kbps128 = 128,
+  ///160_000
+  Kbps160 = 160,
+  ///192_000
+  Kbps192 = 192,
+  ///224_000
+  Kbps224 = 224,
+  ///256_000
+  Kbps256 = 256,
+  ///320_000
+  Kbps320 = 320,
+}
+
+impl From<Bitrate> for mp3lame_encoder::Bitrate {
+  fn from(value: Bitrate) -> Self {
+    match value {
+      Bitrate::Kbps8 => mp3lame_encoder::Bitrate::Kbps8,
+      Bitrate::Kbps16 => mp3lame_encoder::Bitrate::Kbps16,
+      Bitrate::Kbps24 => mp3lame_encoder::Bitrate::Kbps24,
+      Bitrate::Kbps32 => mp3lame_encoder::Bitrate::Kbps32,
+      Bitrate::Kbps40 => mp3lame_encoder::Bitrate::Kbps40,
+      Bitrate::Kbps48 => mp3lame_encoder::Bitrate::Kbps48,
+      Bitrate::Kbps64 => mp3lame_encoder::Bitrate::Kbps64,
+      Bitrate::Kbps80 => mp3lame_encoder::Bitrate::Kbps80,
+      Bitrate::Kbps96 => mp3lame_encoder::Bitrate::Kbps96,
+      Bitrate::Kbps112 => mp3lame_encoder::Bitrate::Kbps112,
+      Bitrate::Kbps128 => mp3lame_encoder::Bitrate::Kbps128,
+      Bitrate::Kbps160 => mp3lame_encoder::Bitrate::Kbps160,
+      Bitrate::Kbps192 => mp3lame_encoder::Bitrate::Kbps192,
+      Bitrate::Kbps224 => mp3lame_encoder::Bitrate::Kbps224,
+      Bitrate::Kbps256 => mp3lame_encoder::Bitrate::Kbps256,
+      Bitrate::Kbps320 => mp3lame_encoder::Bitrate::Kbps320,
+    }
+  }
+}
+
+#[napi]
+/// MPEG mode
+pub enum Mode {
+  Mono,
+  Stereo,
+  JointStereo,
+  DualChannel,
+  NotSet,
+}
+
+impl From<Mode> for mp3lame_encoder::Mode {
+  fn from(value: Mode) -> Self {
+    match value {
+      Mode::Mono => mp3lame_encoder::Mode::Mono,
+      Mode::Stereo => mp3lame_encoder::Mode::Stereo,
+      Mode::JointStereo => mp3lame_encoder::Mode::JointStereo,
+      Mode::DualChannel => mp3lame_encoder::Mode::DaulChannel,
+      Mode::NotSet => mp3lame_encoder::Mode::NotSet,
+    }
+  }
+}
+
+#[napi(object, object_to_js = false)]
+pub struct EncodeOptions {
+  pub channels: u32,
+  pub quality: Option<Quality>,
+  pub bitrate: Option<Bitrate>,
+  pub sample_rate: Option<u32>,
+  pub mode: Option<Mode>,
+}
+
+#[napi]
+pub struct Mp3Encoder {
+  encoder: Encoder,
+}
+
+#[napi]
+impl Mp3Encoder {
+  #[napi(constructor)]
+  pub fn new(options: EncodeOptions) -> Result<Self> {
+    let mut builder = Builder::new().ok_or(LameError::CreateBuilderFailed)?;
+    builder
+      .set_num_channels(options.channels as u8)
+      .map_err(LameError::BuildError)?;
+    if let Some(quality) = options.quality {
+      builder
+        .set_quality(quality.into())
+        .map_err(LameError::BuildError)?;
+    }
+    if let Some(bitrate) = options.bitrate {
+      builder
+        .set_brate(bitrate.into())
+        .map_err(LameError::BuildError)?;
+    }
+    if let Some(sample_rate) = options.sample_rate {
+      builder
+        .set_sample_rate(sample_rate)
+        .map_err(LameError::BuildError)?;
+    }
+    if let Some(mode) = options.mode {
+      builder
+        .set_mode(mode.into())
+        .map_err(LameError::BuildError)?;
+    }
+    Ok(Self {
+      encoder: builder.build().map_err(LameError::BuildError)?,
+    })
+  }
+
+  #[napi]
+  pub fn encode(&mut self, input: &[f32]) -> Result<Uint8Array> {
+    let mut output = Vec::with_capacity(input.len());
+    output.reserve(mp3lame_encoder::max_required_buffer_size(input.len()));
+    let encoded_size = self
+      .encoder
+      .encode(MonoPcm(input), output.spare_capacity_mut())
+      .map_err(LameError::EncodeError)?;
+    unsafe {
+      output.set_len(output.len().wrapping_add(encoded_size));
+    }
+    let encoded_size = self
+      .encoder
+      .flush::<FlushNoGap>(output.spare_capacity_mut())
+      .map_err(LameError::EncodeError)?;
+    unsafe {
+      output.set_len(output.len().wrapping_add(encoded_size));
+    }
+    Ok(output.into())
+  }
+}

packages/frontend/native/package.json

@@ -25,7 +25,7 @@
     ]
   },
   "devDependencies": {
-    "@napi-rs/cli": "3.0.0-alpha.70",
+    "@napi-rs/cli": "3.0.0-alpha.72",
     "@napi-rs/whisper": "^0.0.4",
     "@types/node": "^22.0.0",
     "ava": "^6.2.0",
@@ -38,8 +38,8 @@
   },
   "scripts": {
     "artifacts": "napi artifacts",
-    "build": "napi build -p affine_native --platform --release --no-const-enum",
-    "build:debug": "napi build -p affine_native --platform",
+    "build": "napi build -p affine_native --platform --release --no-dts-cache",
+    "build:debug": "napi build -p affine_native --platform --no-dts-cache",
     "universal": "napi universal",
     "test": "ava",
     "version": "napi version"