feat(native): decode audio and mp3 encoder (#10490)

packages/frontend/native/media_capture/src/audio_decoder.rs

@@ -0,0 +1,179 @@
+use std::{io::Cursor, path::Path};
+
+use napi::{
+  bindgen_prelude::{AbortSignal, AsyncTask, Float32Array, Result, Status, Uint8Array},
+  Task,
+};
+use napi_derive::napi;
+use rubato::{Resampler, SincFixedIn, SincInterpolationParameters, SincInterpolationType};
+use symphonia::core::{
+  audio::{AudioBuffer, Signal},
+  codecs::DecoderOptions,
+  errors::Error,
+  formats::FormatOptions,
+  io::MediaSourceStream,
+  meta::MetadataOptions,
+  probe::Hint,
+};
+
+fn decode<B: AsRef<[u8]> + Send + Sync + 'static>(
+  buf: B,
+  dest_sample_rate: Option<u32>,
+  filename: Option<&str>,
+) -> std::result::Result<Vec<f32>, Error> {
+  // Create the media source
+  let mss = MediaSourceStream::new(Box::new(Cursor::new(buf)), Default::default());
+
+  // Create a probe hint using the file extension
+  let mut hint = Hint::new();
+  if let Some(ext) =
+    filename.and_then(|filename| Path::new(filename).extension().and_then(|ext| ext.to_str()))
+  {
+    hint.with_extension(ext);
+  }
+
+  let format_opts = FormatOptions {
+    enable_gapless: true,
+    ..Default::default()
+  };
+  let metadata_opts = MetadataOptions::default();
+  let decoder_opts = DecoderOptions::default();
+  let probed = symphonia::default::get_probe().format(&hint, mss, &format_opts, &metadata_opts)?;
+
+  let mut format = probed.format;
+
+  let track = format
+    .default_track()
+    .ok_or(Error::Unsupported("No default track found"))?;
+
+  let totol_samples = track
+    .codec_params
+    .n_frames
+    .ok_or(Error::Unsupported("No duration found"))?;
+  let sample_rate = track
+    .codec_params
+    .sample_rate
+    .ok_or(Error::Unsupported("No samplerate found"))?;
+
+  let mut decoder = symphonia::default::get_codecs().make(&track.codec_params, &decoder_opts)?;
+
+  let mut output: Vec<f32> = Vec::with_capacity(totol_samples as usize);
+  // Decode loop
+  while let Ok(packet) = format.next_packet() {
+    let decoded = decoder.decode(&packet)?;
+    let spec = decoded.spec();
+    let mut audio_buf: AudioBuffer<f32> = AudioBuffer::new(decoded.capacity() as u64, *spec);
+    decoded.convert(&mut audio_buf);
+
+    if spec.channels.count() > 1 {
+      // Mix all channels into mono
+      for i in 0..audio_buf.chan(0).len() {
+        let mut sample_sum = 0.0;
+        for ch in 0..spec.channels.count() {
+          sample_sum += audio_buf.chan(ch)[i];
+        }
+        output.push(sample_sum / spec.channels.count() as f32);
+      }
+    } else {
+      output.extend_from_slice(audio_buf.chan(0));
+    }
+  }
+
+  let Some(dest_sample_rate) = dest_sample_rate else {
+    return Ok(output);
+  };
+
+  if sample_rate != dest_sample_rate {
+    // Calculate parameters for resampling
+    let params = SincInterpolationParameters {
+      sinc_len: 256,
+      f_cutoff: 0.95,
+      interpolation: SincInterpolationType::Linear,
+      oversampling_factor: 256,
+      window: rubato::WindowFunction::BlackmanHarris2,
+    };
+
+    let mut resampler = SincFixedIn::<f32>::new(
+      dest_sample_rate as f64 / sample_rate as f64,
+      2.0,
+      params,
+      output.len(),
+      1,
+    )
+    .map_err(|_| Error::Unsupported("Failed to create resampler"))?;
+
+    let waves_in = vec![output];
+    let mut waves_out = resampler
+      .process(&waves_in, None)
+      .map_err(|_| Error::Unsupported("Failed to run resampler"))?;
+    output = waves_out
+      .pop()
+      .ok_or(Error::Unsupported("No resampled output found"))?;
+  }
+
+  Ok(output)
+}
+
+#[napi]
+/// Decode audio file into a Float32Array
+pub fn decode_audio_sync(
+  buf: Uint8Array,
+  dest_sample_rate: Option<u32>,
+  filename: Option<String>,
+) -> Result<Float32Array> {
+  decode(buf, dest_sample_rate, filename.as_deref())
+    .map(Float32Array::new)
+    .map_err(|e| {
+      napi::Error::new(
+        Status::InvalidArg,
+        format!("Decode audio into Float32Array failed: {e}"),
+      )
+    })
+}
+
+pub struct DecodeAudioTask {
+  buf: Uint8Array,
+  dest_sample_rate: Option<u32>,
+  filename: Option<String>,
+}
+
+#[napi]
+impl Task for DecodeAudioTask {
+  type Output = Vec<f32>;
+  type JsValue = Float32Array;
+
+  fn compute(&mut self) -> Result<Self::Output> {
+    decode(
+      std::mem::replace(&mut self.buf, Uint8Array::new(vec![])),
+      self.dest_sample_rate,
+      self.filename.as_deref(),
+    )
+    .map_err(|e| {
+      napi::Error::new(
+        Status::InvalidArg,
+        format!("Decode audio into Float32Array failed: {e}"),
+      )
+    })
+  }
+
+  fn resolve(&mut self, _: napi::Env, output: Self::Output) -> Result<Self::JsValue> {
+    Ok(Float32Array::new(output))
+  }
+}
+
+#[napi]
+pub fn decode_audio(
+  buf: Uint8Array,
+  dest_sample_rate: Option<u32>,
+  filename: Option<String>,
+  signal: Option<AbortSignal>,
+) -> AsyncTask<DecodeAudioTask> {
+  AsyncTask::with_optional_signal(
+    DecodeAudioTask {
+      buf,
+      dest_sample_rate,
+      filename,
+    },
+    signal,
+  )
+}

packages/frontend/native/media_capture/src/lib.rs

@@ -2,3 +2,5 @@
 pub mod macos;
 #[cfg(target_os = "macos")]
 pub(crate) use macos::*;
+pub mod audio_decoder;
+pub mod mp3;

packages/frontend/native/media_capture/src/mp3.rs

@@ -0,0 +1,215 @@
+use mp3lame_encoder::{Builder, Encoder, FlushNoGap, MonoPcm};
+use napi::bindgen_prelude::{Result, Uint8Array};
+use napi_derive::napi;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum LameError {
+  #[error("Create builder failed")]
+  CreateBuilderFailed,
+  #[error("Failed to create encoder")]
+  BuildError(#[from] mp3lame_encoder::BuildError),
+  #[error("Failed to encode")]
+  EncodeError(#[from] mp3lame_encoder::EncodeError),
+}
+
+impl From<LameError> for napi::Error {
+  fn from(value: LameError) -> Self {
+    napi::Error::new(napi::Status::GenericFailure, value.to_string())
+  }
+}
+
+#[napi]
+///Possible quality parameter.
+///From best(0) to worst(9)
+pub enum Quality {
+  ///Best possible quality
+  Best = 0,
+  ///Second best
+  SecondBest = 1,
+  ///Close to best
+  NearBest = 2,
+  ///Very nice
+  VeryNice = 3,
+  ///Nice
+  Nice = 4,
+  ///Good
+  Good = 5,
+  ///Decent
+  Decent = 6,
+  ///Okayish
+  Ok = 7,
+  ///Almost worst
+  SecondWorst = 8,
+  ///Worst
+  Worst = 9,
+}
+
+impl From<Quality> for mp3lame_encoder::Quality {
+  fn from(value: Quality) -> Self {
+    match value {
+      Quality::Best => mp3lame_encoder::Quality::Best,
+      Quality::SecondBest => mp3lame_encoder::Quality::SecondBest,
+      Quality::NearBest => mp3lame_encoder::Quality::NearBest,
+      Quality::VeryNice => mp3lame_encoder::Quality::VeryNice,
+      Quality::Nice => mp3lame_encoder::Quality::Nice,
+      Quality::Good => mp3lame_encoder::Quality::Good,
+      Quality::Decent => mp3lame_encoder::Quality::Decent,
+      Quality::Ok => mp3lame_encoder::Quality::Ok,
+      Quality::SecondWorst => mp3lame_encoder::Quality::SecondWorst,
+      Quality::Worst => mp3lame_encoder::Quality::Worst,
+    }
+  }
+}
+
+#[napi]
+#[repr(u16)]
+///Enumeration of valid values for `set_brate`
+pub enum Bitrate {
+  ///8_000
+  Kbps8 = 8,
+  ///16_000
+  Kbps16 = 16,
+  ///24_000
+  Kbps24 = 24,
+  ///32_000
+  Kbps32 = 32,
+  ///40_000
+  Kbps40 = 40,
+  ///48_000
+  Kbps48 = 48,
+  ///64_000
+  Kbps64 = 64,
+  ///80_000
+  Kbps80 = 80,
+  ///96_000
+  Kbps96 = 96,
+  ///112_000
+  Kbps112 = 112,
+  ///128_000
+  Kbps128 = 128,
+  ///160_000
+  Kbps160 = 160,
+  ///192_000
+  Kbps192 = 192,
+  ///224_000
+  Kbps224 = 224,
+  ///256_000
+  Kbps256 = 256,
+  ///320_000
+  Kbps320 = 320,
+}
+
+impl From<Bitrate> for mp3lame_encoder::Bitrate {
+  fn from(value: Bitrate) -> Self {
+    match value {
+      Bitrate::Kbps8 => mp3lame_encoder::Bitrate::Kbps8,
+      Bitrate::Kbps16 => mp3lame_encoder::Bitrate::Kbps16,
+      Bitrate::Kbps24 => mp3lame_encoder::Bitrate::Kbps24,
+      Bitrate::Kbps32 => mp3lame_encoder::Bitrate::Kbps32,
+      Bitrate::Kbps40 => mp3lame_encoder::Bitrate::Kbps40,
+      Bitrate::Kbps48 => mp3lame_encoder::Bitrate::Kbps48,
+      Bitrate::Kbps64 => mp3lame_encoder::Bitrate::Kbps64,
+      Bitrate::Kbps80 => mp3lame_encoder::Bitrate::Kbps80,
+      Bitrate::Kbps96 => mp3lame_encoder::Bitrate::Kbps96,
+      Bitrate::Kbps112 => mp3lame_encoder::Bitrate::Kbps112,
+      Bitrate::Kbps128 => mp3lame_encoder::Bitrate::Kbps128,
+      Bitrate::Kbps160 => mp3lame_encoder::Bitrate::Kbps160,
+      Bitrate::Kbps192 => mp3lame_encoder::Bitrate::Kbps192,
+      Bitrate::Kbps224 => mp3lame_encoder::Bitrate::Kbps224,
+      Bitrate::Kbps256 => mp3lame_encoder::Bitrate::Kbps256,
+      Bitrate::Kbps320 => mp3lame_encoder::Bitrate::Kbps320,
+    }
+  }
+}
+
+#[napi]
+/// MPEG mode
+pub enum Mode {
+  Mono,
+  Stereo,
+  JointStereo,
+  DualChannel,
+  NotSet,
+}
+
+impl From<Mode> for mp3lame_encoder::Mode {
+  fn from(value: Mode) -> Self {
+    match value {
+      Mode::Mono => mp3lame_encoder::Mode::Mono,
+      Mode::Stereo => mp3lame_encoder::Mode::Stereo,
+      Mode::JointStereo => mp3lame_encoder::Mode::JointStereo,
+      Mode::DualChannel => mp3lame_encoder::Mode::DaulChannel,
+      Mode::NotSet => mp3lame_encoder::Mode::NotSet,
+    }
+  }
+}
+
+#[napi(object, object_to_js = false)]
+pub struct EncodeOptions {
+  pub channels: u32,
+  pub quality: Option<Quality>,
+  pub bitrate: Option<Bitrate>,
+  pub sample_rate: Option<u32>,
+  pub mode: Option<Mode>,
+}
+
+#[napi]
+pub struct Mp3Encoder {
+  encoder: Encoder,
+}
+
+#[napi]
+impl Mp3Encoder {
+  #[napi(constructor)]
+  pub fn new(options: EncodeOptions) -> Result<Self> {
+    let mut builder = Builder::new().ok_or(LameError::CreateBuilderFailed)?;
+    builder
+      .set_num_channels(options.channels as u8)
+      .map_err(LameError::BuildError)?;
+    if let Some(quality) = options.quality {
+      builder
+        .set_quality(quality.into())
+        .map_err(LameError::BuildError)?;
+    }
+    if let Some(bitrate) = options.bitrate {
+      builder
+        .set_brate(bitrate.into())
+        .map_err(LameError::BuildError)?;
+    }
+    if let Some(sample_rate) = options.sample_rate {
+      builder
+        .set_sample_rate(sample_rate)
+        .map_err(LameError::BuildError)?;
+    }
+    if let Some(mode) = options.mode {
+      builder
+        .set_mode(mode.into())
+        .map_err(LameError::BuildError)?;
+    }
+    Ok(Self {
+      encoder: builder.build().map_err(LameError::BuildError)?,
+    })
+  }
+
+  #[napi]
+  pub fn encode(&mut self, input: &[f32]) -> Result<Uint8Array> {
+    let mut output = Vec::with_capacity(input.len());
+    output.reserve(mp3lame_encoder::max_required_buffer_size(input.len()));
+    let encoded_size = self
+      .encoder
+      .encode(MonoPcm(input), output.spare_capacity_mut())
+      .map_err(LameError::EncodeError)?;
+    unsafe {
+      output.set_len(output.len().wrapping_add(encoded_size));
+    }
+    let encoded_size = self
+      .encoder
+      .flush::<FlushNoGap>(output.spare_capacity_mut())
+      .map_err(LameError::EncodeError)?;
+    unsafe {
+      output.set_len(output.len().wrapping_add(encoded_size));
+    }
+    Ok(output.into())
+  }
+}