feat(native): decode audio and mp3 encoder (#10490)

.github/actions/build-rust/action.yml

@@ -44,7 +44,7 @@ runs:
         RUSTUP_HOME: ${{ env.DEV_DRIVE }}/.rustup
 
     - name: Set CC
-      if: ${{ contains(inputs.target, 'linux') && inputs.package != '@affine/native' && inputs.no-build != 'true' }}
+      if: ${{ contains(inputs.target, 'linux') && inputs.no-build != 'true' }}
       working-directory: ${{ env.DEV_DRIVE_WORKSPACE || github.workspace }}
       shell: bash
       run: |

.github/workflows/build-test.yml

@@ -387,6 +387,28 @@ jobs:
           path: dist.tar.gz
           if-no-files-found: error
 
+  native-unit-test:
+    name: Native Unit Test
+    runs-on: ubuntu-latest
+    needs:
+      - optimize_ci
+      - build-native
+    if: needs.optimize_ci.outputs.skip == 'false'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          extra-flags: workspaces focus @affine-tools/cli @affine/monorepo @affine/native
+          electron-install: false
+      - name: Download affine.linux-x64-gnu.node
+        uses: actions/download-artifact@v4
+        with:
+          name: affine.linux-x64-gnu.node
+          path: ./packages/frontend/native
+      - name: Unit Test
+        run: yarn affine @affine/native test
+
   server-test:
     name: Server Test
     runs-on: ubuntu-latest
@@ -897,6 +919,7 @@ jobs:
       - build-native
       - build-server-native
       - build-electron-renderer
+      - native-unit-test
       - server-test
       - rust-test
       - copilot-api-test

Cargo.lock

@@ -70,6 +70,7 @@ dependencies = [
  "coreaudio-rs",
  "dispatch2",
  "libc",
+ "mp3lame-encoder",
  "napi",
  "napi-build",
  "napi-derive",
@@ -77,6 +78,7 @@ dependencies = [
  "objc2-foundation",
  "rubato",
  "screencapturekit",
+ "symphonia",
  "thiserror 2.0.11",
  "uuid",
 ]
@@ -287,6 +289,12 @@ dependencies = [
  "derive_arbitrary",
 ]
 
+[[package]]
+name = "arrayvec"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+
 [[package]]
 name = "async-compat"
 version = "0.2.4"
@@ -339,6 +347,15 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
 
+[[package]]
+name = "autotools"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.74"
@@ -501,6 +518,12 @@ version = "3.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
 
+[[package]]
+name = "bytemuck"
+version = "1.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3"
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -953,9 +976,18 @@ dependencies = [
 
 [[package]]
 name = "ctor"
-version = "0.3.0"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f06b1425736ba96096116f063c9d10be2352a7cde0cbea829a717008e114aec9"
+checksum = "21d960ecacd0a1bf55e73144b72de745e7bf275c7952c50e36e8af0a0cb7ab1f"
+dependencies = [
+ "ctor-proc-macro",
+]
+
+[[package]]
+name = "ctor-proc-macro"
+version = "0.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c426d2ba3e525b39c1f0a9ba41b9fe61878dee11fa4e4a76b6ab440f46c5db5d"
 
 [[package]]
 name = "dashmap"
@@ -1174,6 +1206,12 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "extended"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
+
 [[package]]
 name = "fancy-regex"
 version = "0.13.0"
@@ -2094,6 +2132,27 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "mp3lame-encoder"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc8c8b5cdbe788ccd1098c3d3635298a011cffdebdd3460c9ca5060a7551557b"
+dependencies = [
+ "libc",
+ "mp3lame-sys",
+]
+
+[[package]]
+name = "mp3lame-sys"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acaec8842b2ebd61692a6c8c2b9f3edbf5c36e5e5c4677b5911430eaf859377c"
+dependencies = [
+ "autotools",
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "nanoid"
 version = "0.4.0"
@@ -2105,9 +2164,9 @@ dependencies = [
 
 [[package]]
 name = "napi"
-version = "3.0.0-alpha.28"
+version = "3.0.0-alpha.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dd957e2cc4bd62b730b10ff1f35775f8a81dac84a3bfac273b0ec4336f53ab8"
+checksum = "b1911b4f0d33fbcb5f46ff68319ec053ab8a655f3a17440eae1246a23ba2ad78"
 dependencies = [
  "anyhow",
  "bitflags 2.8.0",
@@ -2121,15 +2180,15 @@ dependencies = [
 
 [[package]]
 name = "napi-build"
-version = "2.1.4"
+version = "2.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db836caddef23662b94e16bf1f26c40eceb09d6aee5d5b06a7ac199320b69b19"
+checksum = "40685973218af4aa4b42486652692c294c44b5a67e4b2202df721c9063f2e51c"
 
 [[package]]
 name = "napi-derive"
-version = "3.0.0-alpha.26"
+version = "3.0.0-alpha.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0f0b6f3f77925d8fd2030855af659ce428a7bb6e10e94852e226f509186ba7c"
+checksum = "c8097918a9af1976700eac6944b120b65ad17bf6d38906703d2b68e17ee89256"
 dependencies = [
  "convert_case 0.7.1",
  "napi-derive-backend",
@@ -2140,9 +2199,9 @@ dependencies = [
 
 [[package]]
 name = "napi-derive-backend"
-version = "2.0.0-alpha.26"
+version = "2.0.0-alpha.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c694bb49a2fa84dd9542d51eece39a57519f9cf1fc2deefa9d119ab8181e374d"
+checksum = "8e5adc92fcdec3aa09f591bd2b139d7c669399f34b8211fe653641b52d40d3b3"
 dependencies = [
  "convert_case 0.7.1",
  "proc-macro2",
@@ -3558,6 +3617,202 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
+[[package]]
+name = "symphonia"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "815c942ae7ee74737bb00f965fa5b5a2ac2ce7b6c01c0cc169bbeaf7abd5f5a9"
+dependencies = [
+ "lazy_static",
+ "symphonia-bundle-flac",
+ "symphonia-bundle-mp3",
+ "symphonia-codec-aac",
+ "symphonia-codec-adpcm",
+ "symphonia-codec-alac",
+ "symphonia-codec-pcm",
+ "symphonia-codec-vorbis",
+ "symphonia-core",
+ "symphonia-format-caf",
+ "symphonia-format-isomp4",
+ "symphonia-format-mkv",
+ "symphonia-format-ogg",
+ "symphonia-format-riff",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-bundle-flac"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72e34f34298a7308d4397a6c7fbf5b84c5d491231ce3dd379707ba673ab3bd97"
+dependencies = [
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+ "symphonia-utils-xiph",
+]
+
+[[package]]
+name = "symphonia-bundle-mp3"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c01c2aae70f0f1fb096b6f0ff112a930b1fb3626178fba3ae68b09dce71706d4"
+dependencies = [
+ "lazy_static",
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-codec-aac"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdbf25b545ad0d3ee3e891ea643ad115aff4ca92f6aec472086b957a58522f70"
+dependencies = [
+ "lazy_static",
+ "log",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-codec-adpcm"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c94e1feac3327cd616e973d5be69ad36b3945f16b06f19c6773fc3ac0b426a0f"
+dependencies = [
+ "log",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-codec-alac"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d8a6666649a08412906476a8b0efd9b9733e241180189e9f92b09c08d0e38f3"
+dependencies = [
+ "log",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-codec-pcm"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f395a67057c2ebc5e84d7bb1be71cce1a7ba99f64e0f0f0e303a03f79116f89b"
+dependencies = [
+ "log",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-codec-vorbis"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a98765fb46a0a6732b007f7e2870c2129b6f78d87db7987e6533c8f164a9f30"
+dependencies = [
+ "log",
+ "symphonia-core",
+ "symphonia-utils-xiph",
+]
+
+[[package]]
+name = "symphonia-core"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "798306779e3dc7d5231bd5691f5a813496dc79d3f56bf82e25789f2094e022c3"
+dependencies = [
+ "arrayvec",
+ "bitflags 1.3.2",
+ "bytemuck",
+ "lazy_static",
+ "log",
+ "rustfft",
+]
+
+[[package]]
+name = "symphonia-format-caf"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e43c99c696a388295a29fe71b133079f5d8b18041cf734c5459c35ad9097af50"
+dependencies = [
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-format-isomp4"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abfdf178d697e50ce1e5d9b982ba1b94c47218e03ec35022d9f0e071a16dc844"
+dependencies = [
+ "encoding_rs",
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+ "symphonia-utils-xiph",
+]
+
+[[package]]
+name = "symphonia-format-mkv"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bb43471a100f7882dc9937395bd5ebee8329298e766250b15b3875652fe3d6f"
+dependencies = [
+ "lazy_static",
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+ "symphonia-utils-xiph",
+]
+
+[[package]]
+name = "symphonia-format-ogg"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ada3505789516bcf00fc1157c67729eded428b455c27ca370e41f4d785bfa931"
+dependencies = [
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+ "symphonia-utils-xiph",
+]
+
+[[package]]
+name = "symphonia-format-riff"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05f7be232f962f937f4b7115cbe62c330929345434c834359425e043bfd15f50"
+dependencies = [
+ "extended",
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-metadata"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc622b9841a10089c5b18e99eb904f4341615d5aa55bbf4eedde1be721a4023c"
+dependencies = [
+ "encoding_rs",
+ "lazy_static",
+ "log",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-utils-xiph"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "484472580fa49991afda5f6550ece662237b00c6f562c7d9638d1b086ed010fe"
+dependencies = [
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.109"

Cargo.toml

@@ -26,9 +26,10 @@ file-format      = { version = "0.26", features = ["reader"] }
 homedir          = "0.3"
 libc             = "0.2"
 mimalloc         = "0.1"
-napi             = { version = "3.0.0-alpha.12", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] }
+mp3lame-encoder  = "0.2"
+napi             = { version = "3.0.0-alpha.31", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] }
 napi-build       = { version = "2" }
-napi-derive      = { version = "3.0.0-alpha.12" }
+napi-derive      = { version = "3.0.0-alpha.28" }
 notify           = { version = "8", features = ["serde"] }
 objc2            = "0.6"
 objc2-foundation = "0.3"
@@ -42,6 +43,7 @@ serde            = "1"
 serde_json       = "1"
 sha3             = "0.10"
 sqlx             = { version = "0.8", default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] }
+symphonia        = { version = "0.5", features = ["all", "opt-simd"] }
 thiserror        = "2"
 tiktoken-rs      = "0.6"
 tokio            = "1.37"

README.md

@@ -160,6 +160,7 @@ We would also like to give thanks to open-source projects that make AFFiNE possi
 - [Jotai](https://github.com/pmndrs/jotai) - Primitive and flexible state management for React.
 - [async-call-rpc](https://github.com/Jack-Works/async-call-rpc) - A lightweight JSON RPC client & server.
 - [Vite](https://github.com/vitejs/vite) - Next generation frontend tooling.
+- [lame](https://lame.sourceforge.io/) - High quality MPEG Audio Layer III (MP3) encoder.
 - Other upstream [dependencies](https://github.com/toeverything/AFFiNE/network/dependencies).
 
 Thanks a lot to the community for providing such powerful and simple libraries, so that we can focus more on the implementation of the product logic, and we hope that in the future our projects will also provide a more easy-to-use knowledge base for everyone.

packages/backend/native/package.json

@@ -33,7 +33,7 @@
     "build:debug": "napi build"
   },
   "devDependencies": {
-    "@napi-rs/cli": "3.0.0-alpha.70",
+    "@napi-rs/cli": "3.0.0-alpha.72",
     "lib0": "^0.2.99",
     "tiktoken": "^1.0.17",
     "tinybench": "^3.0.7",

packages/frontend/native/__tests__/audio.spec.mts

@@ -0,0 +1,23 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { tmpdir } from 'node:os';
+
+import test from 'ava';
+
+import { decodeAudio, Mp3Encoder } from '../index.js';
+
+const __dirname = join(fileURLToPath(import.meta.url), '..');
+
+const wav = await readFile(join(__dirname, 'fixtures', 'recording.wav'));
+
+test('convert wav to mp3', async t => {
+  const audio = await decodeAudio(wav);
+  const mp3 = new Mp3Encoder({
+    channels: 1,
+  });
+  await t.notThrowsAsync(async () => {
+    const mp3Data = mp3.encode(audio);
+    await writeFile(join(tmpdir(), 'recording.mp3'), mp3Data);
+  });
+});

packages/frontend/native/index.d.ts

@@ -61,6 +61,11 @@ export declare class DocStoragePool {
   clearClocks(universalId: string): Promise<void>
 }
 
+export declare class Mp3Encoder {
+  constructor(options: EncodeOptions)
+  encode(input: Float32Array): Uint8Array
+}
+
 export declare class RecordingPermissions {
   audio: boolean
   screen: boolean
@@ -113,6 +118,42 @@ export declare class SqliteConnection {
   checkpoint(): Promise<void>
 }
 
+/**Enumeration of valid values for `set_brate` */
+export declare enum Bitrate {
+  /**8_000 */
+  Kbps8 = 8,
+  /**16_000 */
+  Kbps16 = 16,
+  /**24_000 */
+  Kbps24 = 24,
+  /**32_000 */
+  Kbps32 = 32,
+  /**40_000 */
+  Kbps40 = 40,
+  /**48_000 */
+  Kbps48 = 48,
+  /**64_000 */
+  Kbps64 = 64,
+  /**80_000 */
+  Kbps80 = 80,
+  /**96_000 */
+  Kbps96 = 96,
+  /**112_000 */
+  Kbps112 = 112,
+  /**128_000 */
+  Kbps128 = 128,
+  /**160_000 */
+  Kbps160 = 160,
+  /**192_000 */
+  Kbps192 = 192,
+  /**224_000 */
+  Kbps224 = 224,
+  /**256_000 */
+  Kbps256 = 256,
+  /**320_000 */
+  Kbps320 = 320
+}
+
 export interface Blob {
   key: string
   data: Uint8Array
@@ -127,6 +168,11 @@ export interface BlobRow {
   timestamp: Date
 }
 
+export declare function decodeAudio(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null, signal?: AbortSignal | undefined | null): Promise<Float32Array>
+
+/** Decode audio file into a Float32Array */
+export declare function decodeAudioSync(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null): Float32Array
+
 export interface DocClock {
   docId: string
   timestamp: Date
@@ -149,6 +195,14 @@ export interface DocUpdate {
   bin: Uint8Array
 }
 
+export interface EncodeOptions {
+  channels: number
+  quality?: Quality
+  bitrate?: Bitrate
+  sampleRate?: number
+  mode?: Mode
+}
+
 export interface InsertRow {
   docId?: string
   data: Uint8Array
@@ -163,6 +217,42 @@ export interface ListedBlob {
 
 export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise<string>
 
+/** MPEG mode */
+export declare enum Mode {
+  Mono = 0,
+  Stereo = 1,
+  JointStereo = 2,
+  DualChannel = 3,
+  NotSet = 4
+}
+
+/**
+ *Possible quality parameter.
+ *From best(0) to worst(9)
+ */
+export declare enum Quality {
+  /**Best possible quality */
+  Best = 0,
+  /**Second best */
+  SecondBest = 1,
+  /**Close to best */
+  NearBest = 2,
+  /**Very nice */
+  VeryNice = 3,
+  /**Nice */
+  Nice = 4,
+  /**Good */
+  Good = 5,
+  /**Decent */
+  Decent = 6,
+  /**Okayish */
+  Ok = 7,
+  /**Almost worst */
+  SecondWorst = 8,
+  /**Worst */
+  Worst = 9
+}
+
 export interface SetBlob {
   key: string
   data: Uint8Array

packages/frontend/native/index.js

@@ -60,7 +60,13 @@ const isMuslFromChildProcess = () => {
 }
 
 function requireNative() {
-  if (process.platform === 'android') {
+  if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) {
+    try {
+      nativeBinding = require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH);
+    } catch (err) {
+      loadErrors.push(err);
+    }
+  } else if (process.platform === 'android') {
     if (process.arch === 'arm64') {
       try {
         return require('./affine.android-arm64.node')
@@ -370,9 +376,15 @@ module.exports.ApplicationStateChangedSubscriber = nativeBinding.ApplicationStat
 module.exports.AudioTapStream = nativeBinding.AudioTapStream
 module.exports.DocStorage = nativeBinding.DocStorage
 module.exports.DocStoragePool = nativeBinding.DocStoragePool
+module.exports.Mp3Encoder = nativeBinding.Mp3Encoder
 module.exports.RecordingPermissions = nativeBinding.RecordingPermissions
 module.exports.ShareableContent = nativeBinding.ShareableContent
 module.exports.SqliteConnection = nativeBinding.SqliteConnection
+module.exports.Bitrate = nativeBinding.Bitrate
+module.exports.decodeAudio = nativeBinding.decodeAudio
+module.exports.decodeAudioSync = nativeBinding.decodeAudioSync
 module.exports.mintChallengeResponse = nativeBinding.mintChallengeResponse
+module.exports.Mode = nativeBinding.Mode
+module.exports.Quality = nativeBinding.Quality
 module.exports.ValidationResult = nativeBinding.ValidationResult
 module.exports.verifyChallengeResponse = nativeBinding.verifyChallengeResponse

packages/frontend/native/media_capture/Cargo.toml

@@ -7,9 +7,12 @@ version = "0.0.0"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
+mp3lame-encoder = { workspace = true, features = ["std"] }
 napi            = { workspace = true, features = ["napi4"] }
 napi-derive     = { workspace = true, features = ["type-def"] }
 rubato          = { workspace = true }
+symphonia       = { workspace = true, features = ["all", "opt-simd"] }
+thiserror       = { workspace = true }
 
 [target.'cfg(target_os = "macos")'.dependencies]
 block2           = { workspace = true }
@@ -20,7 +23,6 @@ libc             = { workspace = true }
 objc2            = { workspace = true }
 objc2-foundation = { workspace = true }
 screencapturekit = { workspace = true }
-thiserror        = { workspace = true }
 uuid             = { workspace = true, features = ["v4"] }
 
 [build-dependencies]

packages/frontend/native/media_capture/src/audio_decoder.rs

@@ -0,0 +1,179 @@
+use std::{io::Cursor, path::Path};
+
+use napi::{
+  bindgen_prelude::{AbortSignal, AsyncTask, Float32Array, Result, Status, Uint8Array},
+  Task,
+};
+use napi_derive::napi;
+use rubato::{Resampler, SincFixedIn, SincInterpolationParameters, SincInterpolationType};
+use symphonia::core::{
+  audio::{AudioBuffer, Signal},
+  codecs::DecoderOptions,
+  errors::Error,
+  formats::FormatOptions,
+  io::MediaSourceStream,
+  meta::MetadataOptions,
+  probe::Hint,
+};
+
+fn decode<B: AsRef<[u8]> + Send + Sync + 'static>(
+  buf: B,
+  dest_sample_rate: Option<u32>,
+  filename: Option<&str>,
+) -> std::result::Result<Vec<f32>, Error> {
+  // Create the media source
+  let mss = MediaSourceStream::new(Box::new(Cursor::new(buf)), Default::default());
+
+  // Create a probe hint using the file extension
+  let mut hint = Hint::new();
+  if let Some(ext) =
+    filename.and_then(|filename| Path::new(filename).extension().and_then(|ext| ext.to_str()))
+  {
+    hint.with_extension(ext);
+  }
+
+  let format_opts = FormatOptions {
+    enable_gapless: true,
+    ..Default::default()
+  };
+  let metadata_opts = MetadataOptions::default();
+  let decoder_opts = DecoderOptions::default();
+  let probed = symphonia::default::get_probe().format(&hint, mss, &format_opts, &metadata_opts)?;
+
+  let mut format = probed.format;
+
+  let track = format
+    .default_track()
+    .ok_or(Error::Unsupported("No default track found"))?;
+
+  let totol_samples = track
+    .codec_params
+    .n_frames
+    .ok_or(Error::Unsupported("No duration found"))?;
+  let sample_rate = track
+    .codec_params
+    .sample_rate
+    .ok_or(Error::Unsupported("No samplerate found"))?;
+
+  let mut decoder = symphonia::default::get_codecs().make(&track.codec_params, &decoder_opts)?;
+
+  let mut output: Vec<f32> = Vec::with_capacity(totol_samples as usize);
+  // Decode loop
+  while let Ok(packet) = format.next_packet() {
+    let decoded = decoder.decode(&packet)?;
+    let spec = decoded.spec();
+    let mut audio_buf: AudioBuffer<f32> = AudioBuffer::new(decoded.capacity() as u64, *spec);
+    decoded.convert(&mut audio_buf);
+
+    if spec.channels.count() > 1 {
+      // Mix all channels into mono
+      for i in 0..audio_buf.chan(0).len() {
+        let mut sample_sum = 0.0;
+        for ch in 0..spec.channels.count() {
+          sample_sum += audio_buf.chan(ch)[i];
+        }
+        output.push(sample_sum / spec.channels.count() as f32);
+      }
+    } else {
+      output.extend_from_slice(audio_buf.chan(0));
+    }
+  }
+
+  let Some(dest_sample_rate) = dest_sample_rate else {
+    return Ok(output);
+  };
+
+  if sample_rate != dest_sample_rate {
+    // Calculate parameters for resampling
+    let params = SincInterpolationParameters {
+      sinc_len: 256,
+      f_cutoff: 0.95,
+      interpolation: SincInterpolationType::Linear,
+      oversampling_factor: 256,
+      window: rubato::WindowFunction::BlackmanHarris2,
+    };
+
+    let mut resampler = SincFixedIn::<f32>::new(
+      dest_sample_rate as f64 / sample_rate as f64,
+      2.0,
+      params,
+      output.len(),
+      1,
+    )
+    .map_err(|_| Error::Unsupported("Failed to create resampler"))?;
+
+    let waves_in = vec![output];
+    let mut waves_out = resampler
+      .process(&waves_in, None)
+      .map_err(|_| Error::Unsupported("Failed to run resampler"))?;
+    output = waves_out
+      .pop()
+      .ok_or(Error::Unsupported("No resampled output found"))?;
+  }
+
+  Ok(output)
+}
+
+#[napi]
+/// Decode audio file into a Float32Array
+pub fn decode_audio_sync(
+  buf: Uint8Array,
+  dest_sample_rate: Option<u32>,
+  filename: Option<String>,
+) -> Result<Float32Array> {
+  decode(buf, dest_sample_rate, filename.as_deref())
+    .map(Float32Array::new)
+    .map_err(|e| {
+      napi::Error::new(
+        Status::InvalidArg,
+        format!("Decode audio into Float32Array failed: {e}"),
+      )
+    })
+}
+
+pub struct DecodeAudioTask {
+  buf: Uint8Array,
+  dest_sample_rate: Option<u32>,
+  filename: Option<String>,
+}
+
+#[napi]
+impl Task for DecodeAudioTask {
+  type Output = Vec<f32>;
+  type JsValue = Float32Array;
+
+  fn compute(&mut self) -> Result<Self::Output> {
+    decode(
+      std::mem::replace(&mut self.buf, Uint8Array::new(vec![])),
+      self.dest_sample_rate,
+      self.filename.as_deref(),
+    )
+    .map_err(|e| {
+      napi::Error::new(
+        Status::InvalidArg,
+        format!("Decode audio into Float32Array failed: {e}"),
+      )
+    })
+  }
+
+  fn resolve(&mut self, _: napi::Env, output: Self::Output) -> Result<Self::JsValue> {
+    Ok(Float32Array::new(output))
+  }
+}
+
+#[napi]
+pub fn decode_audio(
+  buf: Uint8Array,
+  dest_sample_rate: Option<u32>,
+  filename: Option<String>,
+  signal: Option<AbortSignal>,
+) -> AsyncTask<DecodeAudioTask> {
+  AsyncTask::with_optional_signal(
+    DecodeAudioTask {
+      buf,
+      dest_sample_rate,
+      filename,
+    },
+    signal,
+  )
+}

packages/frontend/native/media_capture/src/lib.rs

@@ -2,3 +2,5 @@
 pub mod macos;
 #[cfg(target_os = "macos")]
 pub(crate) use macos::*;
+pub mod audio_decoder;
+pub mod mp3;

packages/frontend/native/media_capture/src/mp3.rs

@@ -0,0 +1,215 @@
+use mp3lame_encoder::{Builder, Encoder, FlushNoGap, MonoPcm};
+use napi::bindgen_prelude::{Result, Uint8Array};
+use napi_derive::napi;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum LameError {
+  #[error("Create builder failed")]
+  CreateBuilderFailed,
+  #[error("Failed to create encoder")]
+  BuildError(#[from] mp3lame_encoder::BuildError),
+  #[error("Failed to encode")]
+  EncodeError(#[from] mp3lame_encoder::EncodeError),
+}
+
+impl From<LameError> for napi::Error {
+  fn from(value: LameError) -> Self {
+    napi::Error::new(napi::Status::GenericFailure, value.to_string())
+  }
+}
+
+#[napi]
+///Possible quality parameter.
+///From best(0) to worst(9)
+pub enum Quality {
+  ///Best possible quality
+  Best = 0,
+  ///Second best
+  SecondBest = 1,
+  ///Close to best
+  NearBest = 2,
+  ///Very nice
+  VeryNice = 3,
+  ///Nice
+  Nice = 4,
+  ///Good
+  Good = 5,
+  ///Decent
+  Decent = 6,
+  ///Okayish
+  Ok = 7,
+  ///Almost worst
+  SecondWorst = 8,
+  ///Worst
+  Worst = 9,
+}
+
+impl From<Quality> for mp3lame_encoder::Quality {
+  fn from(value: Quality) -> Self {
+    match value {
+      Quality::Best => mp3lame_encoder::Quality::Best,
+      Quality::SecondBest => mp3lame_encoder::Quality::SecondBest,
+      Quality::NearBest => mp3lame_encoder::Quality::NearBest,
+      Quality::VeryNice => mp3lame_encoder::Quality::VeryNice,
+      Quality::Nice => mp3lame_encoder::Quality::Nice,
+      Quality::Good => mp3lame_encoder::Quality::Good,
+      Quality::Decent => mp3lame_encoder::Quality::Decent,
+      Quality::Ok => mp3lame_encoder::Quality::Ok,
+      Quality::SecondWorst => mp3lame_encoder::Quality::SecondWorst,
+      Quality::Worst => mp3lame_encoder::Quality::Worst,
+    }
+  }
+}
+
+#[napi]
+#[repr(u16)]
+///Enumeration of valid values for `set_brate`
+pub enum Bitrate {
+  ///8_000
+  Kbps8 = 8,
+  ///16_000
+  Kbps16 = 16,
+  ///24_000
+  Kbps24 = 24,
+  ///32_000
+  Kbps32 = 32,
+  ///40_000
+  Kbps40 = 40,
+  ///48_000
+  Kbps48 = 48,
+  ///64_000
+  Kbps64 = 64,
+  ///80_000
+  Kbps80 = 80,
+  ///96_000
+  Kbps96 = 96,
+  ///112_000
+  Kbps112 = 112,
+  ///128_000
+  Kbps128 = 128,
+  ///160_000
+  Kbps160 = 160,
+  ///192_000
+  Kbps192 = 192,
+  ///224_000
+  Kbps224 = 224,
+  ///256_000
+  Kbps256 = 256,
+  ///320_000
+  Kbps320 = 320,
+}
+
+impl From<Bitrate> for mp3lame_encoder::Bitrate {
+  fn from(value: Bitrate) -> Self {
+    match value {
+      Bitrate::Kbps8 => mp3lame_encoder::Bitrate::Kbps8,
+      Bitrate::Kbps16 => mp3lame_encoder::Bitrate::Kbps16,
+      Bitrate::Kbps24 => mp3lame_encoder::Bitrate::Kbps24,
+      Bitrate::Kbps32 => mp3lame_encoder::Bitrate::Kbps32,
+      Bitrate::Kbps40 => mp3lame_encoder::Bitrate::Kbps40,
+      Bitrate::Kbps48 => mp3lame_encoder::Bitrate::Kbps48,
+      Bitrate::Kbps64 => mp3lame_encoder::Bitrate::Kbps64,
+      Bitrate::Kbps80 => mp3lame_encoder::Bitrate::Kbps80,
+      Bitrate::Kbps96 => mp3lame_encoder::Bitrate::Kbps96,
+      Bitrate::Kbps112 => mp3lame_encoder::Bitrate::Kbps112,
+      Bitrate::Kbps128 => mp3lame_encoder::Bitrate::Kbps128,
+      Bitrate::Kbps160 => mp3lame_encoder::Bitrate::Kbps160,
+      Bitrate::Kbps192 => mp3lame_encoder::Bitrate::Kbps192,
+      Bitrate::Kbps224 => mp3lame_encoder::Bitrate::Kbps224,
+      Bitrate::Kbps256 => mp3lame_encoder::Bitrate::Kbps256,
+      Bitrate::Kbps320 => mp3lame_encoder::Bitrate::Kbps320,
+    }
+  }
+}
+
+#[napi]
+/// MPEG mode
+pub enum Mode {
+  Mono,
+  Stereo,
+  JointStereo,
+  DualChannel,
+  NotSet,
+}
+
+impl From<Mode> for mp3lame_encoder::Mode {
+  fn from(value: Mode) -> Self {
+    match value {
+      Mode::Mono => mp3lame_encoder::Mode::Mono,
+      Mode::Stereo => mp3lame_encoder::Mode::Stereo,
+      Mode::JointStereo => mp3lame_encoder::Mode::JointStereo,
+      Mode::DualChannel => mp3lame_encoder::Mode::DaulChannel,
+      Mode::NotSet => mp3lame_encoder::Mode::NotSet,
+    }
+  }
+}
+
+#[napi(object, object_to_js = false)]
+pub struct EncodeOptions {
+  pub channels: u32,
+  pub quality: Option<Quality>,
+  pub bitrate: Option<Bitrate>,
+  pub sample_rate: Option<u32>,
+  pub mode: Option<Mode>,
+}
+
+#[napi]
+pub struct Mp3Encoder {
+  encoder: Encoder,
+}
+
+#[napi]
+impl Mp3Encoder {
+  #[napi(constructor)]
+  pub fn new(options: EncodeOptions) -> Result<Self> {
+    let mut builder = Builder::new().ok_or(LameError::CreateBuilderFailed)?;
+    builder
+      .set_num_channels(options.channels as u8)
+      .map_err(LameError::BuildError)?;
+    if let Some(quality) = options.quality {
+      builder
+        .set_quality(quality.into())
+        .map_err(LameError::BuildError)?;
+    }
+    if let Some(bitrate) = options.bitrate {
+      builder
+        .set_brate(bitrate.into())
+        .map_err(LameError::BuildError)?;
+    }
+    if let Some(sample_rate) = options.sample_rate {
+      builder
+        .set_sample_rate(sample_rate)
+        .map_err(LameError::BuildError)?;
+    }
+    if let Some(mode) = options.mode {
+      builder
+        .set_mode(mode.into())
+        .map_err(LameError::BuildError)?;
+    }
+    Ok(Self {
+      encoder: builder.build().map_err(LameError::BuildError)?,
+    })
+  }
+
+  #[napi]
+  pub fn encode(&mut self, input: &[f32]) -> Result<Uint8Array> {
+    let mut output = Vec::with_capacity(input.len());
+    output.reserve(mp3lame_encoder::max_required_buffer_size(input.len()));
+    let encoded_size = self
+      .encoder
+      .encode(MonoPcm(input), output.spare_capacity_mut())
+      .map_err(LameError::EncodeError)?;
+    unsafe {
+      output.set_len(output.len().wrapping_add(encoded_size));
+    }
+    let encoded_size = self
+      .encoder
+      .flush::<FlushNoGap>(output.spare_capacity_mut())
+      .map_err(LameError::EncodeError)?;
+    unsafe {
+      output.set_len(output.len().wrapping_add(encoded_size));
+    }
+    Ok(output.into())
+  }
+}

packages/frontend/native/package.json

@@ -25,7 +25,7 @@
     ]
   },
   "devDependencies": {
-    "@napi-rs/cli": "3.0.0-alpha.70",
+    "@napi-rs/cli": "3.0.0-alpha.72",
     "@napi-rs/whisper": "^0.0.4",
     "@types/node": "^22.0.0",
     "ava": "^6.2.0",
@@ -38,8 +38,8 @@
   },
   "scripts": {
     "artifacts": "napi artifacts",
-    "build": "napi build -p affine_native --platform --release --no-const-enum",
+    "build": "napi build -p affine_native --platform --release --no-dts-cache",
-    "build:debug": "napi build -p affine_native --platform",
+    "build:debug": "napi build -p affine_native --platform --no-dts-cache",
     "universal": "napi universal",
     "test": "ava",
     "version": "napi version"

yarn.lock

@@ -748,7 +748,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@affine/native@workspace:packages/frontend/native"
   dependencies:
-    "@napi-rs/cli": "npm:3.0.0-alpha.70"
+    "@napi-rs/cli": "npm:3.0.0-alpha.72"
     "@napi-rs/whisper": "npm:^0.0.4"
     "@types/node": "npm:^22.0.0"
     ava: "npm:^6.2.0"
@@ -799,7 +799,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@affine/server-native@workspace:packages/backend/native"
   dependencies:
-    "@napi-rs/cli": "npm:3.0.0-alpha.70"
+    "@napi-rs/cli": "npm:3.0.0-alpha.72"
     lib0: "npm:^0.2.99"
     tiktoken: "npm:^1.0.17"
     tinybench: "npm:^3.0.7"
@@ -8163,9 +8163,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@napi-rs/cli@npm:3.0.0-alpha.70":
+"@napi-rs/cli@npm:3.0.0-alpha.72":
-  version: 3.0.0-alpha.70
+  version: 3.0.0-alpha.72
-  resolution: "@napi-rs/cli@npm:3.0.0-alpha.70"
+  resolution: "@napi-rs/cli@npm:3.0.0-alpha.72"
   dependencies:
     "@inquirer/prompts": "npm:^7.0.0"
     "@napi-rs/cross-toolchain": "npm:^0.0.19"
@@ -8192,7 +8192,7 @@ __metadata:
   bin:
     napi: ./dist/cli.js
     napi-raw: ./cli.mjs
-  checksum: 10/d3a48c1d20c351ef11ee26f04ef1aacad6abb750ba8e30cbbac21d380be8a70580af7b7a7dac05924db25d032a582ac3d0a78d91b88e9ce0ab70eb35184b661b
+  checksum: 10/8575e8eaab880bd3dd41dd3f7fc2e79ca7fcb3197038c4aedc41fcfe46ab10fa34f70b0370cb52325b07ca8ec6115c87f48b8bd7850ca10964933563718a6ede
   languageName: node
   linkType: hard