diff --git a/.github/actions/build-rust/action.yml b/.github/actions/build-rust/action.yml index d28d6b6d91..ca7141a27a 100644 --- a/.github/actions/build-rust/action.yml +++ b/.github/actions/build-rust/action.yml @@ -44,7 +44,7 @@ runs: RUSTUP_HOME: ${{ env.DEV_DRIVE }}/.rustup - name: Set CC - if: ${{ contains(inputs.target, 'linux') && inputs.package != '@affine/native' && inputs.no-build != 'true' }} + if: ${{ contains(inputs.target, 'linux') && inputs.no-build != 'true' }} working-directory: ${{ env.DEV_DRIVE_WORKSPACE || github.workspace }} shell: bash run: | diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index f4e5db3319..84bcf64642 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -387,6 +387,28 @@ jobs: path: dist.tar.gz if-no-files-found: error + native-unit-test: + name: Native Unit Test + runs-on: ubuntu-latest + needs: + - optimize_ci + - build-native + if: needs.optimize_ci.outputs.skip == 'false' + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js + uses: ./.github/actions/setup-node + with: + extra-flags: workspaces focus @affine-tools/cli @affine/monorepo @affine/native + electron-install: false + - name: Download affine.linux-x64-gnu.node + uses: actions/download-artifact@v4 + with: + name: affine.linux-x64-gnu.node + path: ./packages/frontend/native + - name: Unit Test + run: yarn affine @affine/native test + server-test: name: Server Test runs-on: ubuntu-latest @@ -897,6 +919,7 @@ jobs: - build-native - build-server-native - build-electron-renderer + - native-unit-test - server-test - rust-test - copilot-api-test diff --git a/Cargo.lock b/Cargo.lock index a6d308460d..0cc2a0eef0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,6 +70,7 @@ dependencies = [ "coreaudio-rs", "dispatch2", "libc", + "mp3lame-encoder", "napi", "napi-build", "napi-derive", @@ -77,6 +78,7 @@ dependencies = [ "objc2-foundation", "rubato", "screencapturekit", + "symphonia", "thiserror 2.0.11", "uuid", ] @@ -287,6 +289,12 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-compat" version = "0.2.4" @@ -339,6 +347,15 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "autotools" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" +dependencies = [ + "cc", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -501,6 +518,12 @@ version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "bytemuck" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" + [[package]] name = "byteorder" version = "1.5.0" @@ -953,9 +976,18 @@ dependencies = [ [[package]] name = "ctor" -version = "0.3.0" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f06b1425736ba96096116f063c9d10be2352a7cde0cbea829a717008e114aec9" +checksum = "21d960ecacd0a1bf55e73144b72de745e7bf275c7952c50e36e8af0a0cb7ab1f" +dependencies = [ + "ctor-proc-macro", +] + +[[package]] +name = "ctor-proc-macro" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c426d2ba3e525b39c1f0a9ba41b9fe61878dee11fa4e4a76b6ab440f46c5db5d" [[package]] name = "dashmap" @@ -1174,6 +1206,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fancy-regex" version = "0.13.0" @@ -2094,6 +2132,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mp3lame-encoder" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc8c8b5cdbe788ccd1098c3d3635298a011cffdebdd3460c9ca5060a7551557b" +dependencies = [ + "libc", + "mp3lame-sys", +] + +[[package]] +name = "mp3lame-sys" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acaec8842b2ebd61692a6c8c2b9f3edbf5c36e5e5c4677b5911430eaf859377c" +dependencies = [ + "autotools", + "cc", + "libc", +] + [[package]] name = "nanoid" version = "0.4.0" @@ -2105,9 +2164,9 @@ dependencies = [ [[package]] name = "napi" -version = "3.0.0-alpha.28" +version = "3.0.0-alpha.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd957e2cc4bd62b730b10ff1f35775f8a81dac84a3bfac273b0ec4336f53ab8" +checksum = "b1911b4f0d33fbcb5f46ff68319ec053ab8a655f3a17440eae1246a23ba2ad78" dependencies = [ "anyhow", "bitflags 2.8.0", @@ -2121,15 +2180,15 @@ dependencies = [ [[package]] name = "napi-build" -version = "2.1.4" +version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db836caddef23662b94e16bf1f26c40eceb09d6aee5d5b06a7ac199320b69b19" +checksum = "40685973218af4aa4b42486652692c294c44b5a67e4b2202df721c9063f2e51c" [[package]] name = "napi-derive" -version = "3.0.0-alpha.26" +version = "3.0.0-alpha.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0f0b6f3f77925d8fd2030855af659ce428a7bb6e10e94852e226f509186ba7c" +checksum = "c8097918a9af1976700eac6944b120b65ad17bf6d38906703d2b68e17ee89256" dependencies = [ "convert_case 0.7.1", "napi-derive-backend", @@ -2140,9 +2199,9 @@ dependencies = [ [[package]] name = "napi-derive-backend" -version = "2.0.0-alpha.26" +version = "2.0.0-alpha.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c694bb49a2fa84dd9542d51eece39a57519f9cf1fc2deefa9d119ab8181e374d" +checksum = "8e5adc92fcdec3aa09f591bd2b139d7c669399f34b8211fe653641b52d40d3b3" dependencies = [ "convert_case 0.7.1", "proc-macro2", @@ -3558,6 +3617,202 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symphonia" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "815c942ae7ee74737bb00f965fa5b5a2ac2ce7b6c01c0cc169bbeaf7abd5f5a9" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-adpcm", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-caf", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72e34f34298a7308d4397a6c7fbf5b84c5d491231ce3dd379707ba673ab3bd97" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c01c2aae70f0f1fb096b6f0ff112a930b1fb3626178fba3ae68b09dce71706d4" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdbf25b545ad0d3ee3e891ea643ad115aff4ca92f6aec472086b957a58522f70" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c94e1feac3327cd616e973d5be69ad36b3945f16b06f19c6773fc3ac0b426a0f" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d8a6666649a08412906476a8b0efd9b9733e241180189e9f92b09c08d0e38f3" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f395a67057c2ebc5e84d7bb1be71cce1a7ba99f64e0f0f0e303a03f79116f89b" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a98765fb46a0a6732b007f7e2870c2129b6f78d87db7987e6533c8f164a9f30" +dependencies = [ + "log", + "symphonia-core", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-core" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "798306779e3dc7d5231bd5691f5a813496dc79d3f56bf82e25789f2094e022c3" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", + "rustfft", +] + +[[package]] +name = "symphonia-format-caf" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e43c99c696a388295a29fe71b133079f5d8b18041cf734c5459c35ad9097af50" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abfdf178d697e50ce1e5d9b982ba1b94c47218e03ec35022d9f0e071a16dc844" +dependencies = [ + "encoding_rs", + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bb43471a100f7882dc9937395bd5ebee8329298e766250b15b3875652fe3d6f" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada3505789516bcf00fc1157c67729eded428b455c27ca370e41f4d785bfa931" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f7be232f962f937f4b7115cbe62c330929345434c834359425e043bfd15f50" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc622b9841a10089c5b18e99eb904f4341615d5aa55bbf4eedde1be721a4023c" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-utils-xiph" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "484472580fa49991afda5f6550ece662237b00c6f562c7d9638d1b086ed010fe" +dependencies = [ + "symphonia-core", + "symphonia-metadata", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index 65c9eeb22c..5483e78f47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,9 +26,10 @@ file-format = { version = "0.26", features = ["reader"] } homedir = "0.3" libc = "0.2" mimalloc = "0.1" -napi = { version = "3.0.0-alpha.12", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] } +mp3lame-encoder = "0.2" +napi = { version = "3.0.0-alpha.31", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] } napi-build = { version = "2" } -napi-derive = { version = "3.0.0-alpha.12" } +napi-derive = { version = "3.0.0-alpha.28" } notify = { version = "8", features = ["serde"] } objc2 = "0.6" objc2-foundation = "0.3" @@ -42,6 +43,7 @@ serde = "1" serde_json = "1" sha3 = "0.10" sqlx = { version = "0.8", default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] } +symphonia = { version = "0.5", features = ["all", "opt-simd"] } thiserror = "2" tiktoken-rs = "0.6" tokio = "1.37" diff --git a/README.md b/README.md index ddf208f1ba..09dca91399 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,7 @@ We would also like to give thanks to open-source projects that make AFFiNE possi - [Jotai](https://github.com/pmndrs/jotai) - Primitive and flexible state management for React. - [async-call-rpc](https://github.com/Jack-Works/async-call-rpc) - A lightweight JSON RPC client & server. - [Vite](https://github.com/vitejs/vite) - Next generation frontend tooling. +- [lame](https://lame.sourceforge.io/) - High quality MPEG Audio Layer III (MP3) encoder. - Other upstream [dependencies](https://github.com/toeverything/AFFiNE/network/dependencies). Thanks a lot to the community for providing such powerful and simple libraries, so that we can focus more on the implementation of the product logic, and we hope that in the future our projects will also provide a more easy-to-use knowledge base for everyone. diff --git a/packages/backend/native/package.json b/packages/backend/native/package.json index cfe1572d6f..9ab1c451c4 100644 --- a/packages/backend/native/package.json +++ b/packages/backend/native/package.json @@ -33,7 +33,7 @@ "build:debug": "napi build" }, "devDependencies": { - "@napi-rs/cli": "3.0.0-alpha.70", + "@napi-rs/cli": "3.0.0-alpha.72", "lib0": "^0.2.99", "tiktoken": "^1.0.17", "tinybench": "^3.0.7", diff --git a/packages/frontend/native/__tests__/audio.spec.mts b/packages/frontend/native/__tests__/audio.spec.mts new file mode 100644 index 0000000000..9b7692590c --- /dev/null +++ b/packages/frontend/native/__tests__/audio.spec.mts @@ -0,0 +1,23 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { tmpdir } from 'node:os'; + +import test from 'ava'; + +import { decodeAudio, Mp3Encoder } from '../index.js'; + +const __dirname = join(fileURLToPath(import.meta.url), '..'); + +const wav = await readFile(join(__dirname, 'fixtures', 'recording.wav')); + +test('convert wav to mp3', async t => { + const audio = await decodeAudio(wav); + const mp3 = new Mp3Encoder({ + channels: 1, + }); + await t.notThrowsAsync(async () => { + const mp3Data = mp3.encode(audio); + await writeFile(join(tmpdir(), 'recording.mp3'), mp3Data); + }); +}); diff --git a/packages/frontend/native/__tests__/fixtures/recording.wav b/packages/frontend/native/__tests__/fixtures/recording.wav new file mode 100644 index 0000000000..32d3b9694d Binary files /dev/null and b/packages/frontend/native/__tests__/fixtures/recording.wav differ diff --git a/packages/frontend/native/index.d.ts b/packages/frontend/native/index.d.ts index ec88d51d5e..2c6afc574e 100644 --- a/packages/frontend/native/index.d.ts +++ b/packages/frontend/native/index.d.ts @@ -61,6 +61,11 @@ export declare class DocStoragePool { clearClocks(universalId: string): Promise } +export declare class Mp3Encoder { + constructor(options: EncodeOptions) + encode(input: Float32Array): Uint8Array +} + export declare class RecordingPermissions { audio: boolean screen: boolean @@ -113,6 +118,42 @@ export declare class SqliteConnection { checkpoint(): Promise } +/**Enumeration of valid values for `set_brate` */ +export declare enum Bitrate { + /**8_000 */ + Kbps8 = 8, + /**16_000 */ + Kbps16 = 16, + /**24_000 */ + Kbps24 = 24, + /**32_000 */ + Kbps32 = 32, + /**40_000 */ + Kbps40 = 40, + /**48_000 */ + Kbps48 = 48, + /**64_000 */ + Kbps64 = 64, + /**80_000 */ + Kbps80 = 80, + /**96_000 */ + Kbps96 = 96, + /**112_000 */ + Kbps112 = 112, + /**128_000 */ + Kbps128 = 128, + /**160_000 */ + Kbps160 = 160, + /**192_000 */ + Kbps192 = 192, + /**224_000 */ + Kbps224 = 224, + /**256_000 */ + Kbps256 = 256, + /**320_000 */ + Kbps320 = 320 +} + export interface Blob { key: string data: Uint8Array @@ -127,6 +168,11 @@ export interface BlobRow { timestamp: Date } +export declare function decodeAudio(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null, signal?: AbortSignal | undefined | null): Promise + +/** Decode audio file into a Float32Array */ +export declare function decodeAudioSync(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null): Float32Array + export interface DocClock { docId: string timestamp: Date @@ -149,6 +195,14 @@ export interface DocUpdate { bin: Uint8Array } +export interface EncodeOptions { + channels: number + quality?: Quality + bitrate?: Bitrate + sampleRate?: number + mode?: Mode +} + export interface InsertRow { docId?: string data: Uint8Array @@ -163,6 +217,42 @@ export interface ListedBlob { export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise +/** MPEG mode */ +export declare enum Mode { + Mono = 0, + Stereo = 1, + JointStereo = 2, + DualChannel = 3, + NotSet = 4 +} + +/** + *Possible quality parameter. + *From best(0) to worst(9) + */ +export declare enum Quality { + /**Best possible quality */ + Best = 0, + /**Second best */ + SecondBest = 1, + /**Close to best */ + NearBest = 2, + /**Very nice */ + VeryNice = 3, + /**Nice */ + Nice = 4, + /**Good */ + Good = 5, + /**Decent */ + Decent = 6, + /**Okayish */ + Ok = 7, + /**Almost worst */ + SecondWorst = 8, + /**Worst */ + Worst = 9 +} + export interface SetBlob { key: string data: Uint8Array diff --git a/packages/frontend/native/index.js b/packages/frontend/native/index.js index 1e7d719035..2a5fc54f5a 100644 --- a/packages/frontend/native/index.js +++ b/packages/frontend/native/index.js @@ -60,7 +60,13 @@ const isMuslFromChildProcess = () => { } function requireNative() { - if (process.platform === 'android') { + if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) { + try { + nativeBinding = require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH); + } catch (err) { + loadErrors.push(err); + } + } else if (process.platform === 'android') { if (process.arch === 'arm64') { try { return require('./affine.android-arm64.node') @@ -370,9 +376,15 @@ module.exports.ApplicationStateChangedSubscriber = nativeBinding.ApplicationStat module.exports.AudioTapStream = nativeBinding.AudioTapStream module.exports.DocStorage = nativeBinding.DocStorage module.exports.DocStoragePool = nativeBinding.DocStoragePool +module.exports.Mp3Encoder = nativeBinding.Mp3Encoder module.exports.RecordingPermissions = nativeBinding.RecordingPermissions module.exports.ShareableContent = nativeBinding.ShareableContent module.exports.SqliteConnection = nativeBinding.SqliteConnection +module.exports.Bitrate = nativeBinding.Bitrate +module.exports.decodeAudio = nativeBinding.decodeAudio +module.exports.decodeAudioSync = nativeBinding.decodeAudioSync module.exports.mintChallengeResponse = nativeBinding.mintChallengeResponse +module.exports.Mode = nativeBinding.Mode +module.exports.Quality = nativeBinding.Quality module.exports.ValidationResult = nativeBinding.ValidationResult module.exports.verifyChallengeResponse = nativeBinding.verifyChallengeResponse diff --git a/packages/frontend/native/media_capture/Cargo.toml b/packages/frontend/native/media_capture/Cargo.toml index 9dbffcf15f..8722989e75 100644 --- a/packages/frontend/native/media_capture/Cargo.toml +++ b/packages/frontend/native/media_capture/Cargo.toml @@ -7,9 +7,12 @@ version = "0.0.0" crate-type = ["cdylib", "rlib"] [dependencies] -napi = { workspace = true, features = ["napi4"] } -napi-derive = { workspace = true, features = ["type-def"] } -rubato = { workspace = true } +mp3lame-encoder = { workspace = true, features = ["std"] } +napi = { workspace = true, features = ["napi4"] } +napi-derive = { workspace = true, features = ["type-def"] } +rubato = { workspace = true } +symphonia = { workspace = true, features = ["all", "opt-simd"] } +thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] block2 = { workspace = true } @@ -20,7 +23,6 @@ libc = { workspace = true } objc2 = { workspace = true } objc2-foundation = { workspace = true } screencapturekit = { workspace = true } -thiserror = { workspace = true } uuid = { workspace = true, features = ["v4"] } [build-dependencies] diff --git a/packages/frontend/native/media_capture/src/audio_decoder.rs b/packages/frontend/native/media_capture/src/audio_decoder.rs new file mode 100644 index 0000000000..abde907710 --- /dev/null +++ b/packages/frontend/native/media_capture/src/audio_decoder.rs @@ -0,0 +1,179 @@ +use std::{io::Cursor, path::Path}; + +use napi::{ + bindgen_prelude::{AbortSignal, AsyncTask, Float32Array, Result, Status, Uint8Array}, + Task, +}; +use napi_derive::napi; +use rubato::{Resampler, SincFixedIn, SincInterpolationParameters, SincInterpolationType}; +use symphonia::core::{ + audio::{AudioBuffer, Signal}, + codecs::DecoderOptions, + errors::Error, + formats::FormatOptions, + io::MediaSourceStream, + meta::MetadataOptions, + probe::Hint, +}; + +fn decode + Send + Sync + 'static>( + buf: B, + dest_sample_rate: Option, + filename: Option<&str>, +) -> std::result::Result, Error> { + // Create the media source + let mss = MediaSourceStream::new(Box::new(Cursor::new(buf)), Default::default()); + + // Create a probe hint using the file extension + let mut hint = Hint::new(); + if let Some(ext) = + filename.and_then(|filename| Path::new(filename).extension().and_then(|ext| ext.to_str())) + { + hint.with_extension(ext); + } + + let format_opts = FormatOptions { + enable_gapless: true, + ..Default::default() + }; + let metadata_opts = MetadataOptions::default(); + let decoder_opts = DecoderOptions::default(); + let probed = symphonia::default::get_probe().format(&hint, mss, &format_opts, &metadata_opts)?; + + let mut format = probed.format; + + let track = format + .default_track() + .ok_or(Error::Unsupported("No default track found"))?; + + let totol_samples = track + .codec_params + .n_frames + .ok_or(Error::Unsupported("No duration found"))?; + let sample_rate = track + .codec_params + .sample_rate + .ok_or(Error::Unsupported("No samplerate found"))?; + + let mut decoder = symphonia::default::get_codecs().make(&track.codec_params, &decoder_opts)?; + + let mut output: Vec = Vec::with_capacity(totol_samples as usize); + // Decode loop + while let Ok(packet) = format.next_packet() { + let decoded = decoder.decode(&packet)?; + let spec = decoded.spec(); + let mut audio_buf: AudioBuffer = AudioBuffer::new(decoded.capacity() as u64, *spec); + decoded.convert(&mut audio_buf); + + if spec.channels.count() > 1 { + // Mix all channels into mono + for i in 0..audio_buf.chan(0).len() { + let mut sample_sum = 0.0; + for ch in 0..spec.channels.count() { + sample_sum += audio_buf.chan(ch)[i]; + } + output.push(sample_sum / spec.channels.count() as f32); + } + } else { + output.extend_from_slice(audio_buf.chan(0)); + } + } + + let Some(dest_sample_rate) = dest_sample_rate else { + return Ok(output); + }; + + if sample_rate != dest_sample_rate { + // Calculate parameters for resampling + let params = SincInterpolationParameters { + sinc_len: 256, + f_cutoff: 0.95, + interpolation: SincInterpolationType::Linear, + oversampling_factor: 256, + window: rubato::WindowFunction::BlackmanHarris2, + }; + + let mut resampler = SincFixedIn::::new( + dest_sample_rate as f64 / sample_rate as f64, + 2.0, + params, + output.len(), + 1, + ) + .map_err(|_| Error::Unsupported("Failed to create resampler"))?; + + let waves_in = vec![output]; + let mut waves_out = resampler + .process(&waves_in, None) + .map_err(|_| Error::Unsupported("Failed to run resampler"))?; + output = waves_out + .pop() + .ok_or(Error::Unsupported("No resampled output found"))?; + } + + Ok(output) +} + +#[napi] +/// Decode audio file into a Float32Array +pub fn decode_audio_sync( + buf: Uint8Array, + dest_sample_rate: Option, + filename: Option, +) -> Result { + decode(buf, dest_sample_rate, filename.as_deref()) + .map(Float32Array::new) + .map_err(|e| { + napi::Error::new( + Status::InvalidArg, + format!("Decode audio into Float32Array failed: {e}"), + ) + }) +} + +pub struct DecodeAudioTask { + buf: Uint8Array, + dest_sample_rate: Option, + filename: Option, +} + +#[napi] +impl Task for DecodeAudioTask { + type Output = Vec; + type JsValue = Float32Array; + + fn compute(&mut self) -> Result { + decode( + std::mem::replace(&mut self.buf, Uint8Array::new(vec![])), + self.dest_sample_rate, + self.filename.as_deref(), + ) + .map_err(|e| { + napi::Error::new( + Status::InvalidArg, + format!("Decode audio into Float32Array failed: {e}"), + ) + }) + } + + fn resolve(&mut self, _: napi::Env, output: Self::Output) -> Result { + Ok(Float32Array::new(output)) + } +} + +#[napi] +pub fn decode_audio( + buf: Uint8Array, + dest_sample_rate: Option, + filename: Option, + signal: Option, +) -> AsyncTask { + AsyncTask::with_optional_signal( + DecodeAudioTask { + buf, + dest_sample_rate, + filename, + }, + signal, + ) +} diff --git a/packages/frontend/native/media_capture/src/lib.rs b/packages/frontend/native/media_capture/src/lib.rs index a429d055b6..64b04cb383 100644 --- a/packages/frontend/native/media_capture/src/lib.rs +++ b/packages/frontend/native/media_capture/src/lib.rs @@ -2,3 +2,5 @@ pub mod macos; #[cfg(target_os = "macos")] pub(crate) use macos::*; +pub mod audio_decoder; +pub mod mp3; diff --git a/packages/frontend/native/media_capture/src/mp3.rs b/packages/frontend/native/media_capture/src/mp3.rs new file mode 100644 index 0000000000..3dd9c267e5 --- /dev/null +++ b/packages/frontend/native/media_capture/src/mp3.rs @@ -0,0 +1,215 @@ +use mp3lame_encoder::{Builder, Encoder, FlushNoGap, MonoPcm}; +use napi::bindgen_prelude::{Result, Uint8Array}; +use napi_derive::napi; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum LameError { + #[error("Create builder failed")] + CreateBuilderFailed, + #[error("Failed to create encoder")] + BuildError(#[from] mp3lame_encoder::BuildError), + #[error("Failed to encode")] + EncodeError(#[from] mp3lame_encoder::EncodeError), +} + +impl From for napi::Error { + fn from(value: LameError) -> Self { + napi::Error::new(napi::Status::GenericFailure, value.to_string()) + } +} + +#[napi] +///Possible quality parameter. +///From best(0) to worst(9) +pub enum Quality { + ///Best possible quality + Best = 0, + ///Second best + SecondBest = 1, + ///Close to best + NearBest = 2, + ///Very nice + VeryNice = 3, + ///Nice + Nice = 4, + ///Good + Good = 5, + ///Decent + Decent = 6, + ///Okayish + Ok = 7, + ///Almost worst + SecondWorst = 8, + ///Worst + Worst = 9, +} + +impl From for mp3lame_encoder::Quality { + fn from(value: Quality) -> Self { + match value { + Quality::Best => mp3lame_encoder::Quality::Best, + Quality::SecondBest => mp3lame_encoder::Quality::SecondBest, + Quality::NearBest => mp3lame_encoder::Quality::NearBest, + Quality::VeryNice => mp3lame_encoder::Quality::VeryNice, + Quality::Nice => mp3lame_encoder::Quality::Nice, + Quality::Good => mp3lame_encoder::Quality::Good, + Quality::Decent => mp3lame_encoder::Quality::Decent, + Quality::Ok => mp3lame_encoder::Quality::Ok, + Quality::SecondWorst => mp3lame_encoder::Quality::SecondWorst, + Quality::Worst => mp3lame_encoder::Quality::Worst, + } + } +} + +#[napi] +#[repr(u16)] +///Enumeration of valid values for `set_brate` +pub enum Bitrate { + ///8_000 + Kbps8 = 8, + ///16_000 + Kbps16 = 16, + ///24_000 + Kbps24 = 24, + ///32_000 + Kbps32 = 32, + ///40_000 + Kbps40 = 40, + ///48_000 + Kbps48 = 48, + ///64_000 + Kbps64 = 64, + ///80_000 + Kbps80 = 80, + ///96_000 + Kbps96 = 96, + ///112_000 + Kbps112 = 112, + ///128_000 + Kbps128 = 128, + ///160_000 + Kbps160 = 160, + ///192_000 + Kbps192 = 192, + ///224_000 + Kbps224 = 224, + ///256_000 + Kbps256 = 256, + ///320_000 + Kbps320 = 320, +} + +impl From for mp3lame_encoder::Bitrate { + fn from(value: Bitrate) -> Self { + match value { + Bitrate::Kbps8 => mp3lame_encoder::Bitrate::Kbps8, + Bitrate::Kbps16 => mp3lame_encoder::Bitrate::Kbps16, + Bitrate::Kbps24 => mp3lame_encoder::Bitrate::Kbps24, + Bitrate::Kbps32 => mp3lame_encoder::Bitrate::Kbps32, + Bitrate::Kbps40 => mp3lame_encoder::Bitrate::Kbps40, + Bitrate::Kbps48 => mp3lame_encoder::Bitrate::Kbps48, + Bitrate::Kbps64 => mp3lame_encoder::Bitrate::Kbps64, + Bitrate::Kbps80 => mp3lame_encoder::Bitrate::Kbps80, + Bitrate::Kbps96 => mp3lame_encoder::Bitrate::Kbps96, + Bitrate::Kbps112 => mp3lame_encoder::Bitrate::Kbps112, + Bitrate::Kbps128 => mp3lame_encoder::Bitrate::Kbps128, + Bitrate::Kbps160 => mp3lame_encoder::Bitrate::Kbps160, + Bitrate::Kbps192 => mp3lame_encoder::Bitrate::Kbps192, + Bitrate::Kbps224 => mp3lame_encoder::Bitrate::Kbps224, + Bitrate::Kbps256 => mp3lame_encoder::Bitrate::Kbps256, + Bitrate::Kbps320 => mp3lame_encoder::Bitrate::Kbps320, + } + } +} + +#[napi] +/// MPEG mode +pub enum Mode { + Mono, + Stereo, + JointStereo, + DualChannel, + NotSet, +} + +impl From for mp3lame_encoder::Mode { + fn from(value: Mode) -> Self { + match value { + Mode::Mono => mp3lame_encoder::Mode::Mono, + Mode::Stereo => mp3lame_encoder::Mode::Stereo, + Mode::JointStereo => mp3lame_encoder::Mode::JointStereo, + Mode::DualChannel => mp3lame_encoder::Mode::DaulChannel, + Mode::NotSet => mp3lame_encoder::Mode::NotSet, + } + } +} + +#[napi(object, object_to_js = false)] +pub struct EncodeOptions { + pub channels: u32, + pub quality: Option, + pub bitrate: Option, + pub sample_rate: Option, + pub mode: Option, +} + +#[napi] +pub struct Mp3Encoder { + encoder: Encoder, +} + +#[napi] +impl Mp3Encoder { + #[napi(constructor)] + pub fn new(options: EncodeOptions) -> Result { + let mut builder = Builder::new().ok_or(LameError::CreateBuilderFailed)?; + builder + .set_num_channels(options.channels as u8) + .map_err(LameError::BuildError)?; + if let Some(quality) = options.quality { + builder + .set_quality(quality.into()) + .map_err(LameError::BuildError)?; + } + if let Some(bitrate) = options.bitrate { + builder + .set_brate(bitrate.into()) + .map_err(LameError::BuildError)?; + } + if let Some(sample_rate) = options.sample_rate { + builder + .set_sample_rate(sample_rate) + .map_err(LameError::BuildError)?; + } + if let Some(mode) = options.mode { + builder + .set_mode(mode.into()) + .map_err(LameError::BuildError)?; + } + Ok(Self { + encoder: builder.build().map_err(LameError::BuildError)?, + }) + } + + #[napi] + pub fn encode(&mut self, input: &[f32]) -> Result { + let mut output = Vec::with_capacity(input.len()); + output.reserve(mp3lame_encoder::max_required_buffer_size(input.len())); + let encoded_size = self + .encoder + .encode(MonoPcm(input), output.spare_capacity_mut()) + .map_err(LameError::EncodeError)?; + unsafe { + output.set_len(output.len().wrapping_add(encoded_size)); + } + let encoded_size = self + .encoder + .flush::(output.spare_capacity_mut()) + .map_err(LameError::EncodeError)?; + unsafe { + output.set_len(output.len().wrapping_add(encoded_size)); + } + Ok(output.into()) + } +} diff --git a/packages/frontend/native/package.json b/packages/frontend/native/package.json index beca6f1b45..b6f154d2b9 100644 --- a/packages/frontend/native/package.json +++ b/packages/frontend/native/package.json @@ -25,7 +25,7 @@ ] }, "devDependencies": { - "@napi-rs/cli": "3.0.0-alpha.70", + "@napi-rs/cli": "3.0.0-alpha.72", "@napi-rs/whisper": "^0.0.4", "@types/node": "^22.0.0", "ava": "^6.2.0", @@ -38,8 +38,8 @@ }, "scripts": { "artifacts": "napi artifacts", - "build": "napi build -p affine_native --platform --release --no-const-enum", - "build:debug": "napi build -p affine_native --platform", + "build": "napi build -p affine_native --platform --release --no-dts-cache", + "build:debug": "napi build -p affine_native --platform --no-dts-cache", "universal": "napi universal", "test": "ava", "version": "napi version" diff --git a/yarn.lock b/yarn.lock index 12555b7808..bbb48c5285 100644 --- a/yarn.lock +++ b/yarn.lock @@ -748,7 +748,7 @@ __metadata: version: 0.0.0-use.local resolution: "@affine/native@workspace:packages/frontend/native" dependencies: - "@napi-rs/cli": "npm:3.0.0-alpha.70" + "@napi-rs/cli": "npm:3.0.0-alpha.72" "@napi-rs/whisper": "npm:^0.0.4" "@types/node": "npm:^22.0.0" ava: "npm:^6.2.0" @@ -799,7 +799,7 @@ __metadata: version: 0.0.0-use.local resolution: "@affine/server-native@workspace:packages/backend/native" dependencies: - "@napi-rs/cli": "npm:3.0.0-alpha.70" + "@napi-rs/cli": "npm:3.0.0-alpha.72" lib0: "npm:^0.2.99" tiktoken: "npm:^1.0.17" tinybench: "npm:^3.0.7" @@ -8163,9 +8163,9 @@ __metadata: languageName: node linkType: hard -"@napi-rs/cli@npm:3.0.0-alpha.70": - version: 3.0.0-alpha.70 - resolution: "@napi-rs/cli@npm:3.0.0-alpha.70" +"@napi-rs/cli@npm:3.0.0-alpha.72": + version: 3.0.0-alpha.72 + resolution: "@napi-rs/cli@npm:3.0.0-alpha.72" dependencies: "@inquirer/prompts": "npm:^7.0.0" "@napi-rs/cross-toolchain": "npm:^0.0.19" @@ -8192,7 +8192,7 @@ __metadata: bin: napi: ./dist/cli.js napi-raw: ./cli.mjs - checksum: 10/d3a48c1d20c351ef11ee26f04ef1aacad6abb750ba8e30cbbac21d380be8a70580af7b7a7dac05924db25d032a582ac3d0a78d91b88e9ce0ab70eb35184b661b + checksum: 10/8575e8eaab880bd3dd41dd3f7fc2e79ca7fcb3197038c4aedc41fcfe46ab10fa34f70b0370cb52325b07ca8ec6115c87f48b8bd7850ca10964933563718a6ede languageName: node linkType: hard