From 5dbffba08dd6459b6d7314c497bcf14714cb9ae7 Mon Sep 17 00:00:00 2001 From: Brooooooklyn Date: Tue, 25 Feb 2025 06:51:56 +0000 Subject: [PATCH] feat(native): media capture (#9992) --- Cargo.lock | 402 +++++++- Cargo.toml | 6 + .../media-capture-playground/.gitignore | 2 + .../media-capture-playground/package.json | 43 + .../media-capture-playground/server/gemini.ts | 200 ++++ .../media-capture-playground/server/main.ts | 759 +++++++++++++++ .../server/types.d.ts | 4 + .../server/wav-writer.ts | 125 +++ .../tsconfg.node.json | 7 + .../media-capture-playground/tsconfig.json | 10 + .../media-capture-playground/vite.config.ts | 18 + .../media-capture-playground/web/app.tsx | 33 + .../web/components/app-item.tsx | 122 +++ .../web/components/app-list.tsx | 144 +++ .../web/components/icons.tsx | 163 ++++ .../web/components/saved-recording-item.tsx | 872 ++++++++++++++++++ .../web/components/saved-recordings.tsx | 41 + .../media-capture-playground/web/index.html | 13 + .../media-capture-playground/web/main.css | 1 + .../media-capture-playground/web/main.tsx | 11 + .../media-capture-playground/web/types.ts | 55 ++ .../media-capture-playground/web/utils.ts | 19 + packages/frontend/native/.gitignore | 1 + packages/frontend/native/Cargo.toml | 17 +- .../frontend/native/media-capture-exapmle.ts | 149 +++ .../frontend/native/media_capture/Cargo.toml | 26 + .../frontend/native/media_capture/build.rs | 3 + .../frontend/native/media_capture/src/lib.rs | 4 + .../src/macos/audio_stream_basic_desc.rs | 282 ++++++ .../media_capture/src/macos/av_audio_file.rs | 71 ++ .../src/macos/av_audio_format.rs | 95 ++ .../src/macos/av_audio_pcm_buffer.rs | 35 + .../src/macos/ca_tap_description.rs | 84 ++ .../native/media_capture/src/macos/device.rs | 66 ++ .../native/media_capture/src/macos/error.rs | 81 ++ .../native/media_capture/src/macos/mod.rs | 11 + .../native/media_capture/src/macos/pid.rs | 98 ++ .../native/media_capture/src/macos/queue.rs | 12 + .../src/macos/screen_capture_kit.rs | 623 +++++++++++++ .../media_capture/src/macos/tap_audio.rs | 360 ++++++++ packages/frontend/native/package.json | 2 + packages/frontend/native/src/lib.rs | 2 + packages/frontend/native/tsconfig.json | 2 +- tools/utils/src/workspace.gen.ts | 6 + tsconfig.json | 1 + yarn.lock | 784 ++++++++++++++-- 46 files changed, 5791 insertions(+), 74 deletions(-) create mode 100644 packages/frontend/media-capture-playground/.gitignore create mode 100644 packages/frontend/media-capture-playground/package.json create mode 100644 packages/frontend/media-capture-playground/server/gemini.ts create mode 100644 packages/frontend/media-capture-playground/server/main.ts create mode 100644 packages/frontend/media-capture-playground/server/types.d.ts create mode 100644 packages/frontend/media-capture-playground/server/wav-writer.ts create mode 100644 packages/frontend/media-capture-playground/tsconfg.node.json create mode 100644 packages/frontend/media-capture-playground/tsconfig.json create mode 100644 packages/frontend/media-capture-playground/vite.config.ts create mode 100644 packages/frontend/media-capture-playground/web/app.tsx create mode 100644 packages/frontend/media-capture-playground/web/components/app-item.tsx create mode 100644 packages/frontend/media-capture-playground/web/components/app-list.tsx create mode 100644 packages/frontend/media-capture-playground/web/components/icons.tsx create mode 100644 packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx create mode 100644 packages/frontend/media-capture-playground/web/components/saved-recordings.tsx create mode 100644 packages/frontend/media-capture-playground/web/index.html create mode 100644 packages/frontend/media-capture-playground/web/main.css create mode 100644 packages/frontend/media-capture-playground/web/main.tsx create mode 100644 packages/frontend/media-capture-playground/web/types.ts create mode 100644 packages/frontend/media-capture-playground/web/utils.ts create mode 100644 packages/frontend/native/media-capture-exapmle.ts create mode 100644 packages/frontend/native/media_capture/Cargo.toml create mode 100644 packages/frontend/native/media_capture/build.rs create mode 100644 packages/frontend/native/media_capture/src/lib.rs create mode 100644 packages/frontend/native/media_capture/src/macos/audio_stream_basic_desc.rs create mode 100644 packages/frontend/native/media_capture/src/macos/av_audio_file.rs create mode 100644 packages/frontend/native/media_capture/src/macos/av_audio_format.rs create mode 100644 packages/frontend/native/media_capture/src/macos/av_audio_pcm_buffer.rs create mode 100644 packages/frontend/native/media_capture/src/macos/ca_tap_description.rs create mode 100644 packages/frontend/native/media_capture/src/macos/device.rs create mode 100644 packages/frontend/native/media_capture/src/macos/error.rs create mode 100644 packages/frontend/native/media_capture/src/macos/mod.rs create mode 100644 packages/frontend/native/media_capture/src/macos/pid.rs create mode 100644 packages/frontend/native/media_capture/src/macos/queue.rs create mode 100644 packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs create mode 100644 packages/frontend/native/media_capture/src/macos/tap_audio.rs diff --git a/Cargo.lock b/Cargo.lock index 37a83a82e3..e440e0ebd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,25 @@ dependencies = [ "sha3", ] +[[package]] +name = "affine_media_capture" +version = "0.0.0" +dependencies = [ + "block2", + "core-foundation", + "coreaudio-rs", + "dispatch2", + "napi", + "napi-build", + "napi-derive", + "objc2", + "objc2-foundation", + "rubato", + "screencapturekit", + "thiserror 2.0.11", + "uuid", +] + [[package]] name = "affine_mobile_native" version = "0.0.0" @@ -50,6 +69,7 @@ name = "affine_native" version = "0.0.0" dependencies = [ "affine_common", + "affine_media_capture", "affine_nbstore", "affine_sqlite_v1", "napi", @@ -325,6 +345,24 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.8.0", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -340,6 +378,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.8.0" @@ -470,6 +514,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -482,6 +535,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "cgl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff" +dependencies = [ + "libc", +] + [[package]] name = "chrono" version = "0.4.39" @@ -523,6 +585,17 @@ dependencies = [ "half", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.28" @@ -594,12 +667,112 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-audio-types-rs" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02f7359c779907f80443d2b2d1b5a61182abb6d8ffd43b6fcb87a27c327d845f" +dependencies = [ + "core-foundation", +] + +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", + "uuid", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core-graphics" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa95a34622365fa5bbf40b20b75dba8dfa8c94c734aea8ac9a5ca38af14316f1" +dependencies = [ + "bitflags 2.8.0", + "core-foundation", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" +dependencies = [ + "bitflags 2.8.0", + "core-foundation", + "libc", +] + +[[package]] +name = "core-media-rs" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e33a8804301de5fc0f705ea1cdea692233c08bdcee8e42abb258f5de3b9a5e7" +dependencies = [ + "core-audio-types-rs", + "core-foundation", + "core-utils-rs", + "core-video-rs", + "thiserror 2.0.11", +] + +[[package]] +name = "core-utils-rs" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068ec1aa07335261033bf610b2868ea9db05353468b72b0045ae469e00d26121" +dependencies = [ + "core-foundation", + "four-char-code", +] + +[[package]] +name = "core-video-rs" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7559e93f816c05607068cb0b741d997a47709f4ba70b36a02d335f7136b30c46" +dependencies = [ + "core-foundation", + "core-graphics", + "core-utils-rs", + "io-surface", + "thiserror 2.0.11", +] + +[[package]] +name = "coreaudio-rs" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ca07354f6d0640333ef95f48d460a4bcf34812a7e7967f9b44c728a8f37c28" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce857aa0b77d77287acc1ac3e37a05a8c95a2af3647d23b15f263bdaeb7562b" +dependencies = [ + "bindgen", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -745,6 +918,24 @@ dependencies = [ "subtle", ] +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + +[[package]] +name = "dispatch2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a0d569e003ff27784e0e14e4a594048698e0c0f0b66cabcb51511be55a7caa0" +dependencies = [ + "bitflags 2.8.0", + "block2", + "libc", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -860,6 +1051,33 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -869,6 +1087,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "four-char-code" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c661315fd366b2a1f970df7b7cb1a28d2678d49ef4872f7dcc19b4a83150f20b" + [[package]] name = "fs-err" version = "2.11.0" @@ -1288,12 +1512,34 @@ dependencies = [ "hashbrown 0.15.2", ] +[[package]] +name = "io-surface" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8283575d5f0b2e7447ec0840363879d71c0fa325d4c699d5b45208ea4a51f45e" +dependencies = [ + "cgl", + "core-foundation", + "core-foundation-sys", + "leaky-cow", + "libc", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.14" @@ -1338,6 +1584,21 @@ dependencies = [ "spin", ] +[[package]] +name = "leak" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd100e01f1154f2908dfa7d02219aeab25d0b9c7fa955164192e3245255a0c73" + +[[package]] +name = "leaky-cow" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a8225d44241fd324a8af2806ba635fc7c8a7e9a7de4d5cf3ef54e71f5926fc" +dependencies = [ + "leak", +] + [[package]] name = "libc" version = "0.2.169" @@ -1424,6 +1685,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + [[package]] name = "matchers" version = "0.1.0" @@ -1516,7 +1786,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd957e2cc4bd62b730b10ff1f35775f8a81dac84a3bfac273b0ec4336f53ab8" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.8.0", "chrono", "ctor", "napi-build", @@ -1572,7 +1842,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags", + "bitflags 2.8.0", "cfg-if", "cfg_aliases", "libc", @@ -1615,6 +1885,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -1645,6 +1924,16 @@ dependencies = [ "libm", ] +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", + "objc_exception", +] + [[package]] name = "objc2" version = "0.6.0" @@ -1660,7 +1949,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daeaf60f25471d26948a1c2f840e3f7d86f4109e3af4e8e4b5cd70c39690d925" dependencies = [ - "bitflags", + "bitflags 2.8.0", "objc2", ] @@ -1676,13 +1965,22 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a21c6c9014b82c39515db5b396f91645182611c97d24637cf56ac01e5f8d998" dependencies = [ - "bitflags", + "bitflags 2.8.0", "block2", "libc", "objc2", "objc2-core-foundation", ] +[[package]] +name = "objc_exception" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4" +dependencies = [ + "cc", +] + [[package]] name = "object" version = "0.36.7" @@ -1833,6 +2131,15 @@ dependencies = [ "zerocopy 0.7.35", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro2" version = "1.0.93" @@ -1948,13 +2255,22 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "realfft" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390252372b7f2aac8360fc5e72eba10136b166d6faeed97e6d0c8324eb99b2b1" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags", + "bitflags 2.8.0", ] [[package]] @@ -2075,6 +2391,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rubato" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd96992d7e24b3d7f35fdfe02af037a356ac90d41b466945cf3333525a86eea" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "realfft", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -2093,13 +2421,28 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rustfft" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", + "version_check", +] + [[package]] name = "rustix" version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", @@ -2179,6 +2522,22 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "screencapturekit" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd797e2f07759d67fc00c3085c55dae07c37e73dba29dc1159484cbb2946bd6f" +dependencies = [ + "block2", + "core-foundation", + "core-graphics", + "core-media-rs", + "core-utils-rs", + "core-video-rs", + "dispatch", + "objc", +] + [[package]] name = "scroll" version = "0.12.0" @@ -2484,7 +2843,7 @@ checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233" dependencies = [ "atoi", "base64 0.22.1", - "bitflags", + "bitflags 2.8.0", "byteorder", "bytes", "chrono", @@ -2527,7 +2886,7 @@ checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613" dependencies = [ "atoi", "base64 0.22.1", - "bitflags", + "bitflags 2.8.0", "byteorder", "chrono", "crc", @@ -2593,6 +2952,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "stringprep" version = "0.1.5" @@ -2869,6 +3234,16 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "typenum" version = "1.17.0" @@ -3063,6 +3438,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" +dependencies = [ + "getrandom 0.3.1", +] + [[package]] name = "v_htmlescape" version = "0.15.8" @@ -3527,7 +3911,7 @@ version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" dependencies = [ - "bitflags", + "bitflags 2.8.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 66762ae2c0..0c888d98ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,12 @@ affine_common = { path = "./packages/common/native" } affine_nbstore = { path = "./packages/frontend/native/nbstore" } anyhow = "1" base64-simd = "0.8" +block2 = "0.6" chrono = "0.4" +core-foundation = "0.10" +coreaudio-rs = "0.12" criterion2 = { version = "2", default-features = false } +dispatch2 = "0.2" dotenvy = "0.15" file-format = { version = "0.26", features = ["reader"] } homedir = "0.3" @@ -31,6 +35,8 @@ once_cell = "1" parking_lot = "0.12" rand = "0.9" rayon = "1.10" +rubato = "0.16" +screencapturekit = "0.3" serde = "1" serde_json = "1" sha3 = "0.10" diff --git a/packages/frontend/media-capture-playground/.gitignore b/packages/frontend/media-capture-playground/.gitignore new file mode 100644 index 0000000000..4314969635 --- /dev/null +++ b/packages/frontend/media-capture-playground/.gitignore @@ -0,0 +1,2 @@ +recordings +.env \ No newline at end of file diff --git a/packages/frontend/media-capture-playground/package.json b/packages/frontend/media-capture-playground/package.json new file mode 100644 index 0000000000..6b6b331be2 --- /dev/null +++ b/packages/frontend/media-capture-playground/package.json @@ -0,0 +1,43 @@ +{ + "name": "@affine/media-capture-playground", + "private": true, + "type": "module", + "version": "0.0.0", + "scripts": { + "dev:web": "vite", + "dev:server": "tsx --env-file=.env --watch server/main.ts" + }, + "dependencies": { + "@affine/native": "workspace:*", + "@google/generative-ai": "^0.21.0", + "@tailwindcss/vite": "^4.0.6", + "@types/express": "^4", + "@types/multer": "^1", + "@types/react": "^19.0.8", + "@types/react-dom": "^19.0.3", + "@types/socket.io": "^3.0.2", + "@types/socket.io-client": "^3.0.0", + "@vitejs/plugin-react": "^4.3.4", + "chokidar": "^4.0.3", + "express": "^4.21.2", + "express-rate-limit": "^7.1.5", + "fs-extra": "^11.3.0", + "multer": "^1.4.5-lts.1", + "openai": "^4.85.1", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-markdown": "^9.0.3", + "rxjs": "^7.8.1", + "socket.io": "^4.7.4", + "socket.io-client": "^4.7.4", + "swr": "^2.3.2", + "tailwindcss": "^4.0.6", + "tsx": "^4.19.2", + "vite": "^6.1.0" + }, + "devDependencies": { + "@types/fs-extra": "^11", + "@types/react": "^19.0.1", + "@types/react-dom": "^19.0.2" + } +} diff --git a/packages/frontend/media-capture-playground/server/gemini.ts b/packages/frontend/media-capture-playground/server/gemini.ts new file mode 100644 index 0000000000..723a5e742c --- /dev/null +++ b/packages/frontend/media-capture-playground/server/gemini.ts @@ -0,0 +1,200 @@ +import { GoogleGenerativeAI } from '@google/generative-ai'; +import { + GoogleAIFileManager, + type UploadFileResponse, +} from '@google/generative-ai/server'; + +const DEFAULT_MODEL = 'gemini-2.0-flash'; + +export interface TranscriptionResult { + title: string; + summary: string; + segments: { + speaker: string; + start_time: string; + end_time: string; + transcription: string; + }[]; +} + +const PROMPT_TRANSCRIPTION = ` +Generate audio transcription and diarization for the recording. +The recording source is most likely from a video call with multiple speakers. +Output in JSON format with the following structure: + +{ + "segments": [ + { + "speaker": "Speaker A", + "start_time": "MM:SS", + "end_time": "MM:SS", + "transcription": "..." + }, + ... + ], +} +- Use consistent speaker labels throughout +- Accurate timestamps in MM:SS format +- Clean transcription with proper punctuation +- Identify speakers by name if possible, otherwise use "Speaker A/B/C" +`; + +const PROMPT_SUMMARY = ` +Generate a short title and summary of the conversation. The input is in the following JSON format: +{ + "segments": [ + { + "speaker": "Speaker A", + "start_time": "MM:SS", + "end_time": "MM:SS", + "transcription": "..." + }, + ... + ], +} + +Output in JSON format with the following structure: + +{ + "title": "Title of the recording", + "summary": "Summary of the conversation in markdown format" +} + +1. Summary Structure: +- The sumary should be inferred from the speakers' language and context +- All insights should be derived directly from speakers' language and context +- Use hierarchical organization for clear information structure +- Use markdown format for the summary. Use bullet points, lists and other markdown styles when appropriate + +2. Title: +- Come up with a title for the recording. +- The title should be a short description of the recording. +- The title should be a single sentence or a few words. +`; + +export async function gemini( + audioFilePath: string, + options?: { + model?: 'gemini-2.0-flash' | 'gemini-1.5-flash'; + mode?: 'transcript' | 'summary'; + } +) { + if (!process.env.GOOGLE_GEMINI_API_KEY) { + console.error('Missing GOOGLE_GEMINI_API_KEY environment variable'); + throw new Error('GOOGLE_GEMINI_API_KEY is not set'); + } + + // Initialize GoogleGenerativeAI and FileManager with your API_KEY + const genAI = new GoogleGenerativeAI(process.env.GOOGLE_GEMINI_API_KEY); + const fileManager = new GoogleAIFileManager( + process.env.GOOGLE_GEMINI_API_KEY + ); + + async function transcribe( + audioFilePath: string + ): Promise { + let uploadResult: UploadFileResponse | null = null; + + try { + // Upload the audio file + uploadResult = await fileManager.uploadFile(audioFilePath, { + mimeType: 'audio/wav', + displayName: 'audio_transcription.wav', + }); + console.log('File uploaded:', uploadResult.file.uri); + + // Initialize a Gemini model appropriate for your use case. + const model = genAI.getGenerativeModel({ + model: options?.model || DEFAULT_MODEL, + generationConfig: { + responseMimeType: 'application/json', + }, + }); + + // Generate content using a prompt and the uploaded file + const result = await model.generateContent([ + { + fileData: { + fileUri: uploadResult.file.uri, + mimeType: uploadResult.file.mimeType, + }, + }, + { + text: PROMPT_TRANSCRIPTION, + }, + ]); + + const text = result.response.text(); + + try { + const parsed = JSON.parse(text); + return parsed; + } catch (e) { + console.error('Failed to parse transcription JSON:', e); + console.error('Raw text that failed to parse:', text); + return null; + } + } catch (e) { + console.error('Error during transcription:', e); + return null; + } finally { + if (uploadResult) { + await fileManager.deleteFile(uploadResult.file.name); + } + } + } + + async function summarize(transcription: TranscriptionResult) { + try { + const model = genAI.getGenerativeModel({ + model: options?.model || DEFAULT_MODEL, + generationConfig: { + responseMimeType: 'application/json', + }, + }); + + const result = await model.generateContent([ + { + text: PROMPT_SUMMARY + '\n\n' + JSON.stringify(transcription), + }, + ]); + + const text = result.response.text(); + + try { + const parsed = JSON.parse(text); + return parsed; + } catch (e) { + console.error('Failed to parse summary JSON:', e); + console.error('Raw text that failed to parse:', text); + return null; + } + } catch (e) { + console.error('Error during summarization:', e); + return null; + } + } + + const transcription = await transcribe(audioFilePath); + if (!transcription) { + console.error('Transcription failed'); + return null; + } + + const summary = await summarize(transcription); + if (!summary) { + console.error('Summary generation failed'); + return transcription; + } + + const result = { + ...transcription, + ...summary, + }; + console.log('Processing completed:', { + title: result.title, + segmentsCount: result.segments?.length, + }); + + return result; +} diff --git a/packages/frontend/media-capture-playground/server/main.ts b/packages/frontend/media-capture-playground/server/main.ts new file mode 100644 index 0000000000..e132447d63 --- /dev/null +++ b/packages/frontend/media-capture-playground/server/main.ts @@ -0,0 +1,759 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import { exec } from 'node:child_process'; +import { createServer } from 'node:http'; +import { promisify } from 'node:util'; + +import { + type Application, + type AudioTapStream, + ShareableContent, +} from '@affine/native'; +import type { FSWatcher } from 'chokidar'; +import chokidar from 'chokidar'; +import express from 'express'; +import rateLimit from 'express-rate-limit'; +import fs from 'fs-extra'; +import { Server } from 'socket.io'; + +import { gemini, type TranscriptionResult } from './gemini'; +import { WavWriter } from './wav-writer'; + +// Constants +const RECORDING_DIR = './recordings'; +const PORT = process.env.PORT || 6544; + +// Ensure recordings directory exists +fs.ensureDirSync(RECORDING_DIR); +console.log(`📁 Ensuring recordings directory exists at ${RECORDING_DIR}`); + +// Types +interface Recording { + app: Application; + appGroup: Application | null; + buffers: Float32Array[]; + stream: AudioTapStream; + startTime: number; + isWriting: boolean; +} + +interface RecordingStatus { + processId: number; + bundleIdentifier: string; + name: string; + startTime: number; + duration: number; +} + +interface RecordingMetadata { + appName: string; + bundleIdentifier: string; + processId: number; + recordingStartTime: number; + recordingEndTime: number; + recordingDuration: number; + sampleRate: number; + totalSamples: number; +} + +interface AppInfo { + app: Application; + processId: number; + processGroupId: number | null; + bundleIdentifier: string; + name: string; + running: boolean; +} + +interface TranscriptionMetadata { + transcriptionStartTime: number; + transcriptionEndTime: number; + transcriptionStatus: 'not_started' | 'pending' | 'completed' | 'error'; + transcription?: TranscriptionResult; + error?: string; +} + +// State +const recordingMap = new Map(); +let appsSubscriber = () => {}; +let fsWatcher: FSWatcher | null = null; + +// Server setup +const app = express(); +const httpServer = createServer(app); +const io = new Server(httpServer, { + cors: { origin: '*' }, +}); + +app.use(express.json()); + +// Update the static file serving to handle the new folder structure +app.use( + '/recordings', + (req, res, next) => { + // Extract the folder name from the path + const parts = req.path.split('/'); + if (parts.length < 2) { + return res.status(400).json({ error: 'Invalid request path' }); + } + + const folderName = parts[1]; + if (!validateAndSanitizeFolderName(folderName)) { + return res.status(400).json({ error: 'Invalid folder name format' }); + } + + if (req.path.endsWith('.wav')) { + res.setHeader('Content-Type', 'audio/wav'); + } else if (req.path.endsWith('.png')) { + res.setHeader('Content-Type', 'image/png'); + } + next(); + }, + express.static(RECORDING_DIR) +); + +// Recording management +async function saveRecording(recording: Recording): Promise { + try { + recording.isWriting = true; + const app = recording.appGroup || recording.app; + + const totalSamples = recording.buffers.reduce( + (acc, buf) => acc + buf.length, + 0 + ); + + const recordingEndTime = Date.now(); + const recordingDuration = (recordingEndTime - recording.startTime) / 1000; + const expectedSamples = recordingDuration * 44100; + + console.log(`💾 Saving recording for ${app.name}:`); + console.log(`- Process ID: ${app.processId}`); + console.log(`- Bundle ID: ${app.bundleIdentifier}`); + console.log(`- Actual duration: ${recordingDuration.toFixed(2)}s`); + console.log(`- Expected samples: ${Math.floor(expectedSamples)}`); + console.log(`- Actual samples: ${totalSamples}`); + console.log( + `- Sample ratio: ${(totalSamples / expectedSamples).toFixed(2)}` + ); + + // Create a buffer for the mono audio + const buffer = new Float32Array(totalSamples); + let offset = 0; + recording.buffers.forEach(buf => { + buffer.set(buf, offset); + offset += buf.length; + }); + + await fs.ensureDir(RECORDING_DIR); + + const timestamp = Date.now(); + const baseFilename = `${recording.app.bundleIdentifier}-${recording.app.processId}-${timestamp}`; + const recordingDir = `${RECORDING_DIR}/${baseFilename}`; + await fs.ensureDir(recordingDir); + + const wavFilename = `${recordingDir}/recording.wav`; + const transcriptionWavFilename = `${recordingDir}/transcription.wav`; + const metadataFilename = `${recordingDir}/metadata.json`; + const iconFilename = `${recordingDir}/icon.png`; + + // Save high-quality WAV file for playback (44.1kHz) + console.log(`📝 Writing high-quality WAV file to ${wavFilename}`); + const writer = new WavWriter(wavFilename, { targetSampleRate: 44100 }); + writer.write(buffer); + await writer.end(); + console.log('✅ High-quality WAV file written successfully'); + + // Save low-quality WAV file for transcription (8kHz) + console.log( + `📝 Writing transcription WAV file to ${transcriptionWavFilename}` + ); + const transcriptionWriter = new WavWriter(transcriptionWavFilename, { + targetSampleRate: 8000, + }); + transcriptionWriter.write(buffer); + await transcriptionWriter.end(); + console.log('✅ Transcription WAV file written successfully'); + + // Save app icon if available + if (app.icon) { + console.log(`📝 Writing app icon to ${iconFilename}`); + await fs.writeFile(iconFilename, app.icon); + console.log('✅ App icon written successfully'); + } + + console.log(`📝 Writing metadata to ${metadataFilename}`); + // Save metadata (without icon) + const metadata: RecordingMetadata = { + appName: app.name, + bundleIdentifier: app.bundleIdentifier, + processId: app.processId, + recordingStartTime: recording.startTime, + recordingEndTime, + recordingDuration, + sampleRate: 44100, + totalSamples, + }; + + await fs.writeJson(metadataFilename, metadata, { spaces: 2 }); + console.log('✅ Metadata file written successfully'); + + return baseFilename; + } catch (error) { + console.error('❌ Error saving recording:', error); + return null; + } +} + +function getRecordingStatus(): RecordingStatus[] { + return Array.from(recordingMap.entries()).map(([processId, recording]) => ({ + processId, + bundleIdentifier: recording.app.bundleIdentifier, + name: recording.app.name, + startTime: recording.startTime, + duration: Date.now() - recording.startTime, + })); +} + +function emitRecordingStatus() { + io.emit('apps:recording', { recordings: getRecordingStatus() }); +} + +async function startRecording(app: Application) { + if (recordingMap.has(app.processId)) { + console.log( + `⚠ī¸ Recording already in progress for ${app.name} (PID: ${app.processId})` + ); + return; + } + + // Find the root app of the process group + const processGroupId = await getProcessGroupId(app.processId); + const rootApp = processGroupId + ? (shareableContent + .applications() + .find(a => a.processId === processGroupId) ?? app) + : app; + + console.log( + `🎙ī¸ Starting recording for ${rootApp.name} (PID: ${rootApp.processId})` + ); + + const buffers: Float32Array[] = []; + const stream = app.tapAudio((err, samples) => { + if (err) { + console.error(`❌ Audio stream error for ${rootApp.name}:`, err); + return; + } + const recording = recordingMap.get(app.processId); + if (recording && !recording.isWriting) { + buffers.push(new Float32Array(samples)); + } + }); + + recordingMap.set(app.processId, { + app, + appGroup: rootApp, + buffers, + stream, + startTime: Date.now(), + isWriting: false, + }); + + console.log(`✅ Recording started successfully for ${rootApp.name}`); + emitRecordingStatus(); +} + +async function stopRecording(processId: number) { + const recording = recordingMap.get(processId); + if (!recording) { + console.log(`ℹī¸ No active recording found for process ID ${processId}`); + return; + } + + const app = recording.appGroup || recording.app; + + console.log(`⏚ī¸ Stopping recording for ${app.name} (PID: ${app.processId})`); + console.log( + `⏱ī¸ Recording duration: ${((Date.now() - recording.startTime) / 1000).toFixed(2)}s` + ); + + recording.stream.stop(); + const filename = await saveRecording(recording); + recordingMap.delete(processId); + + if (filename) { + console.log(`✅ Recording saved successfully to ${filename}`); + } else { + console.error(`❌ Failed to save recording for ${app.name}`); + } + + emitRecordingStatus(); + return filename; +} + +// File management +async function getRecordings(): Promise< + { + wav: string; + metadata?: RecordingMetadata; + transcription?: TranscriptionMetadata; + }[] +> { + try { + const allItems = await fs.readdir(RECORDING_DIR); + + // First filter out non-directories + const dirs = ( + await Promise.all( + allItems.map(async item => { + const fullPath = `${RECORDING_DIR}/${item}`; + try { + const stat = await fs.stat(fullPath); + return stat.isDirectory() ? item : null; + } catch { + return null; + } + }) + ) + ).filter((d): d is string => d !== null); + + const recordings = await Promise.all( + dirs.map(async dir => { + try { + const recordingPath = `${RECORDING_DIR}/${dir}`; + const metadataPath = `${recordingPath}/metadata.json`; + const transcriptionPath = `${recordingPath}/transcription.json`; + + let metadata: RecordingMetadata | undefined; + try { + metadata = await fs.readJson(metadataPath); + } catch { + // Metadata might not exist + } + + let transcription: TranscriptionMetadata | undefined; + try { + // Check if transcription file exists + const transcriptionExists = await fs.pathExists(transcriptionPath); + if (transcriptionExists) { + transcription = await fs.readJson(transcriptionPath); + } else { + // If transcription.wav exists but no transcription.json, it means transcription is available but not started + transcription = { + transcriptionStartTime: 0, + transcriptionEndTime: 0, + transcriptionStatus: 'not_started', + }; + } + } catch (error) { + console.error(`Error reading transcription for ${dir}:`, error); + } + + return { + wav: dir, + metadata, + transcription, + }; + } catch (error) { + console.error(`Error processing directory ${dir}:`, error); + return null; + } + }) + ); + + // Filter out nulls and sort by recording start time + return recordings + .filter((r): r is NonNullable => r !== null) + .sort( + (a, b) => + (b.metadata?.recordingStartTime ?? 0) - + (a.metadata?.recordingStartTime ?? 0) + ); + } catch (error) { + console.error('Error reading recordings directory:', error); + return []; + } +} + +async function setupRecordingsWatcher() { + if (fsWatcher) { + console.log('🔄 Closing existing recordings watcher'); + await fsWatcher.close(); + } + + try { + console.log('👀 Setting up recordings watcher...'); + const files = await getRecordings(); + console.log(`📊 Found ${files.length} existing recordings`); + io.emit('apps:saved', { recordings: files }); + + fsWatcher = chokidar.watch(RECORDING_DIR, { + ignored: /(^|[/\\])\../, // ignore dotfiles + persistent: true, + ignoreInitial: true, + awaitWriteFinish: { + stabilityThreshold: 500, + pollInterval: 100, + }, + }); + + // Handle file events + fsWatcher + .on('add', async path => { + if (path.endsWith('.wav') || path.endsWith('.json')) { + console.log(`📝 File added: ${path}`); + const files = await getRecordings(); + io.emit('apps:saved', { recordings: files }); + } + }) + .on('change', async path => { + if (path.endsWith('.wav') || path.endsWith('.json')) { + console.log(`📝 File changed: ${path}`); + const files = await getRecordings(); + io.emit('apps:saved', { recordings: files }); + } + }) + .on('unlink', async path => { + if (path.endsWith('.wav') || path.endsWith('.json')) { + console.log(`🗑ī¸ File removed: ${path}`); + const files = await getRecordings(); + io.emit('apps:saved', { recordings: files }); + } + }) + .on('error', error => { + console.error('❌ Error watching recordings directory:', error); + }) + .on('ready', () => { + console.log('✅ Recordings watcher setup complete'); + }); + } catch (error) { + console.error('❌ Error setting up recordings watcher:', error); + } +} + +// Process management +async function getProcessGroupId(pid: number): Promise { + try { + const execAsync = promisify(exec); + const { stdout } = await execAsync(`ps -o pgid -p ${pid}`); + const lines = stdout.trim().split('\n'); + if (lines.length < 2) return null; + + const pgid = parseInt(lines[1].trim(), 10); + return isNaN(pgid) ? null : pgid; + } catch { + return null; + } +} + +// Application management +const shareableContent = new ShareableContent(); + +async function getAllApps(): Promise { + const apps = await Promise.all( + shareableContent.applications().map(async app => { + try { + return { + app, + processId: app.processId, + processGroupId: await getProcessGroupId(app.processId), + bundleIdentifier: app.bundleIdentifier, + name: app.name, + running: app.isRunning, + }; + } catch (error) { + console.error(error); + return null; + } + }) + ); + + const filteredApps = apps.filter( + (v): v is AppInfo => + v !== null && !v.bundleIdentifier.startsWith('com.apple') + ); + + // Stop recording if app is not listed + await Promise.all( + filteredApps.map(async ({ app }) => { + if (!filteredApps.some(a => a.processId === app.processId)) { + await stopRecording(app.processId); + } + }) + ); + + return filteredApps; +} + +function listenToAppStateChanges(apps: AppInfo[]) { + const subscribers = apps.map(({ app }) => { + return ShareableContent.onAppStateChanged(app, () => { + setTimeout(() => { + console.log( + `🔄 Application state changed: ${app.name} (PID: ${app.processId}) is now ${ + app.isRunning ? 'â–ļī¸ running' : '⏚ī¸ stopped' + }` + ); + io.emit('apps:state-changed', { + processId: app.processId, + running: app.isRunning, + }); + if (!app.isRunning) { + stopRecording(app.processId).catch(error => { + console.error('❌ Error stopping recording:', error); + }); + } + }, 50); + }); + }); + + appsSubscriber(); + appsSubscriber = () => { + subscribers.forEach(subscriber => subscriber.unsubscribe()); + }; +} + +// Socket.IO setup +io.on('connection', async socket => { + console.log('🔌 New client connected'); + const initialApps = await getAllApps(); + console.log(`📤 Sending ${initialApps.length} applications to new client`); + socket.emit('apps:all', { apps: initialApps }); + socket.emit('apps:recording', { recordings: getRecordingStatus() }); + + const files = await getRecordings(); + console.log(`📤 Sending ${files.length} saved recordings to new client`); + socket.emit('apps:saved', { recordings: files }); + + listenToAppStateChanges(initialApps); + + socket.on('disconnect', () => { + console.log('🔌 Client disconnected'); + }); +}); + +// Application list change listener +ShareableContent.onApplicationListChanged(() => { + (async () => { + try { + console.log('🔄 Application list changed, updating clients...'); + const apps = await getAllApps(); + console.log(`đŸ“ĸ Broadcasting ${apps.length} applications to all clients`); + io.emit('apps:all', { apps }); + } catch (error) { + console.error('❌ Error handling application list change:', error); + } + })().catch(error => { + console.error('❌ Error in application list change handler:', error); + }); +}); + +// API Routes +const rateLimiter = rateLimit({ + windowMs: 1000, + max: 200, + message: { error: 'Too many requests, please try again later.' }, +}); + +app.get('/permissions', (req, res) => { + const permission = shareableContent.checkRecordingPermissions(); + res.json({ permission }); +}); + +app.get('/apps', async (_req, res) => { + const apps = await getAllApps(); + listenToAppStateChanges(apps); + res.json({ apps }); +}); + +app.get('/apps/saved', rateLimiter, async (_req, res) => { + const files = await getRecordings(); + res.json({ recordings: files }); +}); + +// Utility function to validate and sanitize folder name +function validateAndSanitizeFolderName(folderName: string): string | null { + // Allow alphanumeric characters, hyphens, dots (for bundle IDs) + // Format: bundleId-processId-timestamp + if (!/^[\w.-]+-\d+-\d+$/.test(folderName)) { + return null; + } + + // Remove any path traversal attempts + const sanitized = folderName.replace(/^\.+|\.+$/g, '').replace(/[/\\]/g, ''); + return sanitized; +} + +app.delete('/recordings/:foldername', rateLimiter, async (req, res) => { + const foldername = validateAndSanitizeFolderName(req.params.foldername); + if (!foldername) { + console.error('❌ Invalid folder name format:', req.params.foldername); + return res.status(400).json({ error: 'Invalid folder name format' }); + } + + const recordingDir = `${RECORDING_DIR}/${foldername}`; + + try { + // Ensure the resolved path is within RECORDING_DIR + const resolvedPath = await fs.realpath(recordingDir); + const recordingDirPath = await fs.realpath(RECORDING_DIR); + + if (!resolvedPath.startsWith(recordingDirPath)) { + console.error('❌ Path traversal attempt detected:', { + resolvedPath, + recordingDirPath, + requestedFile: foldername, + }); + return res.status(403).json({ error: 'Access denied' }); + } + + console.log(`🗑ī¸ Deleting recording folder: ${foldername}`); + await fs.remove(recordingDir); + console.log('✅ Recording folder deleted successfully'); + res.status(200).json({ success: true }); + } catch (error) { + const typedError = error as NodeJS.ErrnoException; + if (typedError.code === 'ENOENT') { + console.error('❌ Folder not found:', recordingDir); + res.status(404).json({ error: 'Folder not found' }); + } else { + console.error('❌ Error deleting folder:', { + error: typedError, + code: typedError.code, + message: typedError.message, + path: recordingDir, + }); + res.status(500).json({ + error: `Failed to delete folder: ${typedError.message || 'Unknown error'}`, + }); + } + } +}); + +app.get('/apps/:process_id/icon', (req, res) => { + const processId = parseInt(req.params.process_id); + try { + const app = shareableContent.applicationWithProcessId(processId); + const icon = app.icon; + res.set('Content-Type', 'image/png'); + res.send(icon); + } catch { + res.status(404).json({ error: 'App icon not found' }); + } +}); + +app.post('/apps/:process_id/record', async (req, res) => { + const processId = parseInt(req.params.process_id); + const app = shareableContent.applicationWithProcessId(processId); + await startRecording(app); + res.json({ success: true }); +}); + +app.post('/apps/:process_id/stop', async (req, res) => { + const processId = parseInt(req.params.process_id); + await stopRecording(processId); + res.json({ success: true }); +}); + +// Update transcription endpoint to use folder validation +app.post( + '/recordings/:foldername/transcribe', + rateLimiter, + async (req, res) => { + const foldername = validateAndSanitizeFolderName(req.params.foldername); + if (!foldername) { + console.error('❌ Invalid folder name format:', req.params.foldername); + return res.status(400).json({ error: 'Invalid folder name format' }); + } + + const recordingDir = `${RECORDING_DIR}/${foldername}`; + + try { + // Check if directory exists + await fs.access(recordingDir); + + const transcriptionWavPath = `${recordingDir}/transcription.wav`; + const transcriptionMetadataPath = `${recordingDir}/transcription.json`; + + // Check if transcription file exists + await fs.access(transcriptionWavPath); + + // Create initial transcription metadata + const initialMetadata: TranscriptionMetadata = { + transcriptionStartTime: Date.now(), + transcriptionEndTime: 0, + transcriptionStatus: 'pending', + }; + await fs.writeJson(transcriptionMetadataPath, initialMetadata); + + // Notify clients that transcription has started + io.emit('apps:recording-transcription-start', { filename: foldername }); + + const transcription = await gemini(transcriptionWavPath, { + mode: 'transcript', + }); + + // Update transcription metadata with results + const metadata: TranscriptionMetadata = { + transcriptionStartTime: initialMetadata.transcriptionStartTime, + transcriptionEndTime: Date.now(), + transcriptionStatus: 'completed', + transcription: transcription ?? undefined, + }; + + await fs.writeJson(transcriptionMetadataPath, metadata); + + // Notify clients that transcription is complete + io.emit('apps:recording-transcription-end', { + filename: foldername, + success: true, + transcription, + }); + + res.json({ success: true }); + } catch (error) { + console.error('❌ Error during transcription:', error); + + // Update transcription metadata with error + const metadata: TranscriptionMetadata = { + transcriptionStartTime: Date.now(), + transcriptionEndTime: Date.now(), + transcriptionStatus: 'error', + error: error instanceof Error ? error.message : 'Unknown error', + }; + + await fs + .writeJson(`${recordingDir}/transcription.json`, metadata) + .catch(err => { + console.error('❌ Error saving transcription metadata:', err); + }); + + // Notify clients of transcription error + io.emit('apps:recording-transcription-end', { + filename: foldername, + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + + res.status(500).json({ + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + } +); + +// Start server +httpServer.listen(PORT, () => { + console.log(` +🎙ī¸ Media Capture Server started successfully: +- Port: ${PORT} +- Recordings directory: ${RECORDING_DIR} +- Sample rate: 44.1kHz +- Channels: Mono +`); +}); + +// Initialize file watcher +setupRecordingsWatcher().catch(error => { + console.error('Failed to setup recordings watcher:', error); +}); diff --git a/packages/frontend/media-capture-playground/server/types.d.ts b/packages/frontend/media-capture-playground/server/types.d.ts new file mode 100644 index 0000000000..85b042e1ef --- /dev/null +++ b/packages/frontend/media-capture-playground/server/types.d.ts @@ -0,0 +1,4 @@ +declare module '*.txt' { + const content: string; + export default content; +} diff --git a/packages/frontend/media-capture-playground/server/wav-writer.ts b/packages/frontend/media-capture-playground/server/wav-writer.ts new file mode 100644 index 0000000000..78f3765ddc --- /dev/null +++ b/packages/frontend/media-capture-playground/server/wav-writer.ts @@ -0,0 +1,125 @@ +import fs from 'fs-extra'; + +interface WavWriterConfig { + targetSampleRate?: number; +} + +export class WavWriter { + private readonly file: fs.WriteStream; + private readonly originalSampleRate: number = 44100; + private readonly targetSampleRate: number; + private readonly numChannels = 1; // The audio is mono + private samplesWritten = 0; + private readonly tempFilePath: string; + private readonly finalFilePath: string; + + constructor(finalPath: string, config: WavWriterConfig = {}) { + this.finalFilePath = finalPath; + this.tempFilePath = finalPath + '.tmp'; + this.targetSampleRate = config.targetSampleRate ?? this.originalSampleRate; + this.file = fs.createWriteStream(this.tempFilePath); + this.writeHeader(); // Always write header immediately + } + + private writeHeader() { + const buffer = Buffer.alloc(44); // WAV header is 44 bytes + + // RIFF chunk descriptor + buffer.write('RIFF', 0); + buffer.writeUInt32LE(36, 4); // Initial file size - 8 (will be updated later) + buffer.write('WAVE', 8); + + // fmt sub-chunk + buffer.write('fmt ', 12); + buffer.writeUInt32LE(16, 16); // Subchunk1Size (16 for PCM) + buffer.writeUInt16LE(3, 20); // AudioFormat (3 for IEEE float) + buffer.writeUInt16LE(this.numChannels, 22); // NumChannels + buffer.writeUInt32LE(this.targetSampleRate, 24); // SampleRate + buffer.writeUInt32LE(this.targetSampleRate * this.numChannels * 4, 28); // ByteRate + buffer.writeUInt16LE(this.numChannels * 4, 32); // BlockAlign + buffer.writeUInt16LE(32, 34); // BitsPerSample (32 for float) + + // data sub-chunk + buffer.write('data', 36); + buffer.writeUInt32LE(0, 40); // Initial data size (will be updated later) + + this.file.write(buffer); + } + + private resample(samples: Float32Array): Float32Array { + const ratio = this.originalSampleRate / this.targetSampleRate; + const newLength = Math.floor(samples.length / ratio); + const result = new Float32Array(newLength); + + for (let i = 0; i < newLength; i++) { + const position = i * ratio; + const index = Math.floor(position); + const fraction = position - index; + + // Linear interpolation between adjacent samples + if (index + 1 < samples.length) { + result[i] = + samples[index] * (1 - fraction) + samples[index + 1] * fraction; + } else { + result[i] = samples[index]; + } + } + + return result; + } + + write(samples: Float32Array) { + // Resample the input samples + const resampledData = this.resample(samples); + + // Create a buffer with the correct size (4 bytes per float) + const buffer = Buffer.alloc(resampledData.length * 4); + + // Write each float value properly + for (let i = 0; i < resampledData.length; i++) { + buffer.writeFloatLE(resampledData[i], i * 4); + } + + this.file.write(buffer); + this.samplesWritten += resampledData.length; + } + + async end(): Promise { + return new Promise((resolve, reject) => { + this.file.end(() => { + void this.updateHeaderAndCleanup().then(resolve).catch(reject); + }); + }); + } + + private async updateHeaderAndCleanup(): Promise { + // Read the entire temporary file + const data = await fs.promises.readFile(this.tempFilePath); + + // Update the header with correct sizes + const dataSize = this.samplesWritten * 4; + const fileSize = dataSize + 36; + + data.writeUInt32LE(fileSize, 4); // Update RIFF chunk size + data.writeUInt32LE(dataSize, 40); // Update data chunk size + + // Write the updated file + await fs.promises.writeFile(this.finalFilePath, data); + + // Clean up temp file + await fs.promises.unlink(this.tempFilePath); + } +} + +/** + * Creates a Buffer from Float32Array audio data + * @param float32Array - The Float32Array containing audio samples + * @returns FileData - The audio data as a Buffer + */ +export function FileData(float32Array: Float32Array): Buffer { + const buffer = Buffer.alloc(float32Array.length * 4); // 4 bytes per float + for (let i = 0; i < float32Array.length; i++) { + buffer.writeFloatLE(float32Array[i], i * 4); + } + return buffer; +} diff --git a/packages/frontend/media-capture-playground/tsconfg.node.json b/packages/frontend/media-capture-playground/tsconfg.node.json new file mode 100644 index 0000000000..ed58d9ef2d --- /dev/null +++ b/packages/frontend/media-capture-playground/tsconfg.node.json @@ -0,0 +1,7 @@ +{ + "extends": "../../../tsconfig.node.json", + "compilerOptions": { + "rootDir": "./server" + }, + "include": ["./server"] +} diff --git a/packages/frontend/media-capture-playground/tsconfig.json b/packages/frontend/media-capture-playground/tsconfig.json new file mode 100644 index 0000000000..41a46a0d40 --- /dev/null +++ b/packages/frontend/media-capture-playground/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../../tsconfig.web.json", + "compilerOptions": { + "rootDir": "./web", + "outDir": "./dist", + "tsBuildInfoFile": "./dist/tsconfig.tsbuildinfo" + }, + "include": ["./web", "server/types.d.ts"], + "references": [{ "path": "../native" }] +} diff --git a/packages/frontend/media-capture-playground/vite.config.ts b/packages/frontend/media-capture-playground/vite.config.ts new file mode 100644 index 0000000000..c7ab16ad12 --- /dev/null +++ b/packages/frontend/media-capture-playground/vite.config.ts @@ -0,0 +1,18 @@ +import tailwindcss from '@tailwindcss/vite'; +import react from '@vitejs/plugin-react'; +import { defineConfig } from 'vite'; + +// https://vite.dev/config/ +export default defineConfig({ + plugins: [react(), tailwindcss()], + root: './web', + server: { + proxy: { + '/api': { + target: 'http://localhost:6544', + changeOrigin: true, + rewrite: path => path.replace(/^\/api/, ''), + }, + }, + }, +}); diff --git a/packages/frontend/media-capture-playground/web/app.tsx b/packages/frontend/media-capture-playground/web/app.tsx new file mode 100644 index 0000000000..a3aaa7ff4d --- /dev/null +++ b/packages/frontend/media-capture-playground/web/app.tsx @@ -0,0 +1,33 @@ +import { AppList } from './components/app-list'; +import { SavedRecordings } from './components/saved-recordings'; + +export function App() { + return ( +
+
+
+

+ Running Applications +

+

+ Select an application to start recording its audio +

+
+ +
+
+
+

+ Saved Recordings +

+

+ Listen to and manage your recorded audio files +

+
+ +
+
+
+
+ ); +} diff --git a/packages/frontend/media-capture-playground/web/components/app-item.tsx b/packages/frontend/media-capture-playground/web/components/app-item.tsx new file mode 100644 index 0000000000..cde56c9778 --- /dev/null +++ b/packages/frontend/media-capture-playground/web/components/app-item.tsx @@ -0,0 +1,122 @@ +import React from 'react'; + +import type { AppGroup, RecordingStatus } from '../types'; +import { formatDuration } from '../utils'; + +interface AppItemProps { + app: AppGroup; + recordings?: RecordingStatus[]; +} + +export function AppItem({ app, recordings }: AppItemProps) { + const [imgError, setImgError] = React.useState(false); + const [isRecording, setIsRecording] = React.useState(false); + + const appName = app.rootApp.name || ''; + const bundleId = app.rootApp.bundleIdentifier || ''; + const firstLetter = appName.charAt(0).toUpperCase(); + const isRunning = app.apps.some(a => a.running); + + const recording = recordings?.find((r: RecordingStatus) => + app.apps.some(a => a.processId === r.processId) + ); + + const handleRecordClick = React.useCallback(() => { + const recordingApp = app.apps.find(a => a.running); + if (!recordingApp) { + return; + } + if (isRecording) { + void fetch(`/api/apps/${recordingApp.processId}/stop`, { + method: 'POST', + }) + .then(() => setIsRecording(false)) + .catch(error => console.error('Failed to stop recording:', error)); + } else { + void fetch(`/api/apps/${recordingApp.processId}/record`, { + method: 'POST', + }) + .then(() => setIsRecording(true)) + .catch(error => console.error('Failed to start recording:', error)); + } + }, [app.apps, isRecording]); + + React.useEffect(() => { + setIsRecording(!!recording); + }, [recording]); + + const [duration, setDuration] = React.useState(0); + + React.useEffect(() => { + if (recording) { + const interval = setInterval(() => { + setDuration(Date.now() - recording.startTime); + }, 1000); + return () => clearInterval(interval); + } else { + setDuration(0); + } + return () => {}; + }, [recording]); + + return ( +
+ {imgError ? ( +
+ {firstLetter} +
+ ) : ( + {appName} setImgError(true)} + /> + )} +
+
+ {appName ? ( + + {appName} + + ) : ( + + Unnamed Application + + )} + + PID: {app.rootApp.processId} + + + {recording ? formatDuration(duration) : '00:00:00'} + +
+
+ {bundleId} +
+
+ {(isRunning || isRecording) && ( + + )} +
+ ); +} diff --git a/packages/frontend/media-capture-playground/web/components/app-list.tsx b/packages/frontend/media-capture-playground/web/components/app-list.tsx new file mode 100644 index 0000000000..d5d3e91162 --- /dev/null +++ b/packages/frontend/media-capture-playground/web/components/app-list.tsx @@ -0,0 +1,144 @@ +import React from 'react'; +import useSWRSubscription from 'swr/subscription'; + +import type { App, AppGroup, RecordingStatus } from '../types'; +import { socket } from '../utils'; +import { AppItem } from './app-item'; + +export function AppList() { + const { data: apps = [] } = useSWRSubscription('apps', (_key, { next }) => { + let apps: App[] = []; + // Initial apps fetch + fetch('/api/apps') + .then(res => res.json()) + .then(data => { + apps = data.apps; + next(null, apps); + }) + .catch(err => next(err)); + + // Subscribe to app updates + socket.on('apps:all', data => { + next(null, data.apps); + apps = data.apps; + }); + socket.on('apps:state-changed', data => { + const index = apps.findIndex(a => a.processId === data.processId); + if (index !== -1) { + next( + null, + apps.toSpliced(index, 1, { + ...apps[index], + running: data.running, + }) + ); + } + }); + socket.on('connect', () => { + // Refetch on reconnect + fetch('/api/apps') + .then(res => res.json()) + .then(data => next(null, data.apps)) + .catch(err => next(err)); + }); + + return () => { + socket.off('apps:all'); + socket.off('apps:state-changed'); + socket.off('connect'); + }; + }); + + const { data: recordings = [] } = useSWRSubscription( + 'recordings', + ( + _key: string, + { next }: { next: (err: Error | null, data?: RecordingStatus[]) => void } + ) => { + // Subscribe to recording updates + socket.on('apps:recording', (data: { recordings: RecordingStatus[] }) => { + next(null, data.recordings); + }); + + return () => { + socket.off('apps:recording'); + }; + } + ); + + const appGroups: AppGroup[] = React.useMemo(() => { + const mapping = apps.reduce((acc: Record, app: App) => { + if (!acc[app.processGroupId]) { + acc[app.processGroupId] = { + processGroupId: app.processGroupId, + apps: [], + rootApp: + apps.find((a: App) => a.processId === app.processGroupId) || app, + }; + } + acc[app.processGroupId].apps.push(app); + return acc; + }, {}); + return Object.values(mapping); + }, [apps]); + + const runningApps = (appGroups || []).filter(app => + app.apps.some(a => a.running) + ); + const notRunningApps = (appGroups || []).filter( + app => !app.apps.some(a => a.running) + ); + + return ( +
+
+
+

+ Active Applications +

+ + {runningApps.length} listening + +
+
+ {runningApps.map(app => ( + + ))} + {runningApps.length === 0 && ( +
+ No applications are currently listening +
+ )} +
+
+
+
+

+ Other Applications +

+ + {notRunningApps.length} available + +
+
+ {notRunningApps.map(app => ( + + ))} + {notRunningApps.length === 0 && ( +
+ No other applications found +
+ )} +
+
+
+ ); +} diff --git a/packages/frontend/media-capture-playground/web/components/icons.tsx b/packages/frontend/media-capture-playground/web/components/icons.tsx new file mode 100644 index 0000000000..f7d359b581 --- /dev/null +++ b/packages/frontend/media-capture-playground/web/components/icons.tsx @@ -0,0 +1,163 @@ +import type { ReactElement } from 'react'; + +export function PlayIcon(): ReactElement { + return ( + + + + ); +} + +export function PauseIcon(): ReactElement { + return ( + + + + ); +} + +export function RewindIcon(): ReactElement { + return ( + + + + ); +} + +export function ForwardIcon(): ReactElement { + return ( + + + + ); +} + +export function DeleteIcon(): ReactElement { + return ( + + + + ); +} + +export function LoadingSpinner(): ReactElement { + return ( + + + + + ); +} + +export function ErrorIcon(): ReactElement { + return ( + + + + ); +} + +export function MicrophoneIcon(): ReactElement { + return ( + + + + ); +} + +export function WarningIcon(): ReactElement { + return ( + + + + ); +} + +export function DefaultAppIcon(): ReactElement { + return ( + + + + + ); +} diff --git a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx new file mode 100644 index 0000000000..41e52dda17 --- /dev/null +++ b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx @@ -0,0 +1,872 @@ +import type { ReactElement } from 'react'; +import React from 'react'; +import ReactMarkdown from 'react-markdown'; + +import type { SavedRecording, TranscriptionMetadata } from '../types'; +import { formatDuration, socket } from '../utils'; +import { + DefaultAppIcon, + DeleteIcon, + ErrorIcon, + ForwardIcon, + LoadingSpinner, + MicrophoneIcon, + PauseIcon, + PlayIcon, + RewindIcon, + WarningIcon, +} from './icons'; + +interface SavedRecordingItemProps { + recording: SavedRecording; +} + +// Audio player controls component +function AudioControls({ + audioRef, + playbackRate, + onPlaybackRateChange, + onSeek, + onPlayPause, +}: { + audioRef: React.RefObject; + playbackRate: number; + onPlaybackRateChange: () => void; + onSeek: (seconds: number) => void; + onPlayPause: () => void; +}): ReactElement { + const [currentTime, setCurrentTime] = React.useState('00:00'); + const [duration, setDuration] = React.useState('00:00'); + + React.useEffect(() => { + const audio = audioRef.current; + if (!audio) return; + + const formatTime = (time: number) => { + const minutes = Math.floor(time / 60); + const seconds = Math.floor(time % 60); + return `${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`; + }; + + const updateTime = () => { + setCurrentTime(formatTime(audio.currentTime)); + setDuration(formatTime(audio.duration)); + }; + + audio.addEventListener('timeupdate', updateTime); + audio.addEventListener('loadedmetadata', updateTime); + + return () => { + audio.removeEventListener('timeupdate', updateTime); + audio.removeEventListener('loadedmetadata', updateTime); + }; + }, [audioRef]); + + return ( +
+
+ + + +
+ {currentTime} / {duration} +
+
+ +
+ ); +} + +// Waveform visualization component +function WaveformVisualizer({ + containerRef, + waveformData, + currentTime, + fileName, +}: { + containerRef: React.RefObject; + waveformData: number[]; + currentTime: number; + fileName: string; +}): ReactElement { + return ( +
+
+ {waveformData.map((amplitude, i) => ( +
+ ))} +
+
+ ); +} + +// Update TranscriptionMessage component +function TranscriptionMessage({ + item, + isNewSpeaker, + isCurrentMessage, +}: { + item: { + speaker: string; + start_time: string; + transcription: string; + }; + isNewSpeaker: boolean; + isCurrentMessage: boolean; +}): ReactElement { + return ( +
+
+
+ {isNewSpeaker && ( +
+ {item.speaker} +
+ )} +
+ {item.start_time} +
+
+
+
+
+ {item.transcription} +
+
+
+ ); +} + +// Add new Summary component +function TranscriptionSummary({ summary }: { summary: string }): ReactElement { + return ( +
+
+ Summary +
+
+ {summary} +
+
+ ); +} + +// Update TranscriptionContent component +function TranscriptionContent({ + transcriptionData, + currentAudioTime, +}: { + transcriptionData: { + segments: Array<{ + speaker: string; + start_time: string; + transcription: string; + }>; + summary: string; + title: string; + }; + currentAudioTime: number; +}): ReactElement { + const parseTimestamp = (timestamp: string) => { + // Handle "MM:SS" format (without hours) + const [minutes, seconds] = timestamp.split(':'); + return parseInt(minutes, 10) * 60 + parseInt(seconds, 10); + }; + + return ( +
+ + {transcriptionData.segments.map((item, index) => { + const isNewSpeaker = + index === 0 || + transcriptionData.segments[index - 1].speaker !== item.speaker; + + const startTime = parseTimestamp(item.start_time); + // Use next segment's start time as end time, or add 3 seconds for the last segment + const endTime = + index < transcriptionData.segments.length - 1 + ? parseTimestamp(transcriptionData.segments[index + 1].start_time) + : startTime + 3; + + const isCurrentMessage = + currentAudioTime >= startTime && currentAudioTime < endTime; + + return ( + + ); + })} +
+ ); +} + +// Update TranscriptionStatus component +function TranscriptionStatus({ + transcription, + transcriptionError, + currentAudioTime, +}: { + transcription?: TranscriptionMetadata; + transcriptionError: string | null; + currentAudioTime: number; +}): ReactElement | null { + if (!transcription && !transcriptionError) { + return null; + } + + if (transcription?.transcriptionStatus === 'pending') { + return ( +
+
+
+ + Processing Audio +
+
+
+ +
+ Starting transcription + ... +
+
+ This may take a few moments depending on the length of the + recording +
+
+
+
+
+ ); + } + + if (transcriptionError) { + return ( +
+ + {transcriptionError} +
+ ); + } + + if ( + transcription?.transcriptionStatus === 'completed' && + transcription.transcription + ) { + try { + const transcriptionData = transcription.transcription; + if ( + !transcriptionData.segments || + !Array.isArray(transcriptionData.segments) + ) { + throw new Error('Invalid transcription data format'); + } + + return ( +
+
+
+ + Conversation Transcript +
+ {transcriptionData.title && ( +
+
+ Title +
+
+ {transcriptionData.title} +
+
+ )} + +
+
+ ); + } catch (error) { + return ( +
+ {error instanceof Error + ? error.message + : 'Failed to parse transcription data'} +
+ ); + } + } + + return null; +} + +// Add new RecordingHeader component +function RecordingHeader({ + metadata, + fileName, + recordingDate, + duration, + error, + isDeleting, + showDeleteConfirm, + setShowDeleteConfirm, + handleDeleteClick, +}: { + metadata: SavedRecording['metadata']; + fileName: string; + recordingDate: string; + duration: string; + error: string | null; + isDeleting: boolean; + showDeleteConfirm: boolean; + setShowDeleteConfirm: (show: boolean) => void; + handleDeleteClick: () => void; + transcriptionError: string | null; +}): ReactElement { + const [imgError, setImgError] = React.useState(false); + + return ( +
+
+ {!imgError ? ( + {metadata?.appName setImgError(true)} + /> + ) : ( +
+ +
+ )} +
+
+
+
+ + {metadata?.appName || 'Unknown Application'} + + + {duration} + +
+
+ {showDeleteConfirm ? ( +
+ + +
+ ) : ( + + )} +
+
+
{recordingDate}
+
+ {metadata?.bundleIdentifier || fileName} +
+ {error && ( +
+ + {error} +
+ )} +
+
+ ); +} + +// Add new AudioPlayer component +function AudioPlayer({ + isLoading, + error, + audioRef, + playbackRate, + handlePlaybackRateChange, + handleSeek, + handlePlayPause, + containerRef, + waveformData, + currentTime, + fileName, +}: { + isLoading: boolean; + error: string | null; + audioRef: React.RefObject; + playbackRate: number; + handlePlaybackRateChange: () => void; + handleSeek: (seconds: number) => void; + handlePlayPause: () => void; + containerRef: React.RefObject; + waveformData: number[]; + currentTime: number; + fileName: string; +}): ReactElement { + return ( +
+ {isLoading && !error ? ( +
+ + + Loading audio... + +
+ ) : ( +
+ + +
+ )} +
+ ); +} + +// Add new TranscribeButton component +function TranscribeButton({ + transcriptionStatus, + onTranscribe, +}: { + transcriptionStatus?: TranscriptionMetadata['transcriptionStatus']; + onTranscribe: () => void; +}): ReactElement { + return ( +
+
+ +
+
+ ); +} + +// Main SavedRecordingItem component (simplified) +export function SavedRecordingItem({ + recording, +}: SavedRecordingItemProps): ReactElement { + const [error, setError] = React.useState(null); + const [isLoading, setIsLoading] = React.useState(true); + const [isDeleting, setIsDeleting] = React.useState(false); + const [showDeleteConfirm, setShowDeleteConfirm] = React.useState(false); + const [playbackRate, setPlaybackRate] = React.useState(1); + const [waveformData, setWaveformData] = React.useState([]); + const [currentTime, setCurrentTime] = React.useState(0); + const audioRef = React.useRef(null); + const containerRef = React.useRef(null); + const [segments, setSegments] = React.useState(40); + const [currentAudioTime, setCurrentAudioTime] = React.useState(0); + const [transcriptionError, setTranscriptionError] = React.useState< + string | null + >(null); + + const metadata = recording.metadata; + const fileName = recording.wav; + const recordingDate = metadata + ? new Date(metadata.recordingStartTime).toLocaleString() + : 'Unknown date'; + const duration = metadata + ? formatDuration(metadata.recordingDuration * 1000) + : 'Unknown duration'; + + // Update current audio time + React.useEffect(() => { + const audio = audioRef.current; + if (audio) { + const handleTimeUpdate = () => { + setCurrentAudioTime(audio.currentTime); + }; + audio.addEventListener('timeupdate', handleTimeUpdate); + return () => audio.removeEventListener('timeupdate', handleTimeUpdate); + } + return () => {}; + }, []); + + // Calculate number of segments based on container width + React.useEffect(() => { + const updateSegments = () => { + if (containerRef.current) { + // Each bar should be at least 2px wide (1px bar + 1px gap) + const width = containerRef.current.offsetWidth; + setSegments(Math.floor(width / 2)); + } + }; + + updateSegments(); + const resizeObserver = new ResizeObserver(updateSegments); + if (containerRef.current) { + resizeObserver.observe(containerRef.current); + } + + return () => resizeObserver.disconnect(); + }, []); + + const processAudioData = React.useCallback(async () => { + try { + const response = await fetch(`/api/recordings/${fileName}/recording.wav`); + if (!response.ok) { + throw new Error( + `Failed to fetch audio file (${response.status}): ${response.statusText}` + ); + } + + const audioContext = new AudioContext(); + const arrayBuffer = await response.arrayBuffer(); + + // Ensure we have data to process + if (!arrayBuffer || arrayBuffer.byteLength === 0) { + throw new Error('No audio data received'); + } + + const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + const channelData = audioBuffer.getChannelData(0); + + // Process the audio data in chunks to create the waveform + const numberOfSamples = channelData.length; + const samplesPerSegment = Math.floor(numberOfSamples / segments); + + const waveform: number[] = []; + for (let i = 0; i < segments; i++) { + const start = i * samplesPerSegment; + const end = start + samplesPerSegment; + const segmentData = channelData.slice(start, end); + + // Calculate RMS (root mean square) for better amplitude representation + const rms = Math.sqrt( + segmentData.reduce((sum, sample) => sum + sample * sample, 0) / + segmentData.length + ); + + waveform.push(rms); + } + + // Normalize the waveform data to a 0-1 range + const maxAmplitude = Math.max(...waveform); + const normalizedWaveform = waveform.map(amp => amp / maxAmplitude); + + setWaveformData(normalizedWaveform); + setIsLoading(false); + } catch (err) { + console.error('Error processing audio:', err); + setError( + err instanceof Error ? err.message : 'Failed to process audio data' + ); + setIsLoading(false); + } + }, [fileName, segments]); + + React.useEffect(() => { + const audio = audioRef.current; + if (audio) { + const handleError = (e: ErrorEvent) => { + console.error('Audio error:', e); + setError('Failed to load audio'); + setIsLoading(false); + }; + + const handleLoadedMetadata = () => { + void processAudioData().catch(err => { + console.error('Error processing audio data:', err); + setError('Failed to process audio data'); + setIsLoading(false); + }); + }; + + const handleTimeUpdate = () => { + setCurrentTime(audio.currentTime / audio.duration); + }; + + audio.addEventListener('error', handleError as EventListener); + audio.addEventListener('loadedmetadata', handleLoadedMetadata); + audio.addEventListener('timeupdate', handleTimeUpdate); + + return () => { + audio.removeEventListener('error', handleError as EventListener); + audio.removeEventListener('loadedmetadata', handleLoadedMetadata); + audio.removeEventListener('timeupdate', handleTimeUpdate); + }; + } + return () => {}; + }, [processAudioData]); + + const handlePlayPause = React.useCallback(() => { + if (audioRef.current) { + if (audioRef.current.paused) { + void audioRef.current.play(); + } else { + audioRef.current.pause(); + } + } + }, []); + + const handleSeek = React.useCallback((seconds: number) => { + if (audioRef.current) { + audioRef.current.currentTime += seconds; + } + }, []); + + const handlePlaybackRateChange = React.useCallback(() => { + if (audioRef.current) { + const newRate = playbackRate === 1 ? 1.5 : 1; + audioRef.current.playbackRate = newRate; + setPlaybackRate(newRate); + } + }, [playbackRate]); + + const handleDelete = React.useCallback(async () => { + setIsDeleting(true); + setError(null); // Clear any previous errors + + try { + const response = await fetch(`/api/recordings/${recording.wav}`, { + method: 'DELETE', + }); + + if (!response.ok) { + let errorMessage: string; + try { + const errorData = await response.json(); + errorMessage = errorData.error; + } catch { + errorMessage = `Server error (${response.status}): ${response.statusText}`; + } + throw new Error(errorMessage); + } + + setShowDeleteConfirm(false); + } catch (err) { + console.error('Error deleting recording:', err); + setError( + err instanceof Error ? err.message : 'An unexpected error occurred' + ); + } finally { + setIsDeleting(false); + } + }, [recording.wav]); + + const handleDeleteClick = React.useCallback(() => { + void handleDelete().catch(err => { + console.error('Unexpected error during deletion:', err); + setError('An unexpected error occurred'); + }); + }, [handleDelete]); + + React.useEffect(() => { + // Listen for transcription events + socket.on( + 'apps:recording-transcription-start', + (data: { filename: string }) => { + if (data.filename === recording.wav) { + setTranscriptionError(null); + } + } + ); + + socket.on( + 'apps:recording-transcription-end', + (data: { + filename: string; + success: boolean; + transcription?: string; + error?: string; + }) => { + if (data.filename === recording.wav && !data.success) { + setTranscriptionError(data.error || 'Transcription failed'); + } + } + ); + + return () => { + socket.off('apps:recording-transcription-start'); + socket.off('apps:recording-transcription-end'); + }; + }, [recording.wav]); + + const handleTranscribe = React.useCallback(async () => { + try { + const response = await fetch( + `/api/recordings/${recording.wav}/transcribe`, + { + method: 'POST', + } + ); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || 'Failed to start transcription'); + } + } catch (err) { + setTranscriptionError( + err instanceof Error ? err.message : 'Failed to start transcription' + ); + } + }, [recording.wav]); + + return ( +
+ + } + playbackRate={playbackRate} + handlePlaybackRateChange={handlePlaybackRateChange} + handleSeek={handleSeek} + handlePlayPause={handlePlayPause} + containerRef={containerRef as React.RefObject} + waveformData={waveformData} + currentTime={currentTime} + fileName={fileName} + /> +
+ ); +} diff --git a/packages/frontend/media-capture-playground/web/components/saved-recordings.tsx b/packages/frontend/media-capture-playground/web/components/saved-recordings.tsx new file mode 100644 index 0000000000..2d6d25f184 --- /dev/null +++ b/packages/frontend/media-capture-playground/web/components/saved-recordings.tsx @@ -0,0 +1,41 @@ +import useSWRSubscription from 'swr/subscription'; + +import type { SavedRecording } from '../types'; +import { socket } from '../utils'; +import { SavedRecordingItem } from './saved-recording-item'; + +export function SavedRecordings(): React.ReactElement { + const { data: recordings = [] } = useSWRSubscription( + 'saved-recordings', + ( + _key: string, + { next }: { next: (err: Error | null, data?: SavedRecording[]) => void } + ) => { + // Subscribe to saved recordings updates + socket.on('apps:saved', (data: { recordings: SavedRecording[] }) => { + next(null, data.recordings); + }); + + fetch('/api/apps/saved') + .then(res => res.json()) + .then(data => next(null, data.recordings)) + .catch(err => next(err)); + + return () => { + socket.off('apps:saved'); + }; + } + ); + + if (recordings.length === 0) { + return

No saved recordings

; + } + + return ( +
+ {recordings.map(recording => ( + + ))} +
+ ); +} diff --git a/packages/frontend/media-capture-playground/web/index.html b/packages/frontend/media-capture-playground/web/index.html new file mode 100644 index 0000000000..b65f7dfec4 --- /dev/null +++ b/packages/frontend/media-capture-playground/web/index.html @@ -0,0 +1,13 @@ + + + + + + + Media Capture Playground + + +
+ + + diff --git a/packages/frontend/media-capture-playground/web/main.css b/packages/frontend/media-capture-playground/web/main.css new file mode 100644 index 0000000000..d4b5078586 --- /dev/null +++ b/packages/frontend/media-capture-playground/web/main.css @@ -0,0 +1 @@ +@import 'tailwindcss'; diff --git a/packages/frontend/media-capture-playground/web/main.tsx b/packages/frontend/media-capture-playground/web/main.tsx new file mode 100644 index 0000000000..df8a85c6a4 --- /dev/null +++ b/packages/frontend/media-capture-playground/web/main.tsx @@ -0,0 +1,11 @@ +import './main.css'; + +import { createRoot } from 'react-dom/client'; + +import { App } from './app'; + +const rootElement = document.getElementById('root'); +if (!rootElement) { + throw new Error('Failed to find the root element'); +} +createRoot(rootElement).render(); diff --git a/packages/frontend/media-capture-playground/web/types.ts b/packages/frontend/media-capture-playground/web/types.ts new file mode 100644 index 0000000000..e1ec56a4b0 --- /dev/null +++ b/packages/frontend/media-capture-playground/web/types.ts @@ -0,0 +1,55 @@ +export interface App { + processId: number; + processGroupId: number; + bundleIdentifier: string; + name: string; + running: boolean; +} + +export interface AppGroup { + processGroupId: number; + rootApp: App; + apps: App[]; +} + +export interface RecordingStatus { + processId: number; + bundleIdentifier: string; + name: string; + startTime: number; +} + +export interface RecordingMetadata { + appName: string; + bundleIdentifier: string; + processId: number; + recordingStartTime: number; + recordingEndTime: number; + recordingDuration: number; + sampleRate: number; + totalSamples: number; + icon?: Uint8Array; +} + +export interface TranscriptionMetadata { + transcriptionStartTime: number; + transcriptionEndTime: number; + transcriptionStatus: 'not_started' | 'pending' | 'completed' | 'error'; + transcription?: { + title: string; + segments: Array<{ + speaker: string; + start_time: string; + end_time: string; + transcription: string; + }>; + summary: string; + }; + error?: string; +} + +export interface SavedRecording { + wav: string; + metadata?: RecordingMetadata; + transcription?: TranscriptionMetadata; +} diff --git a/packages/frontend/media-capture-playground/web/utils.ts b/packages/frontend/media-capture-playground/web/utils.ts new file mode 100644 index 0000000000..61a924248f --- /dev/null +++ b/packages/frontend/media-capture-playground/web/utils.ts @@ -0,0 +1,19 @@ +import { io } from 'socket.io-client'; + +// Create a singleton socket instance +export const socket = io('http://localhost:6544'); + +export function formatDuration(ms: number): string { + const seconds = Math.floor(ms / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + return `${hours.toString().padStart(2, '0')}:${(minutes % 60) + .toString() + .padStart(2, '0')}:${(seconds % 60).toString().padStart(2, '0')}`; +} + +// Helper function to convert timestamp (MM:SS.mmm) to seconds +export function timestampToSeconds(timestamp: string): number { + const [minutes, seconds] = timestamp.split(':').map(parseFloat); + return minutes * 60 + seconds; +} diff --git a/packages/frontend/native/.gitignore b/packages/frontend/native/.gitignore index 7f927850c0..682bfaaae3 100644 --- a/packages/frontend/native/.gitignore +++ b/packages/frontend/native/.gitignore @@ -1,2 +1,3 @@ *.fixture lib +*.bin diff --git a/packages/frontend/native/Cargo.toml b/packages/frontend/native/Cargo.toml index f22a5e105a..1c5011a911 100644 --- a/packages/frontend/native/Cargo.toml +++ b/packages/frontend/native/Cargo.toml @@ -7,14 +7,15 @@ version = "0.0.0" crate-type = ["cdylib", "rlib"] [dependencies] -affine_common = { workspace = true } -affine_nbstore = { path = "./nbstore" } -affine_sqlite_v1 = { path = "./sqlite_v1" } -napi = { workspace = true } -napi-derive = { workspace = true } -once_cell = { workspace = true } -sqlx = { workspace = true, default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] } -tokio = { workspace = true, features = ["full"] } +affine_common = { workspace = true } +affine_media_capture = { path = "./media_capture" } +affine_nbstore = { path = "./nbstore" } +affine_sqlite_v1 = { path = "./sqlite_v1" } +napi = { workspace = true } +napi-derive = { workspace = true } +once_cell = { workspace = true } +sqlx = { workspace = true, default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] } +tokio = { workspace = true, features = ["full"] } [build-dependencies] napi-build = { workspace = true } diff --git a/packages/frontend/native/media-capture-exapmle.ts b/packages/frontend/native/media-capture-exapmle.ts new file mode 100644 index 0000000000..712d62bd3a --- /dev/null +++ b/packages/frontend/native/media-capture-exapmle.ts @@ -0,0 +1,149 @@ +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + Whisper, + WhisperFullParams, + WhisperSamplingStrategy, +} from '@napi-rs/whisper'; +import { BehaviorSubject, EMPTY, Observable } from 'rxjs'; +import { + distinctUntilChanged, + exhaustMap, + groupBy, + mergeMap, + switchMap, + tap, +} from 'rxjs/operators'; + +import { type Application, ShareableContent } from './index.js'; + +const rootDir = join(fileURLToPath(import.meta.url), '..'); + +const shareableContent = new ShareableContent(); + +const appList = new Set([ + 'com.tinyspeck.slackmacgap.helper', + 'us.zoom.xos', + 'org.mozilla.firefoxdeveloperedition', +]); + +console.info(shareableContent.applications().map(app => app.bundleIdentifier)); + +const GGLM_LARGE = join(rootDir, 'ggml-large-v3-turbo.bin'); + +const whisper = new Whisper(GGLM_LARGE, { + useGpu: true, + gpuDevice: 1, +}); + +const whisperParams = new WhisperFullParams(WhisperSamplingStrategy.Greedy); + +const SAMPLE_WINDOW_MS = 3000; // 3 seconds, similar to stream.cpp's step_ms +const SAMPLES_PER_WINDOW = (SAMPLE_WINDOW_MS / 1000) * 16000; // 16kHz sample rate + +// eslint-disable-next-line rxjs/finnish +const runningApplications = new BehaviorSubject( + shareableContent.applications() +); + +const applicationListChangedSubscriber = + ShareableContent.onApplicationListChanged(() => { + runningApplications.next(shareableContent.applications()); + }); + +runningApplications + .pipe( + mergeMap(apps => apps.filter(app => appList.has(app.bundleIdentifier))), + groupBy(app => app.bundleIdentifier), + mergeMap(app$ => + app$.pipe( + exhaustMap(app => + new Observable<[Application, boolean]>(subscriber => { + const stateSubscriber = ShareableContent.onAppStateChanged( + app, + err => { + if (err) { + subscriber.error(err); + return; + } + subscriber.next([app, app.isRunning]); + } + ); + return () => { + stateSubscriber.unsubscribe(); + }; + }).pipe( + distinctUntilChanged( + ([_, isRunningA], [__, isRunningB]) => isRunningA === isRunningB + ), + switchMap(([app]) => + !app.isRunning + ? EMPTY + : new Observable(observer => { + const buffers: Float32Array[] = []; + const audioStream = app.tapAudio((err, samples) => { + if (err) { + observer.error(err); + return; + } + + if (samples) { + buffers.push(samples); + observer.next(samples); + + // Calculate total samples in buffer + const totalSamples = buffers.reduce( + (acc, buf) => acc + buf.length, + 0 + ); + + // Process when we have enough samples for our window + if (totalSamples >= SAMPLES_PER_WINDOW) { + // Concatenate all buffers + const concatenated = new Float32Array(totalSamples); + let offset = 0; + buffers.forEach(buf => { + concatenated.set(buf, offset); + offset += buf.length; + }); + + // Transcribe the audio + const result = whisper.full( + whisperParams, + concatenated + ); + + // Print results + console.info(result); + + // Keep any remaining samples for next window + const remainingSamples = + totalSamples - SAMPLES_PER_WINDOW; + if (remainingSamples > 0) { + const lastBuffer = buffers[buffers.length - 1]; + buffers.length = 0; + buffers.push(lastBuffer.slice(-remainingSamples)); + } else { + buffers.length = 0; + } + } + } + }); + + return () => { + audioStream.stop(); + }; + }) + ) + ) + ) + ) + ), + tap({ + finalize: () => { + applicationListChangedSubscriber.unsubscribe(); + }, + }) + ) + .subscribe(); diff --git a/packages/frontend/native/media_capture/Cargo.toml b/packages/frontend/native/media_capture/Cargo.toml new file mode 100644 index 0000000000..222cd65882 --- /dev/null +++ b/packages/frontend/native/media_capture/Cargo.toml @@ -0,0 +1,26 @@ +[package] +edition = "2021" +name = "affine_media_capture" +version = "0.0.0" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +napi = { workspace = true, features = ["napi4"] } +napi-derive = { workspace = true, features = ["type-def"] } +rubato = { workspace = true } + +[target.'cfg(target_os = "macos")'.dependencies] +block2 = { workspace = true } +core-foundation = { workspace = true, features = ["with-uuid"] } +coreaudio-rs = { workspace = true } +dispatch2 = { workspace = true } +objc2 = { workspace = true } +objc2-foundation = { workspace = true } +screencapturekit = { workspace = true } +thiserror = { workspace = true } +uuid = { workspace = true, features = ["v4"] } + +[build-dependencies] +napi-build = { workspace = true } diff --git a/packages/frontend/native/media_capture/build.rs b/packages/frontend/native/media_capture/build.rs new file mode 100644 index 0000000000..bbfc9e4b9e --- /dev/null +++ b/packages/frontend/native/media_capture/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/packages/frontend/native/media_capture/src/lib.rs b/packages/frontend/native/media_capture/src/lib.rs new file mode 100644 index 0000000000..a429d055b6 --- /dev/null +++ b/packages/frontend/native/media_capture/src/lib.rs @@ -0,0 +1,4 @@ +#[cfg(target_os = "macos")] +pub mod macos; +#[cfg(target_os = "macos")] +pub(crate) use macos::*; diff --git a/packages/frontend/native/media_capture/src/macos/audio_stream_basic_desc.rs b/packages/frontend/native/media_capture/src/macos/audio_stream_basic_desc.rs new file mode 100644 index 0000000000..2688154a63 --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/audio_stream_basic_desc.rs @@ -0,0 +1,282 @@ +use std::{fmt::Display, mem, ptr}; + +use coreaudio::sys::{ + kAudioHardwareNoError, kAudioObjectPropertyElementMain, kAudioObjectPropertyScopeGlobal, + kAudioTapPropertyFormat, AudioObjectGetPropertyData, AudioObjectID, AudioObjectPropertyAddress, +}; +use objc2::{Encode, Encoding, RefEncode}; + +use crate::error::CoreAudioError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u32)] +pub enum AudioFormatID { + LinearPcm = 0x6C70636D, // 'lpcm' + Ac3 = 0x61632D33, // 'ac-3' + Ac360958 = 0x63616333, // 'cac3' + AppleIma4 = 0x696D6134, // 'ima4' + Mpeg4Aac = 0x61616320, // 'aac ' + Mpeg4Celp = 0x63656C70, // 'celp' + Mpeg4Hvxc = 0x68767863, // 'hvxc' + Mpeg4TwinVq = 0x74777671, // 'twvq' + Mace3 = 0x4D414333, // 'MAC3' + Mace6 = 0x4D414336, // 'MAC6' + ULaw = 0x756C6177, // 'ulaw' + ALaw = 0x616C6177, // 'alaw' + QDesign = 0x51444D43, // 'QDMC' + QDesign2 = 0x51444D32, // 'QDM2' + Qualcomm = 0x51636C70, // 'Qclp' + MpegLayer1 = 0x2E6D7031, // '.mp1' + MpegLayer2 = 0x2E6D7032, // '.mp2' + MpegLayer3 = 0x2E6D7033, // '.mp3' + TimeCode = 0x74696D65, // 'time' + MidiStream = 0x6D696469, // 'midi' + ParameterValueStream = 0x61707673, // 'apvs' + AppleLossless = 0x616C6163, // 'alac' + Mpeg4AacHe = 0x61616368, // 'aach' + Mpeg4AacLd = 0x6161636C, // 'aacl' + Mpeg4AacEld = 0x61616365, // 'aace' + Mpeg4AacEldSbr = 0x61616366, // 'aacf' + Mpeg4AacEldV2 = 0x61616367, // 'aacg' + Mpeg4AacHeV2 = 0x61616370, // 'aacp' + Mpeg4AacSpatial = 0x61616373, // 'aacs' + MpegdUsac = 0x75736163, // 'usac' + Amr = 0x73616D72, // 'samr' + AmrWb = 0x73617762, // 'sawb' + Audible = 0x41554442, // 'AUDB' + ILbc = 0x696C6263, // 'ilbc' + DviIntelIma = 0x6D730011, + MicrosoftGsm = 0x6D730031, + Aes3 = 0x61657333, // 'aes3' + EnhancedAc3 = 0x65632D33, // 'ec-3' + Flac = 0x666C6163, // 'flac' + Opus = 0x6F707573, // 'opus' + Apac = 0x61706163, // 'apac' + Unknown = 0x00000000, +} + +impl From for AudioFormatID { + fn from(value: u32) -> Self { + match value { + 0x6C70636D => Self::LinearPcm, + 0x61632D33 => Self::Ac3, + 0x63616333 => Self::Ac360958, + 0x696D6134 => Self::AppleIma4, + 0x61616320 => Self::Mpeg4Aac, + 0x63656C70 => Self::Mpeg4Celp, + 0x68767863 => Self::Mpeg4Hvxc, + 0x74777671 => Self::Mpeg4TwinVq, + 0x4D414333 => Self::Mace3, + 0x4D414336 => Self::Mace6, + 0x756C6177 => Self::ULaw, + 0x616C6177 => Self::ALaw, + 0x51444D43 => Self::QDesign, + 0x51444D32 => Self::QDesign2, + 0x51636C70 => Self::Qualcomm, + 0x2E6D7031 => Self::MpegLayer1, + 0x2E6D7032 => Self::MpegLayer2, + 0x2E6D7033 => Self::MpegLayer3, + 0x74696D65 => Self::TimeCode, + 0x6D696469 => Self::MidiStream, + 0x61707673 => Self::ParameterValueStream, + 0x616C6163 => Self::AppleLossless, + 0x61616368 => Self::Mpeg4AacHe, + 0x6161636C => Self::Mpeg4AacLd, + 0x61616365 => Self::Mpeg4AacEld, + 0x61616366 => Self::Mpeg4AacEldSbr, + 0x61616367 => Self::Mpeg4AacEldV2, + 0x61616370 => Self::Mpeg4AacHeV2, + 0x61616373 => Self::Mpeg4AacSpatial, + 0x75736163 => Self::MpegdUsac, + 0x73616D72 => Self::Amr, + 0x73617762 => Self::AmrWb, + 0x41554442 => Self::Audible, + 0x696C6263 => Self::ILbc, + 0x6D730011 => Self::DviIntelIma, + 0x6D730031 => Self::MicrosoftGsm, + 0x61657333 => Self::Aes3, + 0x65632D33 => Self::EnhancedAc3, + 0x666C6163 => Self::Flac, + 0x6F707573 => Self::Opus, + 0x61706163 => Self::Apac, + _ => Self::Unknown, + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct AudioFormatFlags(pub u32); + +#[allow(unused)] +impl AudioFormatFlags { + pub const IS_FLOAT: u32 = 1 << 0; + pub const IS_BIG_ENDIAN: u32 = 1 << 1; + pub const IS_SIGNED_INTEGER: u32 = 1 << 2; + pub const IS_PACKED: u32 = 1 << 3; + pub const IS_ALIGNED_HIGH: u32 = 1 << 4; + pub const IS_NON_INTERLEAVED: u32 = 1 << 5; + pub const IS_NON_MIXABLE: u32 = 1 << 6; + pub const ARE_ALL_CLEAR: u32 = 0x80000000; + + pub const LINEAR_PCM_IS_FLOAT: u32 = Self::IS_FLOAT; + pub const LINEAR_PCM_IS_BIG_ENDIAN: u32 = Self::IS_BIG_ENDIAN; + pub const LINEAR_PCM_IS_SIGNED_INTEGER: u32 = Self::IS_SIGNED_INTEGER; + pub const LINEAR_PCM_IS_PACKED: u32 = Self::IS_PACKED; + pub const LINEAR_PCM_IS_ALIGNED_HIGH: u32 = Self::IS_ALIGNED_HIGH; + pub const LINEAR_PCM_IS_NON_INTERLEAVED: u32 = Self::IS_NON_INTERLEAVED; + pub const LINEAR_PCM_IS_NON_MIXABLE: u32 = Self::IS_NON_MIXABLE; + pub const LINEAR_PCM_SAMPLE_FRACTION_SHIFT: u32 = 7; + pub const LINEAR_PCM_SAMPLE_FRACTION_MASK: u32 = 0x3F << Self::LINEAR_PCM_SAMPLE_FRACTION_SHIFT; + pub const LINEAR_PCM_ARE_ALL_CLEAR: u32 = Self::ARE_ALL_CLEAR; + + pub const APPLE_LOSSLESS_FORMAT_FLAG_16_BIT_SOURCE_DATA: u32 = 1; + pub const APPLE_LOSSLESS_FORMAT_FLAG_20_BIT_SOURCE_DATA: u32 = 2; + pub const APPLE_LOSSLESS_FORMAT_FLAG_24_BIT_SOURCE_DATA: u32 = 3; + pub const APPLE_LOSSLESS_FORMAT_FLAG_32_BIT_SOURCE_DATA: u32 = 4; +} + +impl std::fmt::Display for AudioFormatFlags { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut flags = Vec::new(); + + if self.0 & Self::IS_FLOAT != 0 { + flags.push("FLOAT"); + } + if self.0 & Self::IS_BIG_ENDIAN != 0 { + flags.push("BIG_ENDIAN"); + } + if self.0 & Self::IS_SIGNED_INTEGER != 0 { + flags.push("SIGNED_INTEGER"); + } + if self.0 & Self::IS_PACKED != 0 { + flags.push("PACKED"); + } + if self.0 & Self::IS_ALIGNED_HIGH != 0 { + flags.push("ALIGNED_HIGH"); + } + if self.0 & Self::IS_NON_INTERLEAVED != 0 { + flags.push("NON_INTERLEAVED"); + } + if self.0 & Self::IS_NON_MIXABLE != 0 { + flags.push("NON_MIXABLE"); + } + if self.0 & Self::ARE_ALL_CLEAR != 0 { + flags.push("ALL_CLEAR"); + } + + if flags.is_empty() { + write!(f, "NONE") + } else { + write!(f, "{}", flags.join(" | ")) + } + } +} + +impl std::fmt::Debug for AudioFormatFlags { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "AudioFormatFlags({})", self) + } +} + +impl From for AudioFormatFlags { + fn from(value: u32) -> Self { + Self(value) + } +} + +/// [Apple's documentation](https://developer.apple.com/documentation/coreaudiotypes/audiostreambasicdescription?language=objc) +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +#[allow(non_snake_case)] +pub struct AudioStreamBasicDescription { + pub mSampleRate: f64, + pub mFormatID: u32, + pub mFormatFlags: u32, + pub mBytesPerPacket: u32, + pub mFramesPerPacket: u32, + pub mBytesPerFrame: u32, + pub mChannelsPerFrame: u32, + pub mBitsPerChannel: u32, + pub mReserved: u32, +} + +unsafe impl Encode for AudioStreamBasicDescription { + const ENCODING: Encoding = Encoding::Struct( + "AudioStreamBasicDescription", + &[ + ::ENCODING, + ::ENCODING, + ::ENCODING, + ::ENCODING, + ::ENCODING, + ::ENCODING, + ::ENCODING, + ::ENCODING, + ::ENCODING, + ], + ); +} + +unsafe impl RefEncode for AudioStreamBasicDescription { + const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING); +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct AudioStreamDescription(pub(crate) AudioStreamBasicDescription); + +impl Display for AudioStreamDescription { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "AudioStreamBasicDescription {{ mSampleRate: {}, mFormatID: {:?}, mFormatFlags: {}, \ + mBytesPerPacket: {}, mFramesPerPacket: {}, mBytesPerFrame: {}, mChannelsPerFrame: {}, \ + mBitsPerChannel: {}, mReserved: {} }}", + self.0.mSampleRate, + AudioFormatID::from(self.0.mFormatID), + AudioFormatFlags(self.0.mFormatFlags), + self.0.mBytesPerPacket, + self.0.mFramesPerPacket, + self.0.mBytesPerFrame, + self.0.mChannelsPerFrame, + self.0.mBitsPerChannel, + self.0.mReserved + ) + } +} + +pub fn read_audio_stream_basic_description( + tap_id: AudioObjectID, +) -> std::result::Result { + let mut data_size = mem::size_of::(); + let address = AudioObjectPropertyAddress { + mSelector: kAudioTapPropertyFormat, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }; + let mut data = AudioStreamBasicDescription { + mSampleRate: 0.0, + mFormatID: 0, + mFormatFlags: 0, + mBytesPerPacket: 0, + mFramesPerPacket: 0, + mBytesPerFrame: 0, + mChannelsPerFrame: 0, + mBitsPerChannel: 0, + mReserved: 0, + }; + let status = unsafe { + AudioObjectGetPropertyData( + tap_id, + &address, + 0, + ptr::null_mut(), + (&mut data_size as *mut usize).cast(), + (&mut data as *mut AudioStreamBasicDescription).cast(), + ) + }; + if status != kAudioHardwareNoError as i32 { + return Err(CoreAudioError::GetAudioStreamBasicDescriptionFailed(status)); + } + Ok(AudioStreamDescription(data)) +} diff --git a/packages/frontend/native/media_capture/src/macos/av_audio_file.rs b/packages/frontend/native/media_capture/src/macos/av_audio_file.rs new file mode 100644 index 0000000000..3f6bdc0502 --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/av_audio_file.rs @@ -0,0 +1,71 @@ +use std::ptr; + +use objc2::{ + msg_send, + runtime::{AnyClass, AnyObject}, + AllocAnyThread, +}; +use objc2_foundation::{NSDictionary, NSError, NSNumber, NSString, NSUInteger, NSURL}; + +use crate::{ + av_audio_format::AVAudioFormat, av_audio_pcm_buffer::AVAudioPCMBuffer, error::CoreAudioError, +}; + +#[allow(unused)] +pub(crate) struct AVAudioFile { + inner: *mut AnyObject, +} + +#[allow(unused)] +impl AVAudioFile { + pub(crate) fn new(url: &str, format: &AVAudioFormat) -> Result { + let cls = AnyClass::get(c"AVAudioFile").ok_or(CoreAudioError::AVAudioFileClassNotFound)?; + let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] }; + if obj.is_null() { + return Err(CoreAudioError::AllocAVAudioFileFailed); + } + let url: &NSURL = &*unsafe { NSURL::fileURLWithPath(&NSString::from_str(url)) }; + let settings = &*NSDictionary::from_retained_objects( + &[ + &*NSString::from_str("AVFormatIDKey"), + &*NSString::from_str("AVSampleRateKey"), + &*NSString::from_str("AVNumberOfChannelsKey"), + ], + &[ + NSNumber::initWithUnsignedInt( + NSNumber::alloc(), + format.audio_stream_basic_description.0.mFormatID, + ), + NSNumber::initWithDouble(NSNumber::alloc(), format.get_sample_rate()), + NSNumber::initWithUnsignedInt(NSNumber::alloc(), format.get_channel_count()), + ], + ); + let is_interleaved = format.is_interleaved(); + let mut error: *mut NSError = ptr::null_mut(); + let common_format: NSUInteger = 1; + let obj: *mut AnyObject = unsafe { + msg_send![ + obj, + initForWriting: url, + settings: settings, + commonFormat: common_format, + interleaved: is_interleaved, + error: &mut error + ] + }; + if obj.is_null() { + return Err(CoreAudioError::InitAVAudioFileFailed); + } + Ok(Self { inner: obj }) + } + + pub(crate) fn write(&self, buffer: AVAudioPCMBuffer) -> Result<(), CoreAudioError> { + let mut error: *mut NSError = ptr::null_mut(); + let success: bool = + unsafe { msg_send![self.inner, writeFromBuffer: buffer.inner, error: &mut error] }; + if !success { + return Err(CoreAudioError::WriteAVAudioFileFailed); + } + Ok(()) + } +} diff --git a/packages/frontend/native/media_capture/src/macos/av_audio_format.rs b/packages/frontend/native/media_capture/src/macos/av_audio_format.rs new file mode 100644 index 0000000000..1a126fff05 --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/av_audio_format.rs @@ -0,0 +1,95 @@ +use objc2::{ + msg_send, + runtime::{AnyClass, AnyObject}, + Encode, Encoding, RefEncode, +}; + +use crate::{audio_stream_basic_desc::AudioStreamDescription, error::CoreAudioError}; + +#[derive(Debug)] +#[allow(unused)] +pub(crate) struct AVAudioFormat { + pub(crate) inner: AVAudioFormatRef, + pub(crate) audio_stream_basic_description: AudioStreamDescription, +} + +#[repr(transparent)] +#[derive(Debug, Clone, Copy)] +pub(crate) struct AVAudioFormatRef(pub(crate) *mut AnyObject); + +unsafe impl Encode for AVAudioFormatRef { + const ENCODING: Encoding = Encoding::Struct( + "AVAudioFormat", + &[ + Encoding::Double, + Encoding::UInt, + Encoding::Pointer(&Encoding::Struct( + "AVAudioChannelLayout", + &[ + Encoding::UInt, + Encoding::UInt, + Encoding::Pointer(&Encoding::Struct( + "AudioChannelLayout", + &[ + Encoding::UInt, + Encoding::UInt, + Encoding::Array( + 1, + &Encoding::Struct( + "AudioChannelDescription", + &[ + Encoding::UInt, + Encoding::UInt, + Encoding::Array(3, &Encoding::Float), + ], + ), + ), + Encoding::UInt, + Encoding::UInt, + ], + )), + Encoding::UInt, + ], + )), + Encoding::Pointer(&Encoding::Object), + ], + ); +} + +unsafe impl RefEncode for AVAudioFormatRef { + const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING); +} + +#[allow(unused)] +impl AVAudioFormat { + pub fn new( + audio_stream_basic_description: AudioStreamDescription, + ) -> Result { + let cls = AnyClass::get(c"AVAudioFormat").ok_or(CoreAudioError::AVAudioFormatClassNotFound)?; + let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] }; + if obj.is_null() { + return Err(CoreAudioError::AllocAVAudioFormatFailed); + } + let obj: *mut AnyObject = + unsafe { msg_send![obj, initWithStreamDescription: &audio_stream_basic_description.0] }; + if obj.is_null() { + return Err(CoreAudioError::InitAVAudioFormatFailed); + } + Ok(Self { + inner: AVAudioFormatRef(obj), + audio_stream_basic_description, + }) + } + + pub(crate) fn get_sample_rate(&self) -> f64 { + unsafe { msg_send![self.inner.0, sampleRate] } + } + + pub(crate) fn get_channel_count(&self) -> u32 { + unsafe { msg_send![self.inner.0, channelCount] } + } + + pub(crate) fn is_interleaved(&self) -> bool { + unsafe { msg_send![self.inner.0, isInterleaved] } + } +} diff --git a/packages/frontend/native/media_capture/src/macos/av_audio_pcm_buffer.rs b/packages/frontend/native/media_capture/src/macos/av_audio_pcm_buffer.rs new file mode 100644 index 0000000000..9d36117bb4 --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/av_audio_pcm_buffer.rs @@ -0,0 +1,35 @@ +use block2::RcBlock; +use objc2::{ + msg_send, + runtime::{AnyClass, AnyObject}, +}; + +use crate::{av_audio_format::AVAudioFormat, error::CoreAudioError, tap_audio::AudioBufferList}; + +#[allow(unused)] +pub(crate) struct AVAudioPCMBuffer { + pub(crate) inner: *mut AnyObject, +} + +#[allow(unused)] +impl AVAudioPCMBuffer { + pub(crate) fn new( + audio_format: &AVAudioFormat, + buffer_list: *const AudioBufferList, + ) -> Result { + let cls = + AnyClass::get(c"AVAudioPCMBuffer").ok_or(CoreAudioError::AVAudioPCMBufferClassNotFound)?; + let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] }; + if obj.is_null() { + return Err(CoreAudioError::AllocAVAudioPCMBufferFailed); + } + let deallocator = RcBlock::new(|_buffer_list: *const AudioBufferList| {}); + let obj: *mut AnyObject = unsafe { + msg_send![obj, initWithPCMFormat: audio_format.inner.0, bufferListNoCopy: buffer_list, deallocator: &*deallocator] + }; + if obj.is_null() { + return Err(CoreAudioError::InitAVAudioPCMBufferFailed); + } + Ok(Self { inner: obj }) + } +} diff --git a/packages/frontend/native/media_capture/src/macos/ca_tap_description.rs b/packages/frontend/native/media_capture/src/macos/ca_tap_description.rs new file mode 100644 index 0000000000..38b28bd425 --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/ca_tap_description.rs @@ -0,0 +1,84 @@ +use core_foundation::{ + base::{FromVoid, ItemRef}, + string::CFString, +}; +use coreaudio::sys::AudioObjectID; +use objc2::{ + msg_send, + runtime::{AnyClass, AnyObject}, + AllocAnyThread, +}; +use objc2_foundation::{NSArray, NSNumber, NSString, NSUUID}; + +use crate::error::CoreAudioError; + +pub(crate) struct CATapDescription { + pub(crate) inner: *mut AnyObject, +} + +impl CATapDescription { + pub fn init_stereo_mixdown_of_processes( + process: AudioObjectID, + ) -> std::result::Result { + let cls = + AnyClass::get(c"CATapDescription").ok_or(CoreAudioError::CATapDescriptionClassNotFound)?; + let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] }; + if obj.is_null() { + return Err(CoreAudioError::AllocCATapDescriptionFailed); + } + let processes_array = + NSArray::from_retained_slice(&[NSNumber::initWithUnsignedInt(NSNumber::alloc(), process)]); + let obj: *mut AnyObject = + unsafe { msg_send![obj, initStereoMixdownOfProcesses: &*processes_array] }; + if obj.is_null() { + return Err(CoreAudioError::InitStereoMixdownOfProcessesFailed); + } + + Ok(Self { inner: obj }) + } + + pub fn init_stereo_global_tap_but_exclude_processes( + processes: &[AudioObjectID], + ) -> std::result::Result { + let cls = + AnyClass::get(c"CATapDescription").ok_or(CoreAudioError::CATapDescriptionClassNotFound)?; + let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] }; + if obj.is_null() { + return Err(CoreAudioError::AllocCATapDescriptionFailed); + } + let processes_array = NSArray::from_retained_slice( + processes + .iter() + .map(|p| NSNumber::initWithUnsignedInt(NSNumber::alloc(), *p)) + .collect::>() + .as_slice(), + ); + let obj: *mut AnyObject = + unsafe { msg_send![obj, initStereoMixdownOfProcesses: &*processes_array] }; + if obj.is_null() { + return Err(CoreAudioError::InitStereoMixdownOfProcessesFailed); + } + + Ok(Self { inner: obj }) + } + + pub fn get_uuid(&self) -> std::result::Result, CoreAudioError> { + let uuid: *mut NSUUID = unsafe { msg_send![self.inner, UUID] }; + if uuid.is_null() { + return Err(CoreAudioError::GetCATapDescriptionUUIDFailed); + } + let uuid_string: *mut NSString = unsafe { msg_send![uuid, UUIDString] }; + if uuid_string.is_null() { + return Err(CoreAudioError::ConvertUUIDToCFStringFailed); + } + Ok(unsafe { CFString::from_void(uuid_string.cast()) }) + } +} + +impl Drop for CATapDescription { + fn drop(&mut self) { + unsafe { + let _: () = msg_send![self.inner, release]; + } + } +} diff --git a/packages/frontend/native/media_capture/src/macos/device.rs b/packages/frontend/native/media_capture/src/macos/device.rs new file mode 100644 index 0000000000..b5af983934 --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/device.rs @@ -0,0 +1,66 @@ +use std::{mem, ptr}; + +use core_foundation::{base::TCFType, string::CFString}; +use coreaudio::sys::{ + kAudioDevicePropertyDeviceUID, kAudioHardwareNoError, kAudioObjectPropertyElementMain, + kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, AudioDeviceID, + AudioObjectGetPropertyData, AudioObjectID, AudioObjectPropertyAddress, CFStringRef, +}; + +use crate::error::CoreAudioError; + +pub(crate) fn get_device_uid( + device_id: AudioDeviceID, +) -> std::result::Result { + let system_output_id = get_device_audio_id(device_id)?; + let address = AudioObjectPropertyAddress { + mSelector: kAudioDevicePropertyDeviceUID, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }; + + let mut output_uid: CFStringRef = ptr::null_mut(); + let mut data_size = mem::size_of::(); + let status = unsafe { + AudioObjectGetPropertyData( + system_output_id, + &address, + 0, + ptr::null_mut(), + (&mut data_size as *mut usize).cast(), + (&mut output_uid as *mut CFStringRef).cast(), + ) + }; + + if status != 0 { + return Err(CoreAudioError::GetDeviceUidFailed(status)); + } + Ok(unsafe { CFString::wrap_under_create_rule(output_uid.cast()) }) +} + +pub(crate) fn get_device_audio_id( + device_id: AudioDeviceID, +) -> std::result::Result { + let mut system_output_id: AudioObjectID = 0; + let mut data_size = mem::size_of::(); + + let address = AudioObjectPropertyAddress { + mSelector: device_id, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }; + let status = unsafe { + AudioObjectGetPropertyData( + kAudioObjectSystemObject, + &address, + 0, + ptr::null_mut(), + (&mut data_size as *mut usize).cast(), + (&mut system_output_id as *mut AudioObjectID).cast(), + ) + }; + if status != kAudioHardwareNoError as i32 { + return Err(CoreAudioError::GetDefaultDeviceFailed(status)); + } + Ok(system_output_id) +} diff --git a/packages/frontend/native/media_capture/src/macos/error.rs b/packages/frontend/native/media_capture/src/macos/error.rs new file mode 100644 index 0000000000..314e4ab11f --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/error.rs @@ -0,0 +1,81 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum CoreAudioError { + #[error("Map pid {0} to AudioObjectID failed")] + PidNotFound(i32), + #[error("Create process tap failed, status: {0}")] + CreateProcessTapFailed(i32), + #[error("Get default device failed, status: {0}")] + GetDefaultDeviceFailed(i32), + #[error("Get device uid failed, status: {0}")] + GetDeviceUidFailed(i32), + #[error("Create aggregate device failed, status: {0}")] + CreateAggregateDeviceFailed(i32), + #[error("Get process object list size failed, status: {0}")] + GetProcessObjectListSizeFailed(i32), + #[error("Get process object list failed, status: {0}")] + GetProcessObjectListFailed(i32), + #[error("AudioObjectGetPropertyDataSize failed, status: {0}")] + AudioObjectGetPropertyDataSizeFailed(i32), + #[error("CATapDescription class not found")] + CATapDescriptionClassNotFound, + #[error("Alloc CATapDescription failed")] + AllocCATapDescriptionFailed, + #[error("Call initStereoMixdownOfProcesses on CATapDescription failed")] + InitStereoMixdownOfProcessesFailed, + #[error("Get UUID on CATapDescription failed")] + GetCATapDescriptionUUIDFailed, + #[error("Get mute behavior on CATapDescription failed")] + GetMuteBehaviorFailed, + #[error("Convert UUID to CFString failed")] + ConvertUUIDToCFStringFailed, + #[error("Get AudioStreamBasicDescription failed, status: {0}")] + GetAudioStreamBasicDescriptionFailed(i32), + #[error("AVAudioFormat class not found")] + AVAudioFormatClassNotFound, + #[error("Alloc AVAudioFormat failed")] + AllocAVAudioFormatFailed, + #[error("Init AVAudioFormat failed")] + InitAVAudioFormatFailed, + #[error("Create IOProcIDWithBlock failed, status: {0}")] + CreateIOProcIDWithBlockFailed(i32), + #[error("Get hardware devices failed, status: {0}")] + GetHardwareDevicesFailed(i32), + #[error("AudioDeviceStart failed, status: {0}")] + AudioDeviceStartFailed(i32), + #[error("AudioDeviceStop failed, status: {0}")] + AudioDeviceStopFailed(i32), + #[error("AudioDeviceDestroyIOProcID failed, status: {0}")] + AudioDeviceDestroyIOProcIDFailed(i32), + #[error("AudioHardwareDestroyAggregateDevice failed, status: {0}")] + AudioHardwareDestroyAggregateDeviceFailed(i32), + #[error("AudioHardwareDestroyProcessTap failed, status: {0}")] + AudioHardwareDestroyProcessTapFailed(i32), + #[error("Get aggregate device property full sub device list failed, status: {0}")] + GetAggregateDevicePropertyFullSubDeviceListFailed(i32), + #[error("Add property listener block failed, status: {0}")] + AddPropertyListenerBlockFailed(i32), + #[error("AudioObjectGetPropertyData failed, status: {0}")] + AudioObjectGetPropertyDataFailed(i32), + #[error("AVAudioFile class not found")] + AVAudioFileClassNotFound, + #[error("Alloc AVAudioFile failed")] + AllocAVAudioFileFailed, + #[error("Init AVAudioFile failed")] + InitAVAudioFileFailed, + #[error("AVAudioPCMBuffer class not found")] + AVAudioPCMBufferClassNotFound, + #[error("Alloc AVAudioPCMBuffer failed")] + AllocAVAudioPCMBufferFailed, + #[error("Init AVAudioPCMBuffer failed")] + InitAVAudioPCMBufferFailed, + #[error("Write AVAudioFile failed")] + WriteAVAudioFileFailed, +} + +impl From for napi::Error { + fn from(value: CoreAudioError) -> Self { + napi::Error::new(napi::Status::GenericFailure, value.to_string()) + } +} diff --git a/packages/frontend/native/media_capture/src/macos/mod.rs b/packages/frontend/native/media_capture/src/macos/mod.rs new file mode 100644 index 0000000000..89f25680af --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/mod.rs @@ -0,0 +1,11 @@ +pub mod audio_stream_basic_desc; +pub mod av_audio_file; +pub mod av_audio_format; +pub mod av_audio_pcm_buffer; +pub mod ca_tap_description; +pub mod device; +pub(crate) mod error; +pub mod pid; +pub mod queue; +pub mod screen_capture_kit; +pub mod tap_audio; diff --git a/packages/frontend/native/media_capture/src/macos/pid.rs b/packages/frontend/native/media_capture/src/macos/pid.rs new file mode 100644 index 0000000000..caa5cb901d --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/pid.rs @@ -0,0 +1,98 @@ +use std::{mem::MaybeUninit, ptr}; + +use coreaudio::sys::{ + kAudioHardwareNoError, kAudioHardwarePropertyProcessObjectList, kAudioObjectPropertyElementMain, + kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, AudioObjectGetPropertyData, + AudioObjectGetPropertyDataSize, AudioObjectID, AudioObjectPropertyAddress, + AudioObjectPropertySelector, +}; + +use crate::error::CoreAudioError; + +pub fn audio_process_list() -> Result, CoreAudioError> { + let address = AudioObjectPropertyAddress { + mSelector: kAudioHardwarePropertyProcessObjectList, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }; + + let mut data_size = 0u32; + let status = unsafe { + AudioObjectGetPropertyDataSize( + kAudioObjectSystemObject, + &address, + 0, + ptr::null_mut(), + &mut data_size, + ) + }; + + if status != kAudioHardwareNoError as i32 { + return Err(CoreAudioError::GetProcessObjectListSizeFailed(status)); + } + + let mut process_list: Vec = vec![0; data_size as usize]; + + let status = unsafe { + AudioObjectGetPropertyData( + kAudioObjectSystemObject, + &address, + 0, + ptr::null_mut(), + (&mut data_size as *mut u32).cast(), + process_list.as_mut_ptr().cast(), + ) + }; + + if status != kAudioHardwareNoError as i32 { + return Err(CoreAudioError::GetProcessObjectListFailed(status)); + } + + Ok(process_list) +} + +pub fn get_process_property( + object: &AudioObjectID, + selector: AudioObjectPropertySelector, +) -> Result { + let object_id = *object; + let address = AudioObjectPropertyAddress { + mSelector: selector, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }; + + let mut data_size = 0u32; + let status = unsafe { + AudioObjectGetPropertyDataSize(object_id, &address, 0, ptr::null_mut(), &mut data_size) + }; + + if status != kAudioHardwareNoError as i32 { + return Err(CoreAudioError::AudioObjectGetPropertyDataSizeFailed(status)); + } + get_property_data(object_id, &address, &mut data_size) +} + +pub fn get_property_data( + object_id: AudioObjectID, + address: &AudioObjectPropertyAddress, + data_size: &mut u32, +) -> Result { + let mut property = MaybeUninit::::uninit(); + let status = unsafe { + AudioObjectGetPropertyData( + object_id, + address, + 0, + ptr::null_mut(), + (data_size as *mut u32).cast(), + property.as_mut_ptr().cast(), + ) + }; + + if status != kAudioHardwareNoError as i32 { + return Err(CoreAudioError::AudioObjectGetPropertyDataFailed(status)); + } + + Ok(unsafe { property.assume_init() }) +} diff --git a/packages/frontend/native/media_capture/src/macos/queue.rs b/packages/frontend/native/media_capture/src/macos/queue.rs new file mode 100644 index 0000000000..9c68249703 --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/queue.rs @@ -0,0 +1,12 @@ +pub(crate) fn create_audio_tap_queue() -> *mut dispatch2::ffi::dispatch_queue_s { + let queue_attr = unsafe { + dispatch2::ffi::dispatch_queue_attr_make_with_qos_class( + dispatch2::ffi::DISPATCH_QUEUE_SERIAL, + dispatch2::ffi::dispatch_qos_class_t::QOS_CLASS_USER_INITIATED, + 0, + ) + }; + unsafe { + dispatch2::ffi::dispatch_queue_create(c"ProcessTapRecorder".as_ptr().cast(), queue_attr) + } +} diff --git a/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs new file mode 100644 index 0000000000..45b7201041 --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs @@ -0,0 +1,623 @@ +use std::{ + collections::HashMap, + ffi::c_void, + ptr, + sync::{ + atomic::{AtomicPtr, Ordering}, + Arc, LazyLock, RwLock, + }, +}; + +use block2::{Block, RcBlock}; +use core_foundation::{ + base::TCFType, + string::{CFString, CFStringRef}, +}; +use coreaudio::sys::{ + kAudioHardwarePropertyProcessObjectList, kAudioObjectPropertyElementMain, + kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, kAudioProcessPropertyBundleID, + kAudioProcessPropertyIsRunning, kAudioProcessPropertyIsRunningInput, kAudioProcessPropertyPID, + AudioObjectAddPropertyListenerBlock, AudioObjectID, AudioObjectPropertyAddress, + AudioObjectRemovePropertyListenerBlock, +}; +use napi::{ + bindgen_prelude::{Buffer, Error, Float32Array, Result, Status}, + threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, +}; +use napi_derive::napi; +use objc2::{ + msg_send, + rc::Retained, + runtime::{AnyClass, AnyObject}, + Encode, Encoding, +}; +use objc2_foundation::NSString; +use screencapturekit::shareable_content::SCShareableContent; +use uuid::Uuid; + +use crate::{ + error::CoreAudioError, + pid::{audio_process_list, get_process_property}, + tap_audio::{AggregateDevice, AudioTapStream}, +}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +struct NSSize { + width: f64, + height: f64, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +struct NSPoint { + x: f64, + y: f64, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +struct NSRect { + origin: NSPoint, + size: NSSize, +} + +unsafe impl Encode for NSSize { + const ENCODING: Encoding = Encoding::Struct("NSSize", &[f64::ENCODING, f64::ENCODING]); +} + +unsafe impl Encode for NSPoint { + const ENCODING: Encoding = Encoding::Struct("NSPoint", &[f64::ENCODING, f64::ENCODING]); +} + +unsafe impl Encode for NSRect { + const ENCODING: Encoding = Encoding::Struct("NSRect", &[::ENCODING, ::ENCODING]); +} + +static RUNNING_APPLICATIONS: LazyLock>> = + LazyLock::new(|| RwLock::new(audio_process_list().expect("Failed to get running applications"))); + +static APPLICATION_STATE_CHANGED_SUBSCRIBERS: LazyLock< + RwLock>>>>, +> = LazyLock::new(|| RwLock::new(HashMap::new())); + +static APPLICATION_STATE_CHANGED_LISTENER_BLOCKS: LazyLock< + RwLock>>, +> = LazyLock::new(|| RwLock::new(HashMap::new())); + +static NSRUNNING_APPLICATION_CLASS: LazyLock> = + LazyLock::new(|| AnyClass::get(c"NSRunningApplication")); + +static AVCAPTUREDEVICE_CLASS: LazyLock> = + LazyLock::new(|| AnyClass::get(c"AVCaptureDevice")); + +static SCSTREAM_CLASS: LazyLock> = + LazyLock::new(|| AnyClass::get(c"SCStream")); + +struct TappableApplication { + object_id: AudioObjectID, +} + +impl TappableApplication { + fn new(object_id: AudioObjectID) -> Self { + Self { object_id } + } + + fn process_id(&self) -> std::result::Result { + get_process_property(&self.object_id, kAudioProcessPropertyPID) + } + + fn bundle_identifier(&self) -> Result { + let bundle_id: CFStringRef = + get_process_property(&self.object_id, kAudioProcessPropertyBundleID)?; + Ok(unsafe { CFString::wrap_under_get_rule(bundle_id) }.to_string()) + } + + fn name(&self) -> Result { + let pid = self.process_id()?; + + // Get NSRunningApplication class + let running_app_class = NSRUNNING_APPLICATION_CLASS.as_ref().ok_or_else(|| { + Error::new( + Status::GenericFailure, + "NSRunningApplication class not found", + ) + })?; + + // Get running application with PID + let running_app: *mut AnyObject = + unsafe { msg_send![*running_app_class, runningApplicationWithProcessIdentifier: pid] }; + if running_app.is_null() { + return Ok(String::new()); + } + + // Get localized name + let name: *mut NSString = unsafe { msg_send![running_app, localizedName] }; + if name.is_null() { + return Ok(String::new()); + } + + // Create a safe wrapper and convert to string + let name = unsafe { + Retained::from_raw(name).ok_or_else(|| { + Error::new( + Status::GenericFailure, + "Failed to create safe wrapper for localizedName", + ) + })? + }; + Ok(name.to_string()) + } + + fn icon(&self) -> Result> { + let pid = self.process_id()?; + + // Get NSRunningApplication class + let running_app_class = NSRUNNING_APPLICATION_CLASS.as_ref().ok_or_else(|| { + Error::new( + Status::GenericFailure, + "NSRunningApplication class not found", + ) + })?; + + // Get running application with PID + let running_app: *mut AnyObject = + unsafe { msg_send![*running_app_class, runningApplicationWithProcessIdentifier: pid] }; + if running_app.is_null() { + return Ok(Vec::new()); + } + + unsafe { + // Get original icon + let icon: *mut AnyObject = msg_send![running_app, icon]; + if icon.is_null() { + return Ok(Vec::new()); + } + + // Create a new NSImage with 64x64 size + let nsimage_class = AnyClass::get(c"NSImage") + .ok_or_else(|| Error::new(Status::GenericFailure, "NSImage class not found"))?; + let resized_image: *mut AnyObject = msg_send![nsimage_class, alloc]; + let resized_image: *mut AnyObject = + msg_send![resized_image, initWithSize: NSSize { width: 64.0, height: 64.0 }]; + let _: () = msg_send![resized_image, lockFocus]; + + // Define drawing rectangle for 64x64 image + let draw_rect = NSRect { + origin: NSPoint { x: 0.0, y: 0.0 }, + size: NSSize { + width: 64.0, + height: 64.0, + }, + }; + + // Draw the original icon into draw_rect (using NSCompositingOperationCopy = 2) + let _: () = msg_send![icon, drawInRect: draw_rect, fromRect: NSRect { origin: NSPoint { x: 0.0, y: 0.0 }, size: NSSize { width: 0.0, height: 0.0 } }, operation: 2, fraction: 1.0]; + let _: () = msg_send![resized_image, unlockFocus]; + + // Get TIFF representation from the downsized image + let tiff_data: *mut AnyObject = msg_send![resized_image, TIFFRepresentation]; + if tiff_data.is_null() { + return Ok(Vec::new()); + } + + // Create bitmap image rep from TIFF + let bitmap_class = AnyClass::get(c"NSBitmapImageRep") + .ok_or_else(|| Error::new(Status::GenericFailure, "NSBitmapImageRep class not found"))?; + let bitmap: *mut AnyObject = msg_send![bitmap_class, imageRepWithData: tiff_data]; + if bitmap.is_null() { + return Ok(Vec::new()); + } + + // Create properties dictionary with compression factor + let dict_class = AnyClass::get(c"NSMutableDictionary").ok_or_else(|| { + Error::new( + Status::GenericFailure, + "NSMutableDictionary class not found", + ) + })?; + let properties: *mut AnyObject = msg_send![dict_class, dictionary]; + + // Add compression properties + let compression_key = NSString::from_str("NSImageCompressionFactor"); + let number_class = AnyClass::get(c"NSNumber") + .ok_or_else(|| Error::new(Status::GenericFailure, "NSNumber class not found"))?; + let compression_value: *mut AnyObject = msg_send![number_class, numberWithDouble: 0.8]; + let _: () = msg_send![properties, setObject: compression_value, forKey: &*compression_key]; + + // Get PNG data with properties + let png_data: *mut AnyObject = + msg_send![bitmap, representationUsingType: 4, properties: properties]; // 4 = PNG + + if png_data.is_null() { + return Ok(Vec::new()); + } + + // Get bytes from NSData + let bytes: *const u8 = msg_send![png_data, bytes]; + let length: usize = msg_send![png_data, length]; + + if bytes.is_null() { + return Ok(Vec::new()); + } + + // Copy bytes into a Vec + let data = std::slice::from_raw_parts(bytes, length).to_vec(); + Ok(data) + } + } +} + +#[napi] +pub struct Application { + inner: TappableApplication, + pub(crate) object_id: AudioObjectID, + pub(crate) process_id: i32, + pub(crate) bundle_identifier: String, + pub(crate) name: String, +} + +#[napi] +impl Application { + fn new(app: TappableApplication) -> Result { + let object_id = app.object_id; + let bundle_identifier = app.bundle_identifier()?; + let name = app.name()?; + let process_id = app.process_id()?; + + Ok(Self { + inner: app, + object_id, + process_id, + bundle_identifier, + name, + }) + } + + #[napi] + pub fn tap_global_audio( + excluded_processes: Option>, + audio_stream_callback: Arc>, + ) -> Result { + let mut device = AggregateDevice::create_global_tap_but_exclude_processes( + &excluded_processes + .unwrap_or_default() + .iter() + .map(|app| app.object_id) + .collect::>(), + )?; + device.start(audio_stream_callback) + } + + #[napi(getter)] + pub fn process_id(&self) -> i32 { + self.process_id + } + + #[napi(getter)] + pub fn bundle_identifier(&self) -> String { + self.bundle_identifier.clone() + } + + #[napi(getter)] + pub fn name(&self) -> String { + self.name.clone() + } + + #[napi(getter)] + pub fn icon(&self) -> Result { + let icon = self.inner.icon()?; + Ok(Buffer::from(icon)) + } + + #[napi(getter)] + pub fn get_is_running(&self) -> Result { + Ok(get_process_property( + &self.object_id, + kAudioProcessPropertyIsRunningInput, + )?) + } + + #[napi] + pub fn tap_audio( + &self, + audio_stream_callback: Arc>, + ) -> Result { + let mut device = AggregateDevice::new(self)?; + device.start(audio_stream_callback) + } +} + +#[napi] +pub struct ApplicationListChangedSubscriber { + listener_block: *const Block, +} + +#[napi] +impl ApplicationListChangedSubscriber { + #[napi] + pub fn unsubscribe(&self) -> Result<()> { + let status = unsafe { + AudioObjectRemovePropertyListenerBlock( + kAudioObjectSystemObject, + &AudioObjectPropertyAddress { + mSelector: kAudioHardwarePropertyProcessObjectList, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }, + ptr::null_mut(), + self.listener_block.cast_mut().cast(), + ) + }; + if status != 0 { + return Err(Error::new( + Status::GenericFailure, + "Failed to remove property listener", + )); + } + Ok(()) + } +} + +#[napi] +pub struct ApplicationStateChangedSubscriber { + id: Uuid, + object_id: AudioObjectID, +} + +#[napi] +impl ApplicationStateChangedSubscriber { + #[napi] + pub fn unsubscribe(&self) { + if let Ok(mut lock) = APPLICATION_STATE_CHANGED_SUBSCRIBERS.write() { + if let Some(subscribers) = lock.get_mut(&self.object_id) { + subscribers.remove(&self.id); + if subscribers.is_empty() { + lock.remove(&self.object_id); + if let Some(listener_block) = APPLICATION_STATE_CHANGED_LISTENER_BLOCKS + .write() + .ok() + .as_mut() + .and_then(|map| map.remove(&self.object_id)) + { + unsafe { + AudioObjectRemovePropertyListenerBlock( + self.object_id, + &AudioObjectPropertyAddress { + mSelector: kAudioProcessPropertyIsRunning, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }, + ptr::null_mut(), + listener_block.load(Ordering::Relaxed), + ); + } + } + } + } + } + } +} + +#[napi] +pub struct ShareableContent { + _inner: SCShareableContent, +} + +#[napi] +#[derive(Default)] +pub struct RecordingPermissions { + pub audio: bool, + pub screen: bool, +} + +#[napi] +impl ShareableContent { + #[napi] + pub fn on_application_list_changed( + callback: Arc>, + ) -> Result { + let callback_block: RcBlock = + RcBlock::new(move |_in_number_addresses, _in_addresses: *mut c_void| { + if let Err(err) = RUNNING_APPLICATIONS + .write() + .map_err(|_| { + Error::new( + Status::GenericFailure, + "Poisoned RwLock while writing RunningApplications", + ) + }) + .and_then(|mut running_applications| { + audio_process_list().map_err(From::from).map(|apps| { + *running_applications = apps; + }) + }) + { + callback.call(Err(err), ThreadsafeFunctionCallMode::NonBlocking); + } else { + callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking); + } + }); + let listener_block = &*callback_block as *const Block; + let status = unsafe { + AudioObjectAddPropertyListenerBlock( + kAudioObjectSystemObject, + &AudioObjectPropertyAddress { + mSelector: kAudioHardwarePropertyProcessObjectList, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }, + ptr::null_mut(), + listener_block.cast_mut().cast(), + ) + }; + if status != 0 { + return Err(Error::new( + Status::GenericFailure, + "Failed to add property listener", + )); + } + Ok(ApplicationListChangedSubscriber { listener_block }) + } + + #[napi] + pub fn on_app_state_changed( + app: &Application, + callback: Arc>, + ) -> Result { + let id = Uuid::new_v4(); + let mut lock = APPLICATION_STATE_CHANGED_SUBSCRIBERS.write().map_err(|_| { + Error::new( + Status::GenericFailure, + "Poisoned RwLock while writing ApplicationStateChangedSubscribers", + ) + })?; + if let Some(subscribers) = lock.get_mut(&app.object_id) { + subscribers.insert(id, callback); + } else { + let object_id = app.object_id; + let list_change: RcBlock = + RcBlock::new(move |in_number_addresses, in_addresses: *mut c_void| { + let addresses = unsafe { + std::slice::from_raw_parts( + in_addresses as *mut AudioObjectPropertyAddress, + in_number_addresses as usize, + ) + }; + for address in addresses { + if address.mSelector == kAudioProcessPropertyIsRunning { + if let Some(subscribers) = APPLICATION_STATE_CHANGED_SUBSCRIBERS + .read() + .ok() + .as_ref() + .and_then(|map| map.get(&object_id)) + { + for callback in subscribers.values() { + callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking); + } + } + } + } + }); + let address = AudioObjectPropertyAddress { + mSelector: kAudioProcessPropertyIsRunning, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }; + let listener_block = &*list_change as *const Block; + let status = unsafe { + AudioObjectAddPropertyListenerBlock( + app.object_id, + &address, + ptr::null_mut(), + listener_block.cast_mut().cast(), + ) + }; + if status != 0 { + return Err(Error::new( + Status::GenericFailure, + "Failed to add property listener", + )); + } + let subscribers = { + let mut map = HashMap::new(); + map.insert(id, callback); + map + }; + lock.insert(app.object_id, subscribers); + } + Ok(ApplicationStateChangedSubscriber { + id, + object_id: app.object_id, + }) + } + + #[napi(constructor)] + pub fn new() -> Result { + Ok(Self { + _inner: SCShareableContent::get().map_err(|err| Error::new(Status::GenericFailure, err))?, + }) + } + + #[napi] + pub fn applications(&self) -> Result> { + RUNNING_APPLICATIONS + .read() + .map_err(|_| { + Error::new( + Status::GenericFailure, + "Poisoned RwLock while reading RunningApplications", + ) + })? + .iter() + .filter_map(|id| { + let app = TappableApplication::new(*id); + if !app.bundle_identifier().ok()?.is_empty() { + Some(Application::new(app)) + } else { + None + } + }) + .collect() + } + + #[napi] + pub fn application_with_process_id(&self, process_id: u32) -> Result { + // Find the AudioObjectID for the given process ID + let audio_object_id = { + let running_apps = RUNNING_APPLICATIONS.read().map_err(|_| { + Error::new( + Status::GenericFailure, + "Poisoned RwLock while reading RunningApplications", + ) + })?; + + *running_apps + .iter() + .find(|&&id| { + let app = TappableApplication::new(id); + app + .process_id() + .map(|pid| pid as u32 == process_id) + .unwrap_or(false) + }) + .ok_or_else(|| { + Error::new( + Status::GenericFailure, + format!("No application found with process ID {}", process_id), + ) + })? + }; + + let app = TappableApplication::new(audio_object_id); + Application::new(app) + } + + #[napi] + pub fn check_recording_permissions(&self) -> Result { + let av_capture_class = AVCAPTUREDEVICE_CLASS + .as_ref() + .ok_or_else(|| Error::new(Status::GenericFailure, "AVCaptureDevice class not found"))?; + + let sc_stream_class = SCSTREAM_CLASS + .as_ref() + .ok_or_else(|| Error::new(Status::GenericFailure, "SCStream class not found"))?; + + let media_type = NSString::from_str("com.apple.avfoundation.avcapturedevice.built-in_audio"); + + let audio_status: i32 = unsafe { + msg_send![ + *av_capture_class, + authorizationStatusForMediaType: &*media_type + ] + }; + + let screen_status: bool = unsafe { msg_send![*sc_stream_class, isScreenCaptureAuthorized] }; + + Ok(RecordingPermissions { + // AVAuthorizationStatusAuthorized = 3 + audio: audio_status == 3, + screen: screen_status, + }) + } +} diff --git a/packages/frontend/native/media_capture/src/macos/tap_audio.rs b/packages/frontend/native/media_capture/src/macos/tap_audio.rs new file mode 100644 index 0000000000..093346ac35 --- /dev/null +++ b/packages/frontend/native/media_capture/src/macos/tap_audio.rs @@ -0,0 +1,360 @@ +use std::{ffi::c_void, sync::Arc}; + +use block2::{Block, RcBlock}; +use core_foundation::{ + array::CFArray, + base::{CFType, ItemRef, TCFType}, + boolean::CFBoolean, + dictionary::CFDictionary, + string::CFString, + uuid::CFUUID, +}; +use coreaudio::sys::{ + kAudioAggregateDeviceIsPrivateKey, kAudioAggregateDeviceIsStackedKey, + kAudioAggregateDeviceMainSubDeviceKey, kAudioAggregateDeviceNameKey, + kAudioAggregateDeviceSubDeviceListKey, kAudioAggregateDeviceTapAutoStartKey, + kAudioAggregateDeviceTapListKey, kAudioAggregateDeviceUIDKey, kAudioHardwareNoError, + kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultSystemOutputDevice, + kAudioSubDeviceUIDKey, kAudioSubTapDriftCompensationKey, kAudioSubTapUIDKey, + AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID, AudioDeviceIOProcID, + AudioDeviceStart, AudioDeviceStop, AudioHardwareCreateAggregateDevice, + AudioHardwareDestroyAggregateDevice, AudioObjectID, AudioTimeStamp, OSStatus, +}; +use napi::{ + bindgen_prelude::Float32Array, + threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, + Result, +}; +use napi_derive::napi; +use objc2::{runtime::AnyObject, Encode, Encoding, RefEncode}; + +use crate::{ + ca_tap_description::CATapDescription, device::get_device_uid, error::CoreAudioError, + queue::create_audio_tap_queue, screen_capture_kit::Application, +}; + +extern "C" { + fn AudioHardwareCreateProcessTap( + inDescription: *mut AnyObject, + outTapID: *mut AudioObjectID, + ) -> OSStatus; + + fn AudioHardwareDestroyProcessTap(tapID: AudioObjectID) -> OSStatus; +} + +/// [Apple's documentation](https://developer.apple.com/documentation/coreaudiotypes/audiobuffer?language=objc) +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +#[allow(non_snake_case)] +pub struct AudioBuffer { + pub mNumberChannels: u32, + pub mDataByteSize: u32, + pub mData: *mut c_void, +} + +unsafe impl Encode for AudioBuffer { + const ENCODING: Encoding = Encoding::Struct( + "AudioBuffer", + &[::ENCODING, ::ENCODING, <*mut c_void>::ENCODING], + ); +} + +unsafe impl RefEncode for AudioBuffer { + const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING); +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +#[allow(non_snake_case)] +pub struct AudioBufferList { + pub mNumberBuffers: u32, + pub mBuffers: [AudioBuffer; 1], +} + +unsafe impl Encode for AudioBufferList { + const ENCODING: Encoding = Encoding::Struct( + "AudioBufferList", + &[::ENCODING, <[AudioBuffer; 1]>::ENCODING], + ); +} + +unsafe impl RefEncode for AudioBufferList { + const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING); +} + +pub struct AggregateDevice { + pub tap_id: AudioObjectID, + pub id: AudioObjectID, +} + +impl AggregateDevice { + pub fn new(app: &Application) -> Result { + let mut tap_id: AudioObjectID = 0; + + let tap_description = CATapDescription::init_stereo_mixdown_of_processes(app.object_id)?; + let status = unsafe { AudioHardwareCreateProcessTap(tap_description.inner, &mut tap_id) }; + + if status != 0 { + return Err(CoreAudioError::CreateProcessTapFailed(status).into()); + } + + let description_dict = Self::create_aggregate_description(tap_id, tap_description.get_uuid()?)?; + + let mut aggregate_device_id: AudioObjectID = 0; + + let status = unsafe { + AudioHardwareCreateAggregateDevice( + description_dict.as_concrete_TypeRef().cast(), + &mut aggregate_device_id, + ) + }; + + // Check the status and return the appropriate result + if status != 0 { + return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into()); + } + + Ok(Self { + tap_id, + id: aggregate_device_id, + }) + } + + pub fn create_global_tap_but_exclude_processes(processes: &[AudioObjectID]) -> Result { + let mut tap_id: AudioObjectID = 0; + let tap_description = + CATapDescription::init_stereo_global_tap_but_exclude_processes(processes)?; + let status = unsafe { AudioHardwareCreateProcessTap(tap_description.inner, &mut tap_id) }; + + if status != 0 { + return Err(CoreAudioError::CreateProcessTapFailed(status).into()); + } + + let description_dict = Self::create_aggregate_description(tap_id, tap_description.get_uuid()?)?; + + let mut aggregate_device_id: AudioObjectID = 0; + + let status = unsafe { + AudioHardwareCreateAggregateDevice( + description_dict.as_concrete_TypeRef().cast(), + &mut aggregate_device_id, + ) + }; + + // Check the status and return the appropriate result + if status != 0 { + return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into()); + } + + Ok(Self { + tap_id, + id: aggregate_device_id, + }) + } + + pub fn start( + &mut self, + audio_stream_callback: Arc>, + ) -> Result { + let queue = create_audio_tap_queue(); + let mut in_proc_id: AudioDeviceIOProcID = None; + + let in_io_block: RcBlock< + dyn Fn(*mut c_void, *mut c_void, *mut c_void, *mut c_void, *mut c_void) -> i32, + > = RcBlock::new( + move |_in_now: *mut c_void, + in_input_data: *mut c_void, + in_input_time: *mut c_void, + _out_output_data: *mut c_void, + _in_output_time: *mut c_void| { + let AudioTimeStamp { mSampleTime, .. } = unsafe { &*in_input_time.cast() }; + + // ignore pre-roll + if *mSampleTime < 0.0 { + return kAudioHardwareNoError as i32; + } + let AudioBufferList { mBuffers, .. } = + unsafe { &mut *in_input_data.cast::() }; + let [AudioBuffer { + mData, + mNumberChannels, + mDataByteSize, + }] = mBuffers; + // Only create slice if we have valid data + if !mData.is_null() && *mDataByteSize > 0 { + // Calculate total number of samples (accounting for interleaved stereo) + let total_samples = *mDataByteSize as usize / 4; // 4 bytes per f32 + + // Create a slice of all samples + let samples: &[f32] = + unsafe { std::slice::from_raw_parts(mData.cast::(), total_samples) }; + + // Convert to mono if needed + let mono_samples: Vec = if *mNumberChannels > 1 { + samples + .chunks(*mNumberChannels as usize) + .map(|chunk| chunk.iter().sum::() / *mNumberChannels as f32) + .collect() + } else { + samples.to_vec() + }; + + audio_stream_callback.call( + Ok(mono_samples.into()), + ThreadsafeFunctionCallMode::NonBlocking, + ); + } + + kAudioHardwareNoError as i32 + }, + ); + + let status = unsafe { + AudioDeviceCreateIOProcIDWithBlock( + &mut in_proc_id, + self.id, + queue.cast(), + (&*in_io_block + as *const Block< + dyn Fn(*mut c_void, *mut c_void, *mut c_void, *mut c_void, *mut c_void) -> i32, + >) + .cast_mut() + .cast(), + ) + }; + if status != 0 { + return Err(CoreAudioError::CreateIOProcIDWithBlockFailed(status).into()); + } + let status = unsafe { AudioDeviceStart(self.id, in_proc_id) }; + if status != 0 { + return Err(CoreAudioError::AudioDeviceStartFailed(status).into()); + } + + Ok(AudioTapStream { + device_id: self.id, + in_proc_id, + stop_called: false, + }) + } + + fn create_aggregate_description( + tap_id: AudioObjectID, + tap_uuid_string: ItemRef, + ) -> Result> { + let system_output_uid = get_device_uid(kAudioHardwarePropertyDefaultSystemOutputDevice)?; + let default_input_uid = get_device_uid(kAudioHardwarePropertyDefaultInputDevice)?; + + let aggregate_device_name = CFString::new(&format!("Tap-{}", tap_id)); + let aggregate_device_uid: uuid::Uuid = CFUUID::new().into(); + let aggregate_device_uid_string = aggregate_device_uid.to_string(); + + // Sub-device UID key and dictionary + let sub_device_output_dict = CFDictionary::from_CFType_pairs(&[( + cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(), + system_output_uid.as_CFType(), + )]); + + let sub_device_input_dict = CFDictionary::from_CFType_pairs(&[( + cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(), + default_input_uid.as_CFType(), + )]); + + let tap_device_dict = CFDictionary::from_CFType_pairs(&[ + ( + cfstring_from_bytes_with_nul(kAudioSubTapDriftCompensationKey).as_CFType(), + CFBoolean::false_value().as_CFType(), + ), + ( + cfstring_from_bytes_with_nul(kAudioSubTapUIDKey).as_CFType(), + tap_uuid_string.as_CFType(), + ), + ]); + + let capture_device_list = vec![sub_device_input_dict, sub_device_output_dict]; + + // Sub-device list + let sub_device_list = CFArray::from_CFTypes(&capture_device_list); + + let tap_list = CFArray::from_CFTypes(&[tap_device_dict]); + + // Create the aggregate device description dictionary + let description_dict = CFDictionary::from_CFType_pairs(&[ + ( + cfstring_from_bytes_with_nul(kAudioAggregateDeviceNameKey).as_CFType(), + aggregate_device_name.as_CFType(), + ), + ( + cfstring_from_bytes_with_nul(kAudioAggregateDeviceUIDKey).as_CFType(), + CFString::new(aggregate_device_uid_string.as_str()).as_CFType(), + ), + ( + cfstring_from_bytes_with_nul(kAudioAggregateDeviceMainSubDeviceKey).as_CFType(), + system_output_uid.as_CFType(), + ), + ( + cfstring_from_bytes_with_nul(kAudioAggregateDeviceIsPrivateKey).as_CFType(), + CFBoolean::true_value().as_CFType(), + ), + ( + cfstring_from_bytes_with_nul(kAudioAggregateDeviceIsStackedKey).as_CFType(), + CFBoolean::false_value().as_CFType(), + ), + ( + cfstring_from_bytes_with_nul(kAudioAggregateDeviceTapAutoStartKey).as_CFType(), + CFBoolean::true_value().as_CFType(), + ), + ( + cfstring_from_bytes_with_nul(kAudioAggregateDeviceSubDeviceListKey).as_CFType(), + sub_device_list.as_CFType(), + ), + ( + cfstring_from_bytes_with_nul(kAudioAggregateDeviceTapListKey).as_CFType(), + tap_list.as_CFType(), + ), + ]); + Ok(description_dict) + } +} + +#[napi] +pub struct AudioTapStream { + device_id: AudioObjectID, + in_proc_id: AudioDeviceIOProcID, + stop_called: bool, +} + +#[napi] +impl AudioTapStream { + #[napi] + pub fn stop(&mut self) -> Result<()> { + if self.stop_called { + return Ok(()); + } + self.stop_called = true; + let status = unsafe { AudioDeviceStop(self.device_id, self.in_proc_id) }; + if status != 0 { + return Err(CoreAudioError::AudioDeviceStopFailed(status).into()); + } + let status = unsafe { AudioDeviceDestroyIOProcID(self.device_id, self.in_proc_id) }; + if status != 0 { + return Err(CoreAudioError::AudioDeviceDestroyIOProcIDFailed(status).into()); + } + let status = unsafe { AudioHardwareDestroyAggregateDevice(self.device_id) }; + if status != 0 { + return Err(CoreAudioError::AudioHardwareDestroyAggregateDeviceFailed(status).into()); + } + let status = unsafe { AudioHardwareDestroyProcessTap(self.device_id) }; + if status != 0 { + return Err(CoreAudioError::AudioHardwareDestroyProcessTapFailed(status).into()); + } + Ok(()) + } +} + +fn cfstring_from_bytes_with_nul(bytes: &'static [u8]) -> CFString { + CFString::new( + unsafe { std::ffi::CStr::from_bytes_with_nul_unchecked(bytes) } + .to_string_lossy() + .as_ref(), + ) +} diff --git a/packages/frontend/native/package.json b/packages/frontend/native/package.json index e83789f3fb..beca6f1b45 100644 --- a/packages/frontend/native/package.json +++ b/packages/frontend/native/package.json @@ -26,8 +26,10 @@ }, "devDependencies": { "@napi-rs/cli": "3.0.0-alpha.70", + "@napi-rs/whisper": "^0.0.4", "@types/node": "^22.0.0", "ava": "^6.2.0", + "rxjs": "^7.8.1", "ts-node": "^10.9.2", "typescript": "^5.7.2" }, diff --git a/packages/frontend/native/src/lib.rs b/packages/frontend/native/src/lib.rs index f9b3fdb30e..b17678a551 100644 --- a/packages/frontend/native/src/lib.rs +++ b/packages/frontend/native/src/lib.rs @@ -1,4 +1,6 @@ pub mod hashcash; +#[allow(unused_imports)] +pub use affine_media_capture::*; pub use affine_nbstore::*; pub use affine_sqlite_v1::*; diff --git a/packages/frontend/native/tsconfig.json b/packages/frontend/native/tsconfig.json index 32fe80bd50..ee4e505640 100644 --- a/packages/frontend/native/tsconfig.json +++ b/packages/frontend/native/tsconfig.json @@ -3,6 +3,6 @@ "compilerOptions": { "outDir": "./dist" }, - "include": ["index.d.ts"], + "include": ["index.d.ts", "media-capture-example.ts"], "references": [] } diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts index 5c30988aaa..e1177e13cd 100644 --- a/tools/utils/src/workspace.gen.ts +++ b/tools/utils/src/workspace.gen.ts @@ -715,6 +715,11 @@ export const PackageList = [ 'tools/utils', ], }, + { + location: 'packages/frontend/media-capture-playground', + name: '@affine/media-capture-playground', + workspaceDependencies: ['packages/frontend/native'], + }, { location: 'packages/frontend/native', name: '@affine/native', @@ -866,6 +871,7 @@ export type PackageName = | '@affine/electron-api' | '@affine/graphql' | '@affine/i18n' + | '@affine/media-capture-playground' | '@affine/native' | '@affine/templates' | '@affine/track' diff --git a/tsconfig.json b/tsconfig.json index cd8227ea92..50ed943e9f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -107,6 +107,7 @@ { "path": "./packages/frontend/electron-api" }, { "path": "./packages/frontend/graphql" }, { "path": "./packages/frontend/i18n" }, + { "path": "./packages/frontend/media-capture-playground" }, { "path": "./packages/frontend/native" }, { "path": "./packages/frontend/track" }, { "path": "./tests/affine-cloud" }, diff --git a/yarn.lock b/yarn.lock index 13b67b06e9..e839d9c038 100644 --- a/yarn.lock +++ b/yarn.lock @@ -639,6 +639,40 @@ __metadata: languageName: unknown linkType: soft +"@affine/media-capture-playground@workspace:packages/frontend/media-capture-playground": + version: 0.0.0-use.local + resolution: "@affine/media-capture-playground@workspace:packages/frontend/media-capture-playground" + dependencies: + "@affine/native": "workspace:*" + "@google/generative-ai": "npm:^0.21.0" + "@tailwindcss/vite": "npm:^4.0.6" + "@types/express": "npm:^4" + "@types/fs-extra": "npm:^11" + "@types/multer": "npm:^1" + "@types/react": "npm:^19.0.1" + "@types/react-dom": "npm:^19.0.2" + "@types/socket.io": "npm:^3.0.2" + "@types/socket.io-client": "npm:^3.0.0" + "@vitejs/plugin-react": "npm:^4.3.4" + chokidar: "npm:^4.0.3" + express: "npm:^4.21.2" + express-rate-limit: "npm:^7.1.5" + fs-extra: "npm:^11.3.0" + multer: "npm:^1.4.5-lts.1" + openai: "npm:^4.85.1" + react: "npm:^19.0.0" + react-dom: "npm:^19.0.0" + react-markdown: "npm:^9.0.3" + rxjs: "npm:^7.8.1" + socket.io: "npm:^4.7.4" + socket.io-client: "npm:^4.7.4" + swr: "npm:^2.3.2" + tailwindcss: "npm:^4.0.6" + tsx: "npm:^4.19.2" + vite: "npm:^6.1.0" + languageName: unknown + linkType: soft + "@affine/mobile@workspace:packages/frontend/apps/mobile": version: 0.0.0-use.local resolution: "@affine/mobile@workspace:packages/frontend/apps/mobile" @@ -715,8 +749,10 @@ __metadata: resolution: "@affine/native@workspace:packages/frontend/native" dependencies: "@napi-rs/cli": "npm:3.0.0-alpha.70" + "@napi-rs/whisper": "npm:^0.0.4" "@types/node": "npm:^22.0.0" ava: "npm:^6.2.0" + rxjs: "npm:^7.8.1" ts-node: "npm:^10.9.2" typescript: "npm:^5.7.2" languageName: unknown @@ -1890,7 +1926,7 @@ __metadata: languageName: node linkType: hard -"@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.10.4, @babel/code-frame@npm:^7.22.13, @babel/code-frame@npm:^7.24.2, @babel/code-frame@npm:^7.25.9, @babel/code-frame@npm:^7.26.0, @babel/code-frame@npm:^7.26.2": +"@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.10.4, @babel/code-frame@npm:^7.22.13, @babel/code-frame@npm:^7.24.2, @babel/code-frame@npm:^7.26.0, @babel/code-frame@npm:^7.26.2": version: 7.26.2 resolution: "@babel/code-frame@npm:7.26.2" dependencies: @@ -1954,26 +1990,27 @@ __metadata: languageName: node linkType: hard -"@babel/core@npm:^7.12.3, @babel/core@npm:^7.18.5, @babel/core@npm:^7.18.9, @babel/core@npm:^7.22.1, @babel/core@npm:^7.22.9, @babel/core@npm:^7.23.9": - version: 7.26.7 - resolution: "@babel/core@npm:7.26.7" +"@babel/core@npm:^7.12.3, @babel/core@npm:^7.18.5, @babel/core@npm:^7.18.9, @babel/core@npm:^7.22.1, @babel/core@npm:^7.22.9, @babel/core@npm:^7.23.9, @babel/core@npm:^7.26.0": + version: 7.26.8 + resolution: "@babel/core@npm:7.26.8" dependencies: "@ampproject/remapping": "npm:^2.2.0" "@babel/code-frame": "npm:^7.26.2" - "@babel/generator": "npm:^7.26.5" + "@babel/generator": "npm:^7.26.8" "@babel/helper-compilation-targets": "npm:^7.26.5" "@babel/helper-module-transforms": "npm:^7.26.0" "@babel/helpers": "npm:^7.26.7" - "@babel/parser": "npm:^7.26.7" - "@babel/template": "npm:^7.25.9" - "@babel/traverse": "npm:^7.26.7" - "@babel/types": "npm:^7.26.7" + "@babel/parser": "npm:^7.26.8" + "@babel/template": "npm:^7.26.8" + "@babel/traverse": "npm:^7.26.8" + "@babel/types": "npm:^7.26.8" + "@types/gensync": "npm:^1.0.0" convert-source-map: "npm:^2.0.0" debug: "npm:^4.1.0" gensync: "npm:^1.0.0-beta.2" json5: "npm:^2.2.3" semver: "npm:^6.3.1" - checksum: 10/1ca1c9b1366a1ee77ade9c72302f288b2b148e4190e0f36bc032d09c686b2c7973d3309e4eec2c57243508c16cf907c17dec4e34ba95e7a18badd57c61bbcb7c + checksum: 10/a70d903dfd2c97e044b27fb480584fab6111954ced6987c6628ee4d37071ca446eca7830d72763a8d16a0da64eb83e02e3073d16c09e9eefa4a4ab38162636b4 languageName: node linkType: hard @@ -1991,16 +2028,16 @@ __metadata: languageName: node linkType: hard -"@babel/generator@npm:^7.14.0, @babel/generator@npm:^7.18.13, @babel/generator@npm:^7.24.5, @babel/generator@npm:^7.26.0, @babel/generator@npm:^7.26.5": - version: 7.26.5 - resolution: "@babel/generator@npm:7.26.5" +"@babel/generator@npm:^7.14.0, @babel/generator@npm:^7.18.13, @babel/generator@npm:^7.24.5, @babel/generator@npm:^7.26.0, @babel/generator@npm:^7.26.8": + version: 7.26.8 + resolution: "@babel/generator@npm:7.26.8" dependencies: - "@babel/parser": "npm:^7.26.5" - "@babel/types": "npm:^7.26.5" + "@babel/parser": "npm:^7.26.8" + "@babel/types": "npm:^7.26.8" "@jridgewell/gen-mapping": "npm:^0.3.5" "@jridgewell/trace-mapping": "npm:^0.3.25" jsesc: "npm:^3.0.2" - checksum: 10/aa5f176155431d1fb541ca11a7deddec0fc021f20992ced17dc2f688a0a9584e4ff4280f92e8a39302627345cd325762f70f032764806c579c6fd69432542bcb + checksum: 10/8c5af0f74aad2e575f2f833af0a9a38dda5abe0574752b5e0812677c78e5dc713b6b0c9ac3b30799ba6ef883614f9f0ef79d3aa10ba8f0e54f7f0284381b0059 languageName: node linkType: hard @@ -2207,14 +2244,14 @@ __metadata: languageName: node linkType: hard -"@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.0, @babel/parser@npm:^7.16.8, @babel/parser@npm:^7.20.7, @babel/parser@npm:^7.22.6, @babel/parser@npm:^7.23.9, @babel/parser@npm:^7.24.5, @babel/parser@npm:^7.25.4, @babel/parser@npm:^7.25.9, @babel/parser@npm:^7.26.0, @babel/parser@npm:^7.26.5, @babel/parser@npm:^7.26.7": - version: 7.26.7 - resolution: "@babel/parser@npm:7.26.7" +"@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.0, @babel/parser@npm:^7.16.8, @babel/parser@npm:^7.20.7, @babel/parser@npm:^7.22.6, @babel/parser@npm:^7.23.9, @babel/parser@npm:^7.24.5, @babel/parser@npm:^7.25.4, @babel/parser@npm:^7.26.0, @babel/parser@npm:^7.26.8": + version: 7.26.8 + resolution: "@babel/parser@npm:7.26.8" dependencies: - "@babel/types": "npm:^7.26.7" + "@babel/types": "npm:^7.26.8" bin: parser: ./bin/babel-parser.js - checksum: 10/3ccc384366ca9a9b49c54f5b24c9d8cff9a505f2fbdd1cfc04941c8e1897084cc32f100e77900c12bc14a176cf88daa3c155faad680d9a23491b997fd2a59ffc + checksum: 10/0dd9d6b2022806b696b7a9ffb50b147f13525c497663d758a95adcc3ca0fa1d1bbb605fcc0604acc1cade60c3dbf2c1e0dd22b7aed17f8ad1c58c954208ffe7a languageName: node linkType: hard @@ -2920,6 +2957,28 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-react-jsx-self@npm:^7.25.9": + version: 7.25.9 + resolution: "@babel/plugin-transform-react-jsx-self@npm:7.25.9" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.25.9" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10/41c833cd7f91b1432710f91b1325706e57979b2e8da44e83d86312c78bbe96cd9ef778b4e79e4e17ab25fa32c72b909f2be7f28e876779ede28e27506c41f4ae + languageName: node + linkType: hard + +"@babel/plugin-transform-react-jsx-source@npm:^7.25.9": + version: 7.25.9 + resolution: "@babel/plugin-transform-react-jsx-source@npm:7.25.9" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.25.9" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10/a3e0e5672e344e9d01fb20b504fe29a84918eaa70cec512c4d4b1b035f72803261257343d8e93673365b72c371f35cf34bb0d129720bf178a4c87812c8b9c662 + languageName: node + linkType: hard + "@babel/plugin-transform-react-jsx@npm:^7.0.0, @babel/plugin-transform-react-jsx@npm:^7.25.9": version: 7.25.9 resolution: "@babel/plugin-transform-react-jsx@npm:7.25.9" @@ -3230,39 +3289,39 @@ __metadata: languageName: node linkType: hard -"@babel/template@npm:^7.18.10, @babel/template@npm:^7.20.7, @babel/template@npm:^7.24.0, @babel/template@npm:^7.25.9": - version: 7.25.9 - resolution: "@babel/template@npm:7.25.9" - dependencies: - "@babel/code-frame": "npm:^7.25.9" - "@babel/parser": "npm:^7.25.9" - "@babel/types": "npm:^7.25.9" - checksum: 10/e861180881507210150c1335ad94aff80fd9e9be6202e1efa752059c93224e2d5310186ddcdd4c0f0b0fc658ce48cb47823f15142b5c00c8456dde54f5de80b2 - languageName: node - linkType: hard - -"@babel/traverse@npm:^7.16.8, @babel/traverse@npm:^7.18.9, @babel/traverse@npm:^7.24.5, @babel/traverse@npm:^7.25.9, @babel/traverse@npm:^7.26.5, @babel/traverse@npm:^7.26.7": - version: 7.26.7 - resolution: "@babel/traverse@npm:7.26.7" +"@babel/template@npm:^7.18.10, @babel/template@npm:^7.20.7, @babel/template@npm:^7.24.0, @babel/template@npm:^7.25.9, @babel/template@npm:^7.26.8": + version: 7.26.8 + resolution: "@babel/template@npm:7.26.8" dependencies: "@babel/code-frame": "npm:^7.26.2" - "@babel/generator": "npm:^7.26.5" - "@babel/parser": "npm:^7.26.7" - "@babel/template": "npm:^7.25.9" - "@babel/types": "npm:^7.26.7" - debug: "npm:^4.3.1" - globals: "npm:^11.1.0" - checksum: 10/c821c9682fe0b9edf7f7cbe9cc3e0787ffee3f73b52c13b21b463f8979950a6433f5e7e482a74348d22c0b7a05180e6f72b23eb6732328b49c59fc6388ebf6e5 + "@babel/parser": "npm:^7.26.8" + "@babel/types": "npm:^7.26.8" + checksum: 10/bc45db0fd4e92d35813c2a8e8fa80b8a887c275b323537b8ebd9c64228c1614e81c74236d08f744017a6562987e48b10501688f7a8be5d6a53fb6acb61aa01c8 languageName: node linkType: hard -"@babel/types@npm:^7.0.0, @babel/types@npm:^7.16.8, @babel/types@npm:^7.18.13, @babel/types@npm:^7.18.9, @babel/types@npm:^7.20.7, @babel/types@npm:^7.24.5, @babel/types@npm:^7.25.4, @babel/types@npm:^7.25.9, @babel/types@npm:^7.26.0, @babel/types@npm:^7.26.5, @babel/types@npm:^7.26.7, @babel/types@npm:^7.4.4, @babel/types@npm:^7.8.3": - version: 7.26.7 - resolution: "@babel/types@npm:7.26.7" +"@babel/traverse@npm:^7.16.8, @babel/traverse@npm:^7.18.9, @babel/traverse@npm:^7.24.5, @babel/traverse@npm:^7.25.9, @babel/traverse@npm:^7.26.5, @babel/traverse@npm:^7.26.8": + version: 7.26.8 + resolution: "@babel/traverse@npm:7.26.8" + dependencies: + "@babel/code-frame": "npm:^7.26.2" + "@babel/generator": "npm:^7.26.8" + "@babel/parser": "npm:^7.26.8" + "@babel/template": "npm:^7.26.8" + "@babel/types": "npm:^7.26.8" + debug: "npm:^4.3.1" + globals: "npm:^11.1.0" + checksum: 10/2785718e54d7a243a4c1b92fe9c2cec0d3b8725b095061b8fdb9812bbcf1b94b743b39d96312644efa05692f9c2646772a8154c89625f428aa6b568cebf4ecf9 + languageName: node + linkType: hard + +"@babel/types@npm:^7.0.0, @babel/types@npm:^7.16.8, @babel/types@npm:^7.18.13, @babel/types@npm:^7.18.9, @babel/types@npm:^7.20.7, @babel/types@npm:^7.24.5, @babel/types@npm:^7.25.4, @babel/types@npm:^7.25.9, @babel/types@npm:^7.26.0, @babel/types@npm:^7.26.7, @babel/types@npm:^7.26.8, @babel/types@npm:^7.4.4, @babel/types@npm:^7.8.3": + version: 7.26.8 + resolution: "@babel/types@npm:7.26.8" dependencies: "@babel/helper-string-parser": "npm:^7.25.9" "@babel/helper-validator-identifier": "npm:^7.25.9" - checksum: 10/2264efd02cc261ca5d1c5bc94497c8995238f28afd2b7483b24ea64dd694cf46b00d51815bf0c87f0d0061ea221569c77893aeecb0d4b4bb254e9c2f938d7669 + checksum: 10/e6889246889706ee5e605cbfe62657c829427e0ddef0e4d18679a0d989bdb23e700b5a851d84821c2bdce3ded9ae5b9285fe1028562201b28f816e3ade6c3d0d languageName: node linkType: hard @@ -5463,6 +5522,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/aix-ppc64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/aix-ppc64@npm:0.23.1" + conditions: os=aix & cpu=ppc64 + languageName: node + linkType: hard + "@esbuild/aix-ppc64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/aix-ppc64@npm:0.24.2" @@ -5484,6 +5550,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/android-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/android-arm64@npm:0.23.1" + conditions: os=android & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/android-arm64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/android-arm64@npm:0.24.2" @@ -5505,6 +5578,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/android-arm@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/android-arm@npm:0.23.1" + conditions: os=android & cpu=arm + languageName: node + linkType: hard + "@esbuild/android-arm@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/android-arm@npm:0.24.2" @@ -5526,6 +5606,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/android-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/android-x64@npm:0.23.1" + conditions: os=android & cpu=x64 + languageName: node + linkType: hard + "@esbuild/android-x64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/android-x64@npm:0.24.2" @@ -5547,6 +5634,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/darwin-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/darwin-arm64@npm:0.23.1" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/darwin-arm64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/darwin-arm64@npm:0.24.2" @@ -5568,6 +5662,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/darwin-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/darwin-x64@npm:0.23.1" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + "@esbuild/darwin-x64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/darwin-x64@npm:0.24.2" @@ -5589,6 +5690,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/freebsd-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/freebsd-arm64@npm:0.23.1" + conditions: os=freebsd & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/freebsd-arm64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/freebsd-arm64@npm:0.24.2" @@ -5610,6 +5718,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/freebsd-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/freebsd-x64@npm:0.23.1" + conditions: os=freebsd & cpu=x64 + languageName: node + linkType: hard + "@esbuild/freebsd-x64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/freebsd-x64@npm:0.24.2" @@ -5631,6 +5746,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-arm64@npm:0.23.1" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/linux-arm64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/linux-arm64@npm:0.24.2" @@ -5652,6 +5774,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-arm@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-arm@npm:0.23.1" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + "@esbuild/linux-arm@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/linux-arm@npm:0.24.2" @@ -5673,6 +5802,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-ia32@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-ia32@npm:0.23.1" + conditions: os=linux & cpu=ia32 + languageName: node + linkType: hard + "@esbuild/linux-ia32@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/linux-ia32@npm:0.24.2" @@ -5694,6 +5830,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-loong64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-loong64@npm:0.23.1" + conditions: os=linux & cpu=loong64 + languageName: node + linkType: hard + "@esbuild/linux-loong64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/linux-loong64@npm:0.24.2" @@ -5715,6 +5858,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-mips64el@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-mips64el@npm:0.23.1" + conditions: os=linux & cpu=mips64el + languageName: node + linkType: hard + "@esbuild/linux-mips64el@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/linux-mips64el@npm:0.24.2" @@ -5736,6 +5886,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-ppc64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-ppc64@npm:0.23.1" + conditions: os=linux & cpu=ppc64 + languageName: node + linkType: hard + "@esbuild/linux-ppc64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/linux-ppc64@npm:0.24.2" @@ -5757,6 +5914,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-riscv64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-riscv64@npm:0.23.1" + conditions: os=linux & cpu=riscv64 + languageName: node + linkType: hard + "@esbuild/linux-riscv64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/linux-riscv64@npm:0.24.2" @@ -5778,6 +5942,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-s390x@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-s390x@npm:0.23.1" + conditions: os=linux & cpu=s390x + languageName: node + linkType: hard + "@esbuild/linux-s390x@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/linux-s390x@npm:0.24.2" @@ -5799,6 +5970,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-x64@npm:0.23.1" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + "@esbuild/linux-x64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/linux-x64@npm:0.24.2" @@ -5834,6 +6012,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/netbsd-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/netbsd-x64@npm:0.23.1" + conditions: os=netbsd & cpu=x64 + languageName: node + linkType: hard + "@esbuild/netbsd-x64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/netbsd-x64@npm:0.24.2" @@ -5855,6 +6040,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/openbsd-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/openbsd-arm64@npm:0.23.1" + conditions: os=openbsd & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/openbsd-arm64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/openbsd-arm64@npm:0.24.2" @@ -5876,6 +6068,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/openbsd-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/openbsd-x64@npm:0.23.1" + conditions: os=openbsd & cpu=x64 + languageName: node + linkType: hard + "@esbuild/openbsd-x64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/openbsd-x64@npm:0.24.2" @@ -5897,6 +6096,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/sunos-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/sunos-x64@npm:0.23.1" + conditions: os=sunos & cpu=x64 + languageName: node + linkType: hard + "@esbuild/sunos-x64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/sunos-x64@npm:0.24.2" @@ -5918,6 +6124,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/win32-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/win32-arm64@npm:0.23.1" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/win32-arm64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/win32-arm64@npm:0.24.2" @@ -5939,6 +6152,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/win32-ia32@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/win32-ia32@npm:0.23.1" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + "@esbuild/win32-ia32@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/win32-ia32@npm:0.24.2" @@ -5960,6 +6180,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/win32-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/win32-x64@npm:0.23.1" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + "@esbuild/win32-x64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/win32-x64@npm:0.24.2" @@ -6217,6 +6444,13 @@ __metadata: languageName: node linkType: hard +"@google/generative-ai@npm:^0.21.0": + version: 0.21.0 + resolution: "@google/generative-ai@npm:0.21.0" + checksum: 10/68c4163b7ac0c449cdbdf83b85dd77b8d96ee2b883b0190bcb1c7e76d98b56968d3e554ff62c555c3a3814855bac3bfc1cf2035f2b7ddf71f313b5c846b60f31 + languageName: node + linkType: hard + "@googleapis/androidpublisher@npm:^22.0.0": version: 22.0.0 resolution: "@googleapis/androidpublisher@npm:22.0.0" @@ -8822,6 +9056,47 @@ __metadata: languageName: node linkType: hard +"@napi-rs/whisper-darwin-arm64@npm:0.0.4": + version: 0.0.4 + resolution: "@napi-rs/whisper-darwin-arm64@npm:0.0.4" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@napi-rs/whisper-darwin-x64@npm:0.0.4": + version: 0.0.4 + resolution: "@napi-rs/whisper-darwin-x64@npm:0.0.4" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@napi-rs/whisper-linux-x64-gnu@npm:0.0.4": + version: 0.0.4 + resolution: "@napi-rs/whisper-linux-x64-gnu@npm:0.0.4" + conditions: os=linux & cpu=x64 & libc=glibc + languageName: node + linkType: hard + +"@napi-rs/whisper@npm:^0.0.4": + version: 0.0.4 + resolution: "@napi-rs/whisper@npm:0.0.4" + dependencies: + "@napi-rs/whisper-darwin-arm64": "npm:0.0.4" + "@napi-rs/whisper-darwin-x64": "npm:0.0.4" + "@napi-rs/whisper-linux-x64-gnu": "npm:0.0.4" + dependenciesMeta: + "@napi-rs/whisper-darwin-arm64": + optional: true + "@napi-rs/whisper-darwin-x64": + optional: true + "@napi-rs/whisper-linux-x64-gnu": + optional: true + bin: + download-whisper-model: scripts/download-ggml-model.mjs + checksum: 10/8c181730daf0f0970bf1d6705bf181c0b83a32622d848526a50ce35e847fd3439c8a8c5abf765def814d6a9e549712897e7856a07dbc827ab37ed12551efad33 + languageName: node + linkType: hard + "@napi-rs/xattr-android-arm-eabi@npm:1.0.1": version: 1.0.1 resolution: "@napi-rs/xattr-android-arm-eabi@npm:1.0.1" @@ -14501,6 +14776,20 @@ __metadata: languageName: node linkType: hard +"@tailwindcss/vite@npm:^4.0.6": + version: 4.0.6 + resolution: "@tailwindcss/vite@npm:4.0.6" + dependencies: + "@tailwindcss/node": "npm:^4.0.6" + "@tailwindcss/oxide": "npm:^4.0.6" + lightningcss: "npm:^1.29.1" + tailwindcss: "npm:4.0.6" + peerDependencies: + vite: ^5.2.0 || ^6 + checksum: 10/b01050554d1b8b0234ad71fdca4272a6f6f084541b298114a3e14f91ca081083f656f67acc779ecb5dc573317b736c06fab9358905395f587ff51dad5eb8ccff + languageName: node + linkType: hard + "@tanstack/react-table@npm:^8.20.5": version: 8.21.2 resolution: "@tanstack/react-table@npm:8.21.2" @@ -14806,7 +15095,7 @@ __metadata: languageName: node linkType: hard -"@types/babel__core@npm:^7.18.0": +"@types/babel__core@npm:^7.18.0, @types/babel__core@npm:^7.20.5": version: 7.20.5 resolution: "@types/babel__core@npm:7.20.5" dependencies: @@ -15038,6 +15327,15 @@ __metadata: languageName: node linkType: hard +"@types/estree-jsx@npm:^1.0.0": + version: 1.0.5 + resolution: "@types/estree-jsx@npm:1.0.5" + dependencies: + "@types/estree": "npm:*" + checksum: 10/a028ab0cd7b2950168a05c6a86026eb3a36a54a4adfae57f13911d7b49dffe573d9c2b28421b2d029b49b3d02fcd686611be2622dc3dad6d9791166c083f6008 + languageName: node + linkType: hard + "@types/estree@npm:*, @types/estree@npm:1.0.6, @types/estree@npm:^1.0.0, @types/estree@npm:^1.0.6": version: 1.0.6 resolution: "@types/estree@npm:1.0.6" @@ -15081,7 +15379,7 @@ __metadata: languageName: node linkType: hard -"@types/express@npm:^4.17.13, @types/express@npm:^4.17.21": +"@types/express@npm:^4, @types/express@npm:^4.17.13, @types/express@npm:^4.17.21": version: 4.17.21 resolution: "@types/express@npm:4.17.21" dependencies: @@ -15102,7 +15400,7 @@ __metadata: languageName: node linkType: hard -"@types/fs-extra@npm:^11.0.4": +"@types/fs-extra@npm:^11, @types/fs-extra@npm:^11.0.4": version: 11.0.4 resolution: "@types/fs-extra@npm:11.0.4" dependencies: @@ -15121,6 +15419,13 @@ __metadata: languageName: node linkType: hard +"@types/gensync@npm:^1.0.0": + version: 1.0.4 + resolution: "@types/gensync@npm:1.0.4" + checksum: 10/99c3aa0d3f1198973c7e51bea5947b815f3338ce89ce09a39ac8abb41cd844c5b95189da254ea45e50a395fe25fd215664d8ca76c5438814963597afb01f686e + languageName: node + linkType: hard + "@types/graphql-upload@npm:^17.0.0": version: 17.0.0 resolution: "@types/graphql-upload@npm:17.0.0" @@ -15409,6 +15714,15 @@ __metadata: languageName: node linkType: hard +"@types/multer@npm:^1": + version: 1.4.12 + resolution: "@types/multer@npm:1.4.12" + dependencies: + "@types/express": "npm:*" + checksum: 10/3d2b32da58ddd67f972d4ef1021492f78d65f33f936b6fb25dd461bb6cc7b03bfd1de1a11562c4310680dac8054e4398038db51767a0ffbf1fe62457b3706e95 + languageName: node + linkType: hard + "@types/mustache@npm:^4.2.5": version: 4.2.5 resolution: "@types/mustache@npm:4.2.5" @@ -15686,6 +16000,24 @@ __metadata: languageName: node linkType: hard +"@types/socket.io-client@npm:^3.0.0": + version: 3.0.0 + resolution: "@types/socket.io-client@npm:3.0.0" + dependencies: + socket.io-client: "npm:*" + checksum: 10/6eef7529afc1d732fb4091864aa6396a2f676ad25eda0035dd1f2c228cbcaf8520b15236ee0f2c714e061307eeab2b6c8edad637bc83a317c326ae70729d88b0 + languageName: node + linkType: hard + +"@types/socket.io@npm:^3.0.2": + version: 3.0.2 + resolution: "@types/socket.io@npm:3.0.2" + dependencies: + socket.io: "npm:*" + checksum: 10/168cf77e48b466582bd69d1c25930bb657225e742288d658a03a1e8af254350daa61f9441ec2be0d8354f1dcc747966e4342000bce875549fe54ec03e2fafec3 + languageName: node + linkType: hard + "@types/sockjs@npm:^0.3.36": version: 0.3.36 resolution: "@types/sockjs@npm:0.3.36" @@ -15761,6 +16093,13 @@ __metadata: languageName: node linkType: hard +"@types/unist@npm:^2.0.0": + version: 2.0.11 + resolution: "@types/unist@npm:2.0.11" + checksum: 10/6d436e832bc35c6dde9f056ac515ebf2b3384a1d7f63679d12358766f9b313368077402e9c1126a14d827f10370a5485e628bf61aa91117cf4fc882423191a4e + languageName: node + linkType: hard + "@types/uuid@npm:^10.0.0": version: 10.0.0 resolution: "@types/uuid@npm:10.0.0" @@ -16053,6 +16392,21 @@ __metadata: languageName: node linkType: hard +"@vitejs/plugin-react@npm:^4.3.4": + version: 4.3.4 + resolution: "@vitejs/plugin-react@npm:4.3.4" + dependencies: + "@babel/core": "npm:^7.26.0" + "@babel/plugin-transform-react-jsx-self": "npm:^7.25.9" + "@babel/plugin-transform-react-jsx-source": "npm:^7.25.9" + "@types/babel__core": "npm:^7.20.5" + react-refresh: "npm:^0.14.2" + peerDependencies: + vite: ^4.2.0 || ^5.0.0 || ^6.0.0 + checksum: 10/3b220908ed9b7b96a380a9c53e82fb428ca1f76b798ab59d1c63765bdff24de61b4778dd3655952b7d3d922645aea2d97644503b879aba6e3fcf467605b9913d + languageName: node + linkType: hard + "@vitest/browser@npm:3.0.6": version: 3.0.6 resolution: "@vitest/browser@npm:3.0.6" @@ -18239,6 +18593,13 @@ __metadata: languageName: node linkType: hard +"character-reference-invalid@npm:^2.0.0": + version: 2.0.1 + resolution: "character-reference-invalid@npm:2.0.1" + checksum: 10/98d3b1a52ae510b7329e6ee7f6210df14f1e318c5415975d4c9e7ee0ef4c07875d47c6e74230c64551f12f556b4a8ccc24d9f3691a2aa197019e72a95e9297ee + languageName: node + linkType: hard + "chardet@npm:^0.7.0": version: 0.7.0 resolution: "chardet@npm:0.7.0" @@ -18301,7 +18662,7 @@ __metadata: languageName: node linkType: hard -"chokidar@npm:4.0.3, chokidar@npm:^4.0.1": +"chokidar@npm:4.0.3, chokidar@npm:^4.0.1, chokidar@npm:^4.0.3": version: 4.0.3 resolution: "chokidar@npm:4.0.3" dependencies: @@ -20978,6 +21339,89 @@ __metadata: languageName: node linkType: hard +"esbuild@npm:~0.23.0": + version: 0.23.1 + resolution: "esbuild@npm:0.23.1" + dependencies: + "@esbuild/aix-ppc64": "npm:0.23.1" + "@esbuild/android-arm": "npm:0.23.1" + "@esbuild/android-arm64": "npm:0.23.1" + "@esbuild/android-x64": "npm:0.23.1" + "@esbuild/darwin-arm64": "npm:0.23.1" + "@esbuild/darwin-x64": "npm:0.23.1" + "@esbuild/freebsd-arm64": "npm:0.23.1" + "@esbuild/freebsd-x64": "npm:0.23.1" + "@esbuild/linux-arm": "npm:0.23.1" + "@esbuild/linux-arm64": "npm:0.23.1" + "@esbuild/linux-ia32": "npm:0.23.1" + "@esbuild/linux-loong64": "npm:0.23.1" + "@esbuild/linux-mips64el": "npm:0.23.1" + "@esbuild/linux-ppc64": "npm:0.23.1" + "@esbuild/linux-riscv64": "npm:0.23.1" + "@esbuild/linux-s390x": "npm:0.23.1" + "@esbuild/linux-x64": "npm:0.23.1" + "@esbuild/netbsd-x64": "npm:0.23.1" + "@esbuild/openbsd-arm64": "npm:0.23.1" + "@esbuild/openbsd-x64": "npm:0.23.1" + "@esbuild/sunos-x64": "npm:0.23.1" + "@esbuild/win32-arm64": "npm:0.23.1" + "@esbuild/win32-ia32": "npm:0.23.1" + "@esbuild/win32-x64": "npm:0.23.1" + dependenciesMeta: + "@esbuild/aix-ppc64": + optional: true + "@esbuild/android-arm": + optional: true + "@esbuild/android-arm64": + optional: true + "@esbuild/android-x64": + optional: true + "@esbuild/darwin-arm64": + optional: true + "@esbuild/darwin-x64": + optional: true + "@esbuild/freebsd-arm64": + optional: true + "@esbuild/freebsd-x64": + optional: true + "@esbuild/linux-arm": + optional: true + "@esbuild/linux-arm64": + optional: true + "@esbuild/linux-ia32": + optional: true + "@esbuild/linux-loong64": + optional: true + "@esbuild/linux-mips64el": + optional: true + "@esbuild/linux-ppc64": + optional: true + "@esbuild/linux-riscv64": + optional: true + "@esbuild/linux-s390x": + optional: true + "@esbuild/linux-x64": + optional: true + "@esbuild/netbsd-x64": + optional: true + "@esbuild/openbsd-arm64": + optional: true + "@esbuild/openbsd-x64": + optional: true + "@esbuild/sunos-x64": + optional: true + "@esbuild/win32-arm64": + optional: true + "@esbuild/win32-ia32": + optional: true + "@esbuild/win32-x64": + optional: true + bin: + esbuild: bin/esbuild + checksum: 10/f55fbd0bfb0f86ce67a6d2c6f6780729d536c330999ecb9f5a38d578fb9fda820acbbc67d6d1d377eed8fed50fc38f14ff9cb014f86dafab94269a7fb2177018 + languageName: node + linkType: hard + "escalade@npm:^3.1.1, escalade@npm:^3.2.0": version: 3.2.0 resolution: "escalade@npm:3.2.0" @@ -21330,6 +21774,13 @@ __metadata: languageName: node linkType: hard +"estree-util-is-identifier-name@npm:^3.0.0": + version: 3.0.0 + resolution: "estree-util-is-identifier-name@npm:3.0.0" + checksum: 10/cdc9187614fdb269d714eddfdf72c270a79daa9ed51e259bb78527983be6dcc68da6a914ccc41175b662194c67fbd2a1cd262f85fac1eef7111cfddfaf6f77f8 + languageName: node + linkType: hard + "estree-walker@npm:2.0.2, estree-walker@npm:^2.0.2": version: 2.0.2 resolution: "estree-walker@npm:2.0.2" @@ -21542,6 +21993,15 @@ __metadata: languageName: node linkType: hard +"express-rate-limit@npm:^7.1.5": + version: 7.5.0 + resolution: "express-rate-limit@npm:7.5.0" + peerDependencies: + express: ^4.11 || 5 || ^5.0.0-beta.1 + checksum: 10/eff34c83bf586789933a332a339b66649e2cca95c8e977d193aa8bead577d3182ac9f0e9c26f39389287539b8038890ff023f910b54ebb506a26a2ce135b92ca + languageName: node + linkType: hard + "express@npm:4.21.2, express@npm:^4.21.1, express@npm:^4.21.2": version: 4.21.2 resolution: "express@npm:4.21.2" @@ -22220,7 +22680,7 @@ __metadata: languageName: node linkType: hard -"fs-extra@npm:^11.1.0, fs-extra@npm:^11.1.1, fs-extra@npm:^11.2.0": +"fs-extra@npm:^11.1.0, fs-extra@npm:^11.1.1, fs-extra@npm:^11.2.0, fs-extra@npm:^11.3.0": version: 11.3.0 resolution: "fs-extra@npm:11.3.0" dependencies: @@ -23145,6 +23605,29 @@ __metadata: languageName: node linkType: hard +"hast-util-to-jsx-runtime@npm:^2.0.0": + version: 2.3.2 + resolution: "hast-util-to-jsx-runtime@npm:2.3.2" + dependencies: + "@types/estree": "npm:^1.0.0" + "@types/hast": "npm:^3.0.0" + "@types/unist": "npm:^3.0.0" + comma-separated-tokens: "npm:^2.0.0" + devlop: "npm:^1.0.0" + estree-util-is-identifier-name: "npm:^3.0.0" + hast-util-whitespace: "npm:^3.0.0" + mdast-util-mdx-expression: "npm:^2.0.0" + mdast-util-mdx-jsx: "npm:^3.0.0" + mdast-util-mdxjs-esm: "npm:^2.0.0" + property-information: "npm:^6.0.0" + space-separated-tokens: "npm:^2.0.0" + style-to-object: "npm:^1.0.0" + unist-util-position: "npm:^5.0.0" + vfile-message: "npm:^4.0.0" + checksum: 10/3d72f83e2d8c29adc6576d2c6b41479902fd51fac8cfb2b67c35fd68fcb9c25c274699442e4dee901a7ab926a0ff6851713ed5d92448ac09ae0f10daf293476c + languageName: node + linkType: hard + "hast-util-whitespace@npm:^3.0.0": version: 3.0.0 resolution: "hast-util-whitespace@npm:3.0.0" @@ -23326,6 +23809,13 @@ __metadata: languageName: node linkType: hard +"html-url-attributes@npm:^3.0.0": + version: 3.0.1 + resolution: "html-url-attributes@npm:3.0.1" + checksum: 10/494074c2f730c5c0e517aa1b10111fb36732534a2d2b70427582c4a615472b47da472cf3a17562cc653826d378d20960f2783e0400f4f7cf0c3c2d91c6188d13 + languageName: node + linkType: hard + "html-validate@npm:^9.0.0": version: 9.3.0 resolution: "html-validate@npm:9.3.0" @@ -23894,6 +24384,13 @@ __metadata: languageName: node linkType: hard +"inline-style-parser@npm:0.2.4": + version: 0.2.4 + resolution: "inline-style-parser@npm:0.2.4" + checksum: 10/80814479d1f3c9cbd102f9de4cd6558cf43cc2e48640e81c4371c3634f1e8b6dfeb2f21063cfa31d46cc83e834c20cd59ed9eeed9bfd45ef5bc02187ad941faf + languageName: node + linkType: hard + "input-otp@npm:^1.4.1": version: 1.4.2 resolution: "input-otp@npm:1.4.2" @@ -24021,6 +24518,23 @@ __metadata: languageName: node linkType: hard +"is-alphabetical@npm:^2.0.0": + version: 2.0.1 + resolution: "is-alphabetical@npm:2.0.1" + checksum: 10/56207db8d9de0850f0cd30f4966bf731eb82cedfe496cbc2e97e7c3bacaf66fc54a972d2d08c0d93bb679cb84976a05d24c5ad63de56fabbfc60aadae312edaa + languageName: node + linkType: hard + +"is-alphanumerical@npm:^2.0.0": + version: 2.0.1 + resolution: "is-alphanumerical@npm:2.0.1" + dependencies: + is-alphabetical: "npm:^2.0.0" + is-decimal: "npm:^2.0.0" + checksum: 10/87acc068008d4c9c4e9f5bd5e251041d42e7a50995c77b1499cf6ed248f971aadeddb11f239cabf09f7975ee58cac7a48ffc170b7890076d8d227b24a68663c9 + languageName: node + linkType: hard + "is-arguments@npm:@nolyfill/is-arguments@^1": version: 1.0.29 resolution: "@nolyfill/is-arguments@npm:1.0.29" @@ -24080,6 +24594,13 @@ __metadata: languageName: node linkType: hard +"is-decimal@npm:^2.0.0": + version: 2.0.1 + resolution: "is-decimal@npm:2.0.1" + checksum: 10/97132de7acdce77caa7b797632970a2ecd649a88e715db0e4dbc00ab0708b5e7574ba5903962c860cd4894a14fd12b100c0c4ac8aed445cf6f55c6cf747a4158 + languageName: node + linkType: hard + "is-docker@npm:^2.0.0, is-docker@npm:^2.1.1": version: 2.2.1 resolution: "is-docker@npm:2.2.1" @@ -24151,6 +24672,13 @@ __metadata: languageName: node linkType: hard +"is-hexadecimal@npm:^2.0.0": + version: 2.0.1 + resolution: "is-hexadecimal@npm:2.0.1" + checksum: 10/66a2ea85994c622858f063f23eda506db29d92b52580709eb6f4c19550552d4dcf3fb81952e52f7cf972097237959e00adc7bb8c9400cd12886e15bf06145321 + languageName: node + linkType: hard + "is-inside-container@npm:^1.0.0": version: 1.0.0 resolution: "is-inside-container@npm:1.0.0" @@ -26165,6 +26693,54 @@ __metadata: languageName: node linkType: hard +"mdast-util-mdx-expression@npm:^2.0.0": + version: 2.0.1 + resolution: "mdast-util-mdx-expression@npm:2.0.1" + dependencies: + "@types/estree-jsx": "npm:^1.0.0" + "@types/hast": "npm:^3.0.0" + "@types/mdast": "npm:^4.0.0" + devlop: "npm:^1.0.0" + mdast-util-from-markdown: "npm:^2.0.0" + mdast-util-to-markdown: "npm:^2.0.0" + checksum: 10/70e860f8ee22c4f478449942750055d649d4380bf43b235d0710af510189d285fb057e401d20b59596d9789f4e270fce08ca892dc849676f9e3383b991d52485 + languageName: node + linkType: hard + +"mdast-util-mdx-jsx@npm:^3.0.0": + version: 3.2.0 + resolution: "mdast-util-mdx-jsx@npm:3.2.0" + dependencies: + "@types/estree-jsx": "npm:^1.0.0" + "@types/hast": "npm:^3.0.0" + "@types/mdast": "npm:^4.0.0" + "@types/unist": "npm:^3.0.0" + ccount: "npm:^2.0.0" + devlop: "npm:^1.1.0" + mdast-util-from-markdown: "npm:^2.0.0" + mdast-util-to-markdown: "npm:^2.0.0" + parse-entities: "npm:^4.0.0" + stringify-entities: "npm:^4.0.0" + unist-util-stringify-position: "npm:^4.0.0" + vfile-message: "npm:^4.0.0" + checksum: 10/62cd650a522e5d72ea6afd6d4a557fc86525b802d097a29a2fbe17d22e7b97c502a580611873e4d685777fe77c6ff8d39fb6e37d026b3acbc86c3b24927f4ad9 + languageName: node + linkType: hard + +"mdast-util-mdxjs-esm@npm:^2.0.0": + version: 2.0.1 + resolution: "mdast-util-mdxjs-esm@npm:2.0.1" + dependencies: + "@types/estree-jsx": "npm:^1.0.0" + "@types/hast": "npm:^3.0.0" + "@types/mdast": "npm:^4.0.0" + devlop: "npm:^1.0.0" + mdast-util-from-markdown: "npm:^2.0.0" + mdast-util-to-markdown: "npm:^2.0.0" + checksum: 10/05474226e163a3f407fccb5780b0d8585a95e548e5da4a85227df43f281b940c7941a9a9d4af1be4f885fe554731647addb057a728e87aa1f503ff9cc72c9163 + languageName: node + linkType: hard + "mdast-util-phrasing@npm:^4.0.0": version: 4.1.0 resolution: "mdast-util-phrasing@npm:4.1.0" @@ -27223,6 +27799,21 @@ __metadata: languageName: node linkType: hard +"multer@npm:^1.4.5-lts.1": + version: 1.4.5-lts.1 + resolution: "multer@npm:1.4.5-lts.1" + dependencies: + append-field: "npm:^1.0.0" + busboy: "npm:^1.0.0" + concat-stream: "npm:^1.5.2" + mkdirp: "npm:^0.5.4" + object-assign: "npm:^4.1.1" + type-is: "npm:^1.6.4" + xtend: "npm:^4.0.0" + checksum: 10/957c09956f3b7f79d8586cac5e2a50e9a5c3011eb841667b5e4590c5f31d9464f5b46aecd399c83e183a15b88b019cccf0e4fa5620db40bf16b9e3af7fab3ac6 + languageName: node + linkType: hard + "multicast-dns@npm:^7.2.5": version: 7.2.5 resolution: "multicast-dns@npm:7.2.5" @@ -28026,7 +28617,7 @@ __metadata: languageName: node linkType: hard -"openai@npm:^4.83.0": +"openai@npm:^4.83.0, openai@npm:^4.85.1": version: 4.85.1 resolution: "openai@npm:4.85.1" dependencies: @@ -28387,6 +28978,21 @@ __metadata: languageName: node linkType: hard +"parse-entities@npm:^4.0.0": + version: 4.0.2 + resolution: "parse-entities@npm:4.0.2" + dependencies: + "@types/unist": "npm:^2.0.0" + character-entities-legacy: "npm:^3.0.0" + character-reference-invalid: "npm:^2.0.0" + decode-named-character-reference: "npm:^1.0.0" + is-alphanumerical: "npm:^2.0.0" + is-decimal: "npm:^2.0.0" + is-hexadecimal: "npm:^2.0.0" + checksum: 10/b0ce693d0b3d7ed1cea6fe814e6e077c71532695f01178e846269e9a2bc2f7ff34ca4bb8db80b48af0451100f25bb010df6591c9bb6306e4680ccb423d1e4038 + languageName: node + linkType: hard + "parse-filepath@npm:^1.0.2": version: 1.0.2 resolution: "parse-filepath@npm:1.0.2" @@ -29982,6 +30588,27 @@ __metadata: languageName: node linkType: hard +"react-markdown@npm:^9.0.3": + version: 9.0.3 + resolution: "react-markdown@npm:9.0.3" + dependencies: + "@types/hast": "npm:^3.0.0" + devlop: "npm:^1.0.0" + hast-util-to-jsx-runtime: "npm:^2.0.0" + html-url-attributes: "npm:^3.0.0" + mdast-util-to-hast: "npm:^13.0.0" + remark-parse: "npm:^11.0.0" + remark-rehype: "npm:^11.0.0" + unified: "npm:^11.0.0" + unist-util-visit: "npm:^5.0.0" + vfile: "npm:^6.0.0" + peerDependencies: + "@types/react": ">=18" + react: ">=18" + checksum: 10/b97eb9a61762762043263286ece030bd878acabe24bbf433767a9518cb18e434e26a75b1b810a7cb966e304ddb4e16bd4a15edcc808113b11b4fb85a68d99e8d + languageName: node + linkType: hard + "react-paginate@npm:^8.2.0": version: 8.3.0 resolution: "react-paginate@npm:8.3.0" @@ -30002,6 +30629,13 @@ __metadata: languageName: node linkType: hard +"react-refresh@npm:^0.14.2": + version: 0.14.2 + resolution: "react-refresh@npm:0.14.2" + checksum: 10/512abf97271ab8623486061be04b608c39d932e3709f9af1720b41573415fa4993d0009fa5138b6705b60a98f4102f744d4e26c952b14f41a0e455521c6be4cc + languageName: node + linkType: hard + "react-refresh@npm:^0.16.0": version: 0.16.0 resolution: "react-refresh@npm:0.16.0" @@ -30502,6 +31136,19 @@ __metadata: languageName: node linkType: hard +"remark-rehype@npm:^11.0.0": + version: 11.1.1 + resolution: "remark-rehype@npm:11.1.1" + dependencies: + "@types/hast": "npm:^3.0.0" + "@types/mdast": "npm:^4.0.0" + mdast-util-to-hast: "npm:^13.0.0" + unified: "npm:^11.0.0" + vfile: "npm:^6.0.0" + checksum: 10/39404bd19c57b2b69660be7e3d587ddb2240495845d42fad3bcc506c9c132d07abacb0a20182b73c530857b2da0c463ad5658382b448243ce432152ab49af08d + languageName: node + linkType: hard + "remark-stringify@npm:^11.0.0": version: 11.0.0 resolution: "remark-stringify@npm:11.0.0" @@ -31713,7 +32360,7 @@ __metadata: languageName: node linkType: hard -"socket.io-client@npm:^4.8.1": +"socket.io-client@npm:*, socket.io-client@npm:^4.7.4, socket.io-client@npm:^4.8.1": version: 4.8.1 resolution: "socket.io-client@npm:4.8.1" dependencies: @@ -31735,7 +32382,7 @@ __metadata: languageName: node linkType: hard -"socket.io@npm:4.8.1, socket.io@npm:^4.8.1": +"socket.io@npm:*, socket.io@npm:4.8.1, socket.io@npm:^4.7.4, socket.io@npm:^4.8.1": version: 4.8.1 resolution: "socket.io@npm:4.8.1" dependencies: @@ -32370,6 +33017,15 @@ __metadata: languageName: node linkType: hard +"style-to-object@npm:^1.0.0": + version: 1.0.8 + resolution: "style-to-object@npm:1.0.8" + dependencies: + inline-style-parser: "npm:0.2.4" + checksum: 10/530b067325e3119bfaf75bdbe25cc86b02b559db00d881a74b98a2d5bb10ac953d1b455ed90c825963cf3b4bdaa1bda45f406d78d987391434b8d8ab3835df4e + languageName: node + linkType: hard + "styled-jsx@npm:5.1.6": version: 5.1.6 resolution: "styled-jsx@npm:5.1.6" @@ -32547,7 +33203,7 @@ __metadata: languageName: node linkType: hard -"swr@npm:2.3.2, swr@npm:^2.2.5": +"swr@npm:2.3.2, swr@npm:^2.2.5, swr@npm:^2.3.2": version: 2.3.2 resolution: "swr@npm:2.3.2" dependencies: @@ -32616,7 +33272,7 @@ __metadata: languageName: node linkType: hard -"tailwindcss@npm:4.0.6, tailwindcss@npm:^4.0.0": +"tailwindcss@npm:4.0.6, tailwindcss@npm:^4.0.0, tailwindcss@npm:^4.0.6": version: 4.0.6 resolution: "tailwindcss@npm:4.0.6" checksum: 10/b1141d98730b89164a38dab2c6fe0150a0910ab4ab45865d345b1fbf9eb9c0893f8cbe1392b7bcf59e52eb9f2e9750970b1d912f2ac4458e4b9e4cabe4518798 @@ -33223,6 +33879,22 @@ __metadata: languageName: node linkType: hard +"tsx@npm:^4.19.2": + version: 4.19.2 + resolution: "tsx@npm:4.19.2" + dependencies: + esbuild: "npm:~0.23.0" + fsevents: "npm:~2.3.3" + get-tsconfig: "npm:^4.7.5" + dependenciesMeta: + fsevents: + optional: true + bin: + tsx: dist/cli.mjs + checksum: 10/4c5610ed1fb2f80d766681f8ac7827e1e8118dfe354c18f74800691f3ef1e9ed676a29842ab818806bcf8613cdc97c6af84b5645e768ddb7f4b0527b9100deda + languageName: node + linkType: hard + "tunnel-agent@npm:^0.6.0": version: 0.6.0 resolution: "tunnel-agent@npm:0.6.0"