Jayden/AFFiNE-Mirror
1
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-02-16 13:57:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(native): media capture (#9992)

Browse Source
This commit is contained in:
Brooooooklyn
2025-02-25 06:51:56 +00:00
parent 2ec7de7e32
commit 5dbffba08d
46 changed files with 5791 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/frontend/native/media_capture/src/lib.rs Normal file
View File

@@ -0,0 +1,4 @@
#[cfg(target_os = "macos")]
pub mod macos;
#[cfg(target_os = "macos")]
pub(crate) use macos::*;

282
packages/frontend/native/media_capture/src/macos/audio_stream_basic_desc.rs Normal file
View File

@@ -0,0 +1,282 @@
use std::{fmt::Display, mem, ptr};
use coreaudio::sys::{
kAudioHardwareNoError, kAudioObjectPropertyElementMain, kAudioObjectPropertyScopeGlobal,
kAudioTapPropertyFormat, AudioObjectGetPropertyData, AudioObjectID, AudioObjectPropertyAddress,
};
use objc2::{Encode, Encoding, RefEncode};
use crate::error::CoreAudioError;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u32)]
pub enum AudioFormatID {
LinearPcm = 0x6C70636D, // 'lpcm'
Ac3 = 0x61632D33, // 'ac-3'
Ac360958 = 0x63616333, // 'cac3'
AppleIma4 = 0x696D6134, // 'ima4'
Mpeg4Aac = 0x61616320, // 'aac '
Mpeg4Celp = 0x63656C70, // 'celp'
Mpeg4Hvxc = 0x68767863, // 'hvxc'
Mpeg4TwinVq = 0x74777671, // 'twvq'
Mace3 = 0x4D414333, // 'MAC3'
Mace6 = 0x4D414336, // 'MAC6'
ULaw = 0x756C6177, // 'ulaw'
ALaw = 0x616C6177, // 'alaw'
QDesign = 0x51444D43, // 'QDMC'
QDesign2 = 0x51444D32, // 'QDM2'
Qualcomm = 0x51636C70, // 'Qclp'
MpegLayer1 = 0x2E6D7031, // '.mp1'
MpegLayer2 = 0x2E6D7032, // '.mp2'
MpegLayer3 = 0x2E6D7033, // '.mp3'
TimeCode = 0x74696D65, // 'time'
MidiStream = 0x6D696469, // 'midi'
ParameterValueStream = 0x61707673, // 'apvs'
AppleLossless = 0x616C6163, // 'alac'
Mpeg4AacHe = 0x61616368, // 'aach'
Mpeg4AacLd = 0x6161636C, // 'aacl'
Mpeg4AacEld = 0x61616365, // 'aace'
Mpeg4AacEldSbr = 0x61616366, // 'aacf'
Mpeg4AacEldV2 = 0x61616367, // 'aacg'
Mpeg4AacHeV2 = 0x61616370, // 'aacp'
Mpeg4AacSpatial = 0x61616373, // 'aacs'
MpegdUsac = 0x75736163, // 'usac'
Amr = 0x73616D72, // 'samr'
AmrWb = 0x73617762, // 'sawb'
Audible = 0x41554442, // 'AUDB'
ILbc = 0x696C6263, // 'ilbc'
DviIntelIma = 0x6D730011,
MicrosoftGsm = 0x6D730031,
Aes3 = 0x61657333, // 'aes3'
EnhancedAc3 = 0x65632D33, // 'ec-3'
Flac = 0x666C6163, // 'flac'
Opus = 0x6F707573, // 'opus'
Apac = 0x61706163, // 'apac'
Unknown = 0x00000000,
}
impl From<u32> for AudioFormatID {
fn from(value: u32) -> Self {
match value {
0x6C70636D => Self::LinearPcm,
0x61632D33 => Self::Ac3,
0x63616333 => Self::Ac360958,
0x696D6134 => Self::AppleIma4,
0x61616320 => Self::Mpeg4Aac,
0x63656C70 => Self::Mpeg4Celp,
0x68767863 => Self::Mpeg4Hvxc,
0x74777671 => Self::Mpeg4TwinVq,
0x4D414333 => Self::Mace3,
0x4D414336 => Self::Mace6,
0x756C6177 => Self::ULaw,
0x616C6177 => Self::ALaw,
0x51444D43 => Self::QDesign,
0x51444D32 => Self::QDesign2,
0x51636C70 => Self::Qualcomm,
0x2E6D7031 => Self::MpegLayer1,
0x2E6D7032 => Self::MpegLayer2,
0x2E6D7033 => Self::MpegLayer3,
0x74696D65 => Self::TimeCode,
0x6D696469 => Self::MidiStream,
0x61707673 => Self::ParameterValueStream,
0x616C6163 => Self::AppleLossless,
0x61616368 => Self::Mpeg4AacHe,
0x6161636C => Self::Mpeg4AacLd,
0x61616365 => Self::Mpeg4AacEld,
0x61616366 => Self::Mpeg4AacEldSbr,
0x61616367 => Self::Mpeg4AacEldV2,
0x61616370 => Self::Mpeg4AacHeV2,
0x61616373 => Self::Mpeg4AacSpatial,
0x75736163 => Self::MpegdUsac,
0x73616D72 => Self::Amr,
0x73617762 => Self::AmrWb,
0x41554442 => Self::Audible,
0x696C6263 => Self::ILbc,
0x6D730011 => Self::DviIntelIma,
0x6D730031 => Self::MicrosoftGsm,
0x61657333 => Self::Aes3,
0x65632D33 => Self::EnhancedAc3,
0x666C6163 => Self::Flac,
0x6F707573 => Self::Opus,
0x61706163 => Self::Apac,
_ => Self::Unknown,
}
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub struct AudioFormatFlags(pub u32);
#[allow(unused)]
impl AudioFormatFlags {
pub const IS_FLOAT: u32 = 1 << 0;
pub const IS_BIG_ENDIAN: u32 = 1 << 1;
pub const IS_SIGNED_INTEGER: u32 = 1 << 2;
pub const IS_PACKED: u32 = 1 << 3;
pub const IS_ALIGNED_HIGH: u32 = 1 << 4;
pub const IS_NON_INTERLEAVED: u32 = 1 << 5;
pub const IS_NON_MIXABLE: u32 = 1 << 6;
pub const ARE_ALL_CLEAR: u32 = 0x80000000;
pub const LINEAR_PCM_IS_FLOAT: u32 = Self::IS_FLOAT;
pub const LINEAR_PCM_IS_BIG_ENDIAN: u32 = Self::IS_BIG_ENDIAN;
pub const LINEAR_PCM_IS_SIGNED_INTEGER: u32 = Self::IS_SIGNED_INTEGER;
pub const LINEAR_PCM_IS_PACKED: u32 = Self::IS_PACKED;
pub const LINEAR_PCM_IS_ALIGNED_HIGH: u32 = Self::IS_ALIGNED_HIGH;
pub const LINEAR_PCM_IS_NON_INTERLEAVED: u32 = Self::IS_NON_INTERLEAVED;
pub const LINEAR_PCM_IS_NON_MIXABLE: u32 = Self::IS_NON_MIXABLE;
pub const LINEAR_PCM_SAMPLE_FRACTION_SHIFT: u32 = 7;
pub const LINEAR_PCM_SAMPLE_FRACTION_MASK: u32 = 0x3F << Self::LINEAR_PCM_SAMPLE_FRACTION_SHIFT;
pub const LINEAR_PCM_ARE_ALL_CLEAR: u32 = Self::ARE_ALL_CLEAR;
pub const APPLE_LOSSLESS_FORMAT_FLAG_16_BIT_SOURCE_DATA: u32 = 1;
pub const APPLE_LOSSLESS_FORMAT_FLAG_20_BIT_SOURCE_DATA: u32 = 2;
pub const APPLE_LOSSLESS_FORMAT_FLAG_24_BIT_SOURCE_DATA: u32 = 3;
pub const APPLE_LOSSLESS_FORMAT_FLAG_32_BIT_SOURCE_DATA: u32 = 4;
}
impl std::fmt::Display for AudioFormatFlags {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut flags = Vec::new();
if self.0 & Self::IS_FLOAT != 0 {
flags.push("FLOAT");
}
if self.0 & Self::IS_BIG_ENDIAN != 0 {
flags.push("BIG_ENDIAN");
}
if self.0 & Self::IS_SIGNED_INTEGER != 0 {
flags.push("SIGNED_INTEGER");
}
if self.0 & Self::IS_PACKED != 0 {
flags.push("PACKED");
}
if self.0 & Self::IS_ALIGNED_HIGH != 0 {
flags.push("ALIGNED_HIGH");
}
if self.0 & Self::IS_NON_INTERLEAVED != 0 {
flags.push("NON_INTERLEAVED");
}
if self.0 & Self::IS_NON_MIXABLE != 0 {
flags.push("NON_MIXABLE");
}
if self.0 & Self::ARE_ALL_CLEAR != 0 {
flags.push("ALL_CLEAR");
}
if flags.is_empty() {
write!(f, "NONE")
} else {
write!(f, "{}", flags.join(" | "))
}
}
}
impl std::fmt::Debug for AudioFormatFlags {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "AudioFormatFlags({})", self)
}
}
impl From<u32> for AudioFormatFlags {
fn from(value: u32) -> Self {
Self(value)
}
}
/// [Apple's documentation](https://developer.apple.com/documentation/coreaudiotypes/audiostreambasicdescription?language=objc)
#[repr(C)]
#[derive(Clone, Copy, Debug, PartialEq)]
#[allow(non_snake_case)]
pub struct AudioStreamBasicDescription {
pub mSampleRate: f64,
pub mFormatID: u32,
pub mFormatFlags: u32,
pub mBytesPerPacket: u32,
pub mFramesPerPacket: u32,
pub mBytesPerFrame: u32,
pub mChannelsPerFrame: u32,
pub mBitsPerChannel: u32,
pub mReserved: u32,
}
unsafe impl Encode for AudioStreamBasicDescription {
const ENCODING: Encoding = Encoding::Struct(
"AudioStreamBasicDescription",
&[
<f64>::ENCODING,
<u32>::ENCODING,
<u32>::ENCODING,
<u32>::ENCODING,
<u32>::ENCODING,
<u32>::ENCODING,
<u32>::ENCODING,
<u32>::ENCODING,
<u32>::ENCODING,
],
);
}
unsafe impl RefEncode for AudioStreamBasicDescription {
const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);
}
#[derive(Debug, Clone, Copy)]
#[repr(transparent)]
pub struct AudioStreamDescription(pub(crate) AudioStreamBasicDescription);
impl Display for AudioStreamDescription {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"AudioStreamBasicDescription {{ mSampleRate: {}, mFormatID: {:?}, mFormatFlags: {}, \
mBytesPerPacket: {}, mFramesPerPacket: {}, mBytesPerFrame: {}, mChannelsPerFrame: {}, \
mBitsPerChannel: {}, mReserved: {} }}",
self.0.mSampleRate,
AudioFormatID::from(self.0.mFormatID),
AudioFormatFlags(self.0.mFormatFlags),
self.0.mBytesPerPacket,
self.0.mFramesPerPacket,
self.0.mBytesPerFrame,
self.0.mChannelsPerFrame,
self.0.mBitsPerChannel,
self.0.mReserved
)
}
}
pub fn read_audio_stream_basic_description(
tap_id: AudioObjectID,
) -> std::result::Result<AudioStreamDescription, CoreAudioError> {
let mut data_size = mem::size_of::<AudioStreamBasicDescription>();
let address = AudioObjectPropertyAddress {
mSelector: kAudioTapPropertyFormat,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain,
};
let mut data = AudioStreamBasicDescription {
mSampleRate: 0.0,
mFormatID: 0,
mFormatFlags: 0,
mBytesPerPacket: 0,
mFramesPerPacket: 0,
mBytesPerFrame: 0,
mChannelsPerFrame: 0,
mBitsPerChannel: 0,
mReserved: 0,
};
let status = unsafe {
AudioObjectGetPropertyData(
tap_id,
&address,
0,
ptr::null_mut(),
(&mut data_size as *mut usize).cast(),
(&mut data as *mut AudioStreamBasicDescription).cast(),
)
};
if status != kAudioHardwareNoError as i32 {
return Err(CoreAudioError::GetAudioStreamBasicDescriptionFailed(status));
}
Ok(AudioStreamDescription(data))
}

71
packages/frontend/native/media_capture/src/macos/av_audio_file.rs Normal file
View File

@@ -0,0 +1,71 @@
use std::ptr;
use objc2::{
msg_send,
runtime::{AnyClass, AnyObject},
AllocAnyThread,
};
use objc2_foundation::{NSDictionary, NSError, NSNumber, NSString, NSUInteger, NSURL};
use crate::{
av_audio_format::AVAudioFormat, av_audio_pcm_buffer::AVAudioPCMBuffer, error::CoreAudioError,
};
#[allow(unused)]
pub(crate) struct AVAudioFile {
inner: *mut AnyObject,
}
#[allow(unused)]
impl AVAudioFile {
pub(crate) fn new(url: &str, format: &AVAudioFormat) -> Result<Self, CoreAudioError> {
let cls = AnyClass::get(c"AVAudioFile").ok_or(CoreAudioError::AVAudioFileClassNotFound)?;
let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };
if obj.is_null() {
return Err(CoreAudioError::AllocAVAudioFileFailed);
}
let url: &NSURL = &*unsafe { NSURL::fileURLWithPath(&NSString::from_str(url)) };
let settings = &*NSDictionary::from_retained_objects(
&[
&*NSString::from_str("AVFormatIDKey"),
&*NSString::from_str("AVSampleRateKey"),
&*NSString::from_str("AVNumberOfChannelsKey"),
],
&[
NSNumber::initWithUnsignedInt(
NSNumber::alloc(),
format.audio_stream_basic_description.0.mFormatID,
),
NSNumber::initWithDouble(NSNumber::alloc(), format.get_sample_rate()),
NSNumber::initWithUnsignedInt(NSNumber::alloc(), format.get_channel_count()),
],
);
let is_interleaved = format.is_interleaved();
let mut error: *mut NSError = ptr::null_mut();
let common_format: NSUInteger = 1;
let obj: *mut AnyObject = unsafe {
msg_send![
obj,
initForWriting: url,
settings: settings,
commonFormat: common_format,
interleaved: is_interleaved,
error: &mut error
]
};
if obj.is_null() {
return Err(CoreAudioError::InitAVAudioFileFailed);
}
Ok(Self { inner: obj })
}
pub(crate) fn write(&self, buffer: AVAudioPCMBuffer) -> Result<(), CoreAudioError> {
let mut error: *mut NSError = ptr::null_mut();
let success: bool =
unsafe { msg_send![self.inner, writeFromBuffer: buffer.inner, error: &mut error] };
if !success {
return Err(CoreAudioError::WriteAVAudioFileFailed);
}
Ok(())
}
}

95
packages/frontend/native/media_capture/src/macos/av_audio_format.rs Normal file
View File

@@ -0,0 +1,95 @@
use objc2::{
msg_send,
runtime::{AnyClass, AnyObject},
Encode, Encoding, RefEncode,
};
use crate::{audio_stream_basic_desc::AudioStreamDescription, error::CoreAudioError};
#[derive(Debug)]
#[allow(unused)]
pub(crate) struct AVAudioFormat {
pub(crate) inner: AVAudioFormatRef,
pub(crate) audio_stream_basic_description: AudioStreamDescription,
}
#[repr(transparent)]
#[derive(Debug, Clone, Copy)]
pub(crate) struct AVAudioFormatRef(pub(crate) *mut AnyObject);
unsafe impl Encode for AVAudioFormatRef {
const ENCODING: Encoding = Encoding::Struct(
"AVAudioFormat",
&[
Encoding::Double,
Encoding::UInt,
Encoding::Pointer(&Encoding::Struct(
"AVAudioChannelLayout",
&[
Encoding::UInt,
Encoding::UInt,
Encoding::Pointer(&Encoding::Struct(
"AudioChannelLayout",
&[
Encoding::UInt,
Encoding::UInt,
Encoding::Array(
1,
&Encoding::Struct(
"AudioChannelDescription",
&[
Encoding::UInt,
Encoding::UInt,
Encoding::Array(3, &Encoding::Float),
],
),
),
Encoding::UInt,
Encoding::UInt,
],
)),
Encoding::UInt,
],
)),
Encoding::Pointer(&Encoding::Object),
],
);
}
unsafe impl RefEncode for AVAudioFormatRef {
const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);
}
#[allow(unused)]
impl AVAudioFormat {
pub fn new(
audio_stream_basic_description: AudioStreamDescription,
) -> Result<Self, CoreAudioError> {
let cls = AnyClass::get(c"AVAudioFormat").ok_or(CoreAudioError::AVAudioFormatClassNotFound)?;
let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };
if obj.is_null() {
return Err(CoreAudioError::AllocAVAudioFormatFailed);
}
let obj: *mut AnyObject =
unsafe { msg_send![obj, initWithStreamDescription: &audio_stream_basic_description.0] };
if obj.is_null() {
return Err(CoreAudioError::InitAVAudioFormatFailed);
}
Ok(Self {
inner: AVAudioFormatRef(obj),
audio_stream_basic_description,
})
}
pub(crate) fn get_sample_rate(&self) -> f64 {
unsafe { msg_send![self.inner.0, sampleRate] }
}
pub(crate) fn get_channel_count(&self) -> u32 {
unsafe { msg_send![self.inner.0, channelCount] }
}
pub(crate) fn is_interleaved(&self) -> bool {
unsafe { msg_send![self.inner.0, isInterleaved] }
}
}

35
packages/frontend/native/media_capture/src/macos/av_audio_pcm_buffer.rs Normal file
View File

@@ -0,0 +1,35 @@
use block2::RcBlock;
use objc2::{
msg_send,
runtime::{AnyClass, AnyObject},
};
use crate::{av_audio_format::AVAudioFormat, error::CoreAudioError, tap_audio::AudioBufferList};
#[allow(unused)]
pub(crate) struct AVAudioPCMBuffer {
pub(crate) inner: *mut AnyObject,
}
#[allow(unused)]
impl AVAudioPCMBuffer {
pub(crate) fn new(
audio_format: &AVAudioFormat,
buffer_list: *const AudioBufferList,
) -> Result<Self, CoreAudioError> {
let cls =
AnyClass::get(c"AVAudioPCMBuffer").ok_or(CoreAudioError::AVAudioPCMBufferClassNotFound)?;
let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };
if obj.is_null() {
return Err(CoreAudioError::AllocAVAudioPCMBufferFailed);
}
let deallocator = RcBlock::new(|_buffer_list: *const AudioBufferList| {});
let obj: *mut AnyObject = unsafe {
msg_send![obj, initWithPCMFormat: audio_format.inner.0, bufferListNoCopy: buffer_list, deallocator: &*deallocator]
};
if obj.is_null() {
return Err(CoreAudioError::InitAVAudioPCMBufferFailed);
}
Ok(Self { inner: obj })
}
}

84
packages/frontend/native/media_capture/src/macos/ca_tap_description.rs Normal file
View File

@@ -0,0 +1,84 @@
use core_foundation::{
base::{FromVoid, ItemRef},
string::CFString,
};
use coreaudio::sys::AudioObjectID;
use objc2::{
msg_send,
runtime::{AnyClass, AnyObject},
AllocAnyThread,
};
use objc2_foundation::{NSArray, NSNumber, NSString, NSUUID};
use crate::error::CoreAudioError;
pub(crate) struct CATapDescription {
pub(crate) inner: *mut AnyObject,
}
impl CATapDescription {
pub fn init_stereo_mixdown_of_processes(
process: AudioObjectID,
) -> std::result::Result<Self, CoreAudioError> {
let cls =
AnyClass::get(c"CATapDescription").ok_or(CoreAudioError::CATapDescriptionClassNotFound)?;
let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };
if obj.is_null() {
return Err(CoreAudioError::AllocCATapDescriptionFailed);
}
let processes_array =
NSArray::from_retained_slice(&[NSNumber::initWithUnsignedInt(NSNumber::alloc(), process)]);
let obj: *mut AnyObject =
unsafe { msg_send![obj, initStereoMixdownOfProcesses: &*processes_array] };
if obj.is_null() {
return Err(CoreAudioError::InitStereoMixdownOfProcessesFailed);
}
Ok(Self { inner: obj })
}
pub fn init_stereo_global_tap_but_exclude_processes(
processes: &[AudioObjectID],
) -> std::result::Result<Self, CoreAudioError> {
let cls =
AnyClass::get(c"CATapDescription").ok_or(CoreAudioError::CATapDescriptionClassNotFound)?;
let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };
if obj.is_null() {
return Err(CoreAudioError::AllocCATapDescriptionFailed);
}
let processes_array = NSArray::from_retained_slice(
processes
.iter()
.map(|p| NSNumber::initWithUnsignedInt(NSNumber::alloc(), *p))
.collect::<Vec<_>>()
.as_slice(),
);
let obj: *mut AnyObject =
unsafe { msg_send![obj, initStereoMixdownOfProcesses: &*processes_array] };
if obj.is_null() {
return Err(CoreAudioError::InitStereoMixdownOfProcessesFailed);
}
Ok(Self { inner: obj })
}
pub fn get_uuid(&self) -> std::result::Result<ItemRef<CFString>, CoreAudioError> {
let uuid: *mut NSUUID = unsafe { msg_send![self.inner, UUID] };
if uuid.is_null() {
return Err(CoreAudioError::GetCATapDescriptionUUIDFailed);
}
let uuid_string: *mut NSString = unsafe { msg_send![uuid, UUIDString] };
if uuid_string.is_null() {
return Err(CoreAudioError::ConvertUUIDToCFStringFailed);
}
Ok(unsafe { CFString::from_void(uuid_string.cast()) })
}
}
impl Drop for CATapDescription {
fn drop(&mut self) {
unsafe {
let _: () = msg_send![self.inner, release];
}
}
}

66
packages/frontend/native/media_capture/src/macos/device.rs Normal file
View File

@@ -0,0 +1,66 @@
use std::{mem, ptr};
use core_foundation::{base::TCFType, string::CFString};
use coreaudio::sys::{
kAudioDevicePropertyDeviceUID, kAudioHardwareNoError, kAudioObjectPropertyElementMain,
kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, AudioDeviceID,
AudioObjectGetPropertyData, AudioObjectID, AudioObjectPropertyAddress, CFStringRef,
};
use crate::error::CoreAudioError;
pub(crate) fn get_device_uid(
device_id: AudioDeviceID,
) -> std::result::Result<CFString, CoreAudioError> {
let system_output_id = get_device_audio_id(device_id)?;
let address = AudioObjectPropertyAddress {
mSelector: kAudioDevicePropertyDeviceUID,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain,
};
let mut output_uid: CFStringRef = ptr::null_mut();
let mut data_size = mem::size_of::<CFStringRef>();
let status = unsafe {
AudioObjectGetPropertyData(
system_output_id,
&address,
0,
ptr::null_mut(),
(&mut data_size as *mut usize).cast(),
(&mut output_uid as *mut CFStringRef).cast(),
)
};
if status != 0 {
return Err(CoreAudioError::GetDeviceUidFailed(status));
}
Ok(unsafe { CFString::wrap_under_create_rule(output_uid.cast()) })
}
pub(crate) fn get_device_audio_id(
device_id: AudioDeviceID,
) -> std::result::Result<AudioObjectID, CoreAudioError> {
let mut system_output_id: AudioObjectID = 0;
let mut data_size = mem::size_of::<AudioObjectID>();
let address = AudioObjectPropertyAddress {
mSelector: device_id,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain,
};
let status = unsafe {
AudioObjectGetPropertyData(
kAudioObjectSystemObject,
&address,
0,
ptr::null_mut(),
(&mut data_size as *mut usize).cast(),
(&mut system_output_id as *mut AudioObjectID).cast(),
)
};
if status != kAudioHardwareNoError as i32 {
return Err(CoreAudioError::GetDefaultDeviceFailed(status));
}
Ok(system_output_id)
}

81
packages/frontend/native/media_capture/src/macos/error.rs Normal file
View File

@@ -0,0 +1,81 @@
use thiserror::Error;
#[derive(Error, Debug)]
pub enum CoreAudioError {
#[error("Map pid {0} to AudioObjectID failed")]
PidNotFound(i32),
#[error("Create process tap failed, status: {0}")]
CreateProcessTapFailed(i32),
#[error("Get default device failed, status: {0}")]
GetDefaultDeviceFailed(i32),
#[error("Get device uid failed, status: {0}")]
GetDeviceUidFailed(i32),
#[error("Create aggregate device failed, status: {0}")]
CreateAggregateDeviceFailed(i32),
#[error("Get process object list size failed, status: {0}")]
GetProcessObjectListSizeFailed(i32),
#[error("Get process object list failed, status: {0}")]
GetProcessObjectListFailed(i32),
#[error("AudioObjectGetPropertyDataSize failed, status: {0}")]
AudioObjectGetPropertyDataSizeFailed(i32),
#[error("CATapDescription class not found")]
CATapDescriptionClassNotFound,
#[error("Alloc CATapDescription failed")]
AllocCATapDescriptionFailed,
#[error("Call initStereoMixdownOfProcesses on CATapDescription failed")]
InitStereoMixdownOfProcessesFailed,
#[error("Get UUID on CATapDescription failed")]
GetCATapDescriptionUUIDFailed,
#[error("Get mute behavior on CATapDescription failed")]
GetMuteBehaviorFailed,
#[error("Convert UUID to CFString failed")]
ConvertUUIDToCFStringFailed,
#[error("Get AudioStreamBasicDescription failed, status: {0}")]
GetAudioStreamBasicDescriptionFailed(i32),
#[error("AVAudioFormat class not found")]
AVAudioFormatClassNotFound,
#[error("Alloc AVAudioFormat failed")]
AllocAVAudioFormatFailed,
#[error("Init AVAudioFormat failed")]
InitAVAudioFormatFailed,
#[error("Create IOProcIDWithBlock failed, status: {0}")]
CreateIOProcIDWithBlockFailed(i32),
#[error("Get hardware devices failed, status: {0}")]
GetHardwareDevicesFailed(i32),
#[error("AudioDeviceStart failed, status: {0}")]
AudioDeviceStartFailed(i32),
#[error("AudioDeviceStop failed, status: {0}")]
AudioDeviceStopFailed(i32),
#[error("AudioDeviceDestroyIOProcID failed, status: {0}")]
AudioDeviceDestroyIOProcIDFailed(i32),
#[error("AudioHardwareDestroyAggregateDevice failed, status: {0}")]
AudioHardwareDestroyAggregateDeviceFailed(i32),
#[error("AudioHardwareDestroyProcessTap failed, status: {0}")]
AudioHardwareDestroyProcessTapFailed(i32),
#[error("Get aggregate device property full sub device list failed, status: {0}")]
GetAggregateDevicePropertyFullSubDeviceListFailed(i32),
#[error("Add property listener block failed, status: {0}")]
AddPropertyListenerBlockFailed(i32),
#[error("AudioObjectGetPropertyData failed, status: {0}")]
AudioObjectGetPropertyDataFailed(i32),
#[error("AVAudioFile class not found")]
AVAudioFileClassNotFound,
#[error("Alloc AVAudioFile failed")]
AllocAVAudioFileFailed,
#[error("Init AVAudioFile failed")]
InitAVAudioFileFailed,
#[error("AVAudioPCMBuffer class not found")]
AVAudioPCMBufferClassNotFound,
#[error("Alloc AVAudioPCMBuffer failed")]
AllocAVAudioPCMBufferFailed,
#[error("Init AVAudioPCMBuffer failed")]
InitAVAudioPCMBufferFailed,
#[error("Write AVAudioFile failed")]
WriteAVAudioFileFailed,
}
impl From<CoreAudioError> for napi::Error {
fn from(value: CoreAudioError) -> Self {
napi::Error::new(napi::Status::GenericFailure, value.to_string())
}
}

11
packages/frontend/native/media_capture/src/macos/mod.rs Normal file
View File

@@ -0,0 +1,11 @@
pub mod audio_stream_basic_desc;
pub mod av_audio_file;
pub mod av_audio_format;
pub mod av_audio_pcm_buffer;
pub mod ca_tap_description;
pub mod device;
pub(crate) mod error;
pub mod pid;
pub mod queue;
pub mod screen_capture_kit;
pub mod tap_audio;

98
packages/frontend/native/media_capture/src/macos/pid.rs Normal file
View File

@@ -0,0 +1,98 @@
use std::{mem::MaybeUninit, ptr};
use coreaudio::sys::{
kAudioHardwareNoError, kAudioHardwarePropertyProcessObjectList, kAudioObjectPropertyElementMain,
kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, AudioObjectGetPropertyData,
AudioObjectGetPropertyDataSize, AudioObjectID, AudioObjectPropertyAddress,
AudioObjectPropertySelector,
};
use crate::error::CoreAudioError;
pub fn audio_process_list() -> Result<Vec<AudioObjectID>, CoreAudioError> {
let address = AudioObjectPropertyAddress {
mSelector: kAudioHardwarePropertyProcessObjectList,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain,
};
let mut data_size = 0u32;
let status = unsafe {
AudioObjectGetPropertyDataSize(
kAudioObjectSystemObject,
&address,
0,
ptr::null_mut(),
&mut data_size,
)
};
if status != kAudioHardwareNoError as i32 {
return Err(CoreAudioError::GetProcessObjectListSizeFailed(status));
}
let mut process_list: Vec<AudioObjectID> = vec![0; data_size as usize];
let status = unsafe {
AudioObjectGetPropertyData(
kAudioObjectSystemObject,
&address,
0,
ptr::null_mut(),
(&mut data_size as *mut u32).cast(),
process_list.as_mut_ptr().cast(),
)
};
if status != kAudioHardwareNoError as i32 {
return Err(CoreAudioError::GetProcessObjectListFailed(status));
}
Ok(process_list)
}
pub fn get_process_property<T: Sized>(
object: &AudioObjectID,
selector: AudioObjectPropertySelector,
) -> Result<T, CoreAudioError> {
let object_id = *object;
let address = AudioObjectPropertyAddress {
mSelector: selector,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain,
};
let mut data_size = 0u32;
let status = unsafe {
AudioObjectGetPropertyDataSize(object_id, &address, 0, ptr::null_mut(), &mut data_size)
};
if status != kAudioHardwareNoError as i32 {
return Err(CoreAudioError::AudioObjectGetPropertyDataSizeFailed(status));
}
get_property_data(object_id, &address, &mut data_size)
}
pub fn get_property_data<T: Sized>(
object_id: AudioObjectID,
address: &AudioObjectPropertyAddress,
data_size: &mut u32,
) -> Result<T, CoreAudioError> {
let mut property = MaybeUninit::<T>::uninit();
let status = unsafe {
AudioObjectGetPropertyData(
object_id,
address,
0,
ptr::null_mut(),
(data_size as *mut u32).cast(),
property.as_mut_ptr().cast(),
)
};
if status != kAudioHardwareNoError as i32 {
return Err(CoreAudioError::AudioObjectGetPropertyDataFailed(status));
}
Ok(unsafe { property.assume_init() })
}

12
packages/frontend/native/media_capture/src/macos/queue.rs Normal file
View File

@@ -0,0 +1,12 @@
pub(crate) fn create_audio_tap_queue() -> *mut dispatch2::ffi::dispatch_queue_s {
let queue_attr = unsafe {
dispatch2::ffi::dispatch_queue_attr_make_with_qos_class(
dispatch2::ffi::DISPATCH_QUEUE_SERIAL,
dispatch2::ffi::dispatch_qos_class_t::QOS_CLASS_USER_INITIATED,
0,
)
};
unsafe {
dispatch2::ffi::dispatch_queue_create(c"ProcessTapRecorder".as_ptr().cast(), queue_attr)
}
}

623
packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs Normal file
View File

@@ -0,0 +1,623 @@
use std::{
collections::HashMap,
ffi::c_void,
ptr,
sync::{
atomic::{AtomicPtr, Ordering},
Arc, LazyLock, RwLock,
},
};
use block2::{Block, RcBlock};
use core_foundation::{
base::TCFType,
string::{CFString, CFStringRef},
};
use coreaudio::sys::{
kAudioHardwarePropertyProcessObjectList, kAudioObjectPropertyElementMain,
kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, kAudioProcessPropertyBundleID,
kAudioProcessPropertyIsRunning, kAudioProcessPropertyIsRunningInput, kAudioProcessPropertyPID,
AudioObjectAddPropertyListenerBlock, AudioObjectID, AudioObjectPropertyAddress,
AudioObjectRemovePropertyListenerBlock,
};
use napi::{
bindgen_prelude::{Buffer, Error, Float32Array, Result, Status},
threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
};
use napi_derive::napi;
use objc2::{
msg_send,
rc::Retained,
runtime::{AnyClass, AnyObject},
Encode, Encoding,
};
use objc2_foundation::NSString;
use screencapturekit::shareable_content::SCShareableContent;
use uuid::Uuid;
use crate::{
error::CoreAudioError,
pid::{audio_process_list, get_process_property},
tap_audio::{AggregateDevice, AudioTapStream},
};
#[repr(C)]
#[derive(Debug, Copy, Clone)]
struct NSSize {
width: f64,
height: f64,
}
#[repr(C)]
#[derive(Debug, Copy, Clone)]
struct NSPoint {
x: f64,
y: f64,
}
#[repr(C)]
#[derive(Debug, Copy, Clone)]
struct NSRect {
origin: NSPoint,
size: NSSize,
}
unsafe impl Encode for NSSize {
const ENCODING: Encoding = Encoding::Struct("NSSize", &[f64::ENCODING, f64::ENCODING]);
}
unsafe impl Encode for NSPoint {
const ENCODING: Encoding = Encoding::Struct("NSPoint", &[f64::ENCODING, f64::ENCODING]);
}
unsafe impl Encode for NSRect {
const ENCODING: Encoding = Encoding::Struct("NSRect", &[<NSPoint>::ENCODING, <NSSize>::ENCODING]);
}
static RUNNING_APPLICATIONS: LazyLock<RwLock<Vec<AudioObjectID>>> =
LazyLock::new(|| RwLock::new(audio_process_list().expect("Failed to get running applications")));
static APPLICATION_STATE_CHANGED_SUBSCRIBERS: LazyLock<
RwLock<HashMap<AudioObjectID, HashMap<Uuid, Arc<ThreadsafeFunction<(), ()>>>>>,
> = LazyLock::new(|| RwLock::new(HashMap::new()));
static APPLICATION_STATE_CHANGED_LISTENER_BLOCKS: LazyLock<
RwLock<HashMap<AudioObjectID, AtomicPtr<c_void>>>,
> = LazyLock::new(|| RwLock::new(HashMap::new()));
static NSRUNNING_APPLICATION_CLASS: LazyLock<Option<&'static AnyClass>> =
LazyLock::new(|| AnyClass::get(c"NSRunningApplication"));
static AVCAPTUREDEVICE_CLASS: LazyLock<Option<&'static AnyClass>> =
LazyLock::new(|| AnyClass::get(c"AVCaptureDevice"));
static SCSTREAM_CLASS: LazyLock<Option<&'static AnyClass>> =
LazyLock::new(|| AnyClass::get(c"SCStream"));
struct TappableApplication {
object_id: AudioObjectID,
}
impl TappableApplication {
fn new(object_id: AudioObjectID) -> Self {
Self { object_id }
}
fn process_id(&self) -> std::result::Result<i32, CoreAudioError> {
get_process_property(&self.object_id, kAudioProcessPropertyPID)
}
fn bundle_identifier(&self) -> Result<String> {
let bundle_id: CFStringRef =
get_process_property(&self.object_id, kAudioProcessPropertyBundleID)?;
Ok(unsafe { CFString::wrap_under_get_rule(bundle_id) }.to_string())
}
fn name(&self) -> Result<String> {
let pid = self.process_id()?;
// Get NSRunningApplication class
let running_app_class = NSRUNNING_APPLICATION_CLASS.as_ref().ok_or_else(|| {
Error::new(
Status::GenericFailure,
"NSRunningApplication class not found",
)
})?;
// Get running application with PID
let running_app: *mut AnyObject =
unsafe { msg_send![*running_app_class, runningApplicationWithProcessIdentifier: pid] };
if running_app.is_null() {
return Ok(String::new());
}
// Get localized name
let name: *mut NSString = unsafe { msg_send![running_app, localizedName] };
if name.is_null() {
return Ok(String::new());
}
// Create a safe wrapper and convert to string
let name = unsafe {
Retained::from_raw(name).ok_or_else(|| {
Error::new(
Status::GenericFailure,
"Failed to create safe wrapper for localizedName",
)
})?
};
Ok(name.to_string())
}
fn icon(&self) -> Result<Vec<u8>> {
let pid = self.process_id()?;
// Get NSRunningApplication class
let running_app_class = NSRUNNING_APPLICATION_CLASS.as_ref().ok_or_else(|| {
Error::new(
Status::GenericFailure,
"NSRunningApplication class not found",
)
})?;
// Get running application with PID
let running_app: *mut AnyObject =
unsafe { msg_send![*running_app_class, runningApplicationWithProcessIdentifier: pid] };
if running_app.is_null() {
return Ok(Vec::new());
}
unsafe {
// Get original icon
let icon: *mut AnyObject = msg_send![running_app, icon];
if icon.is_null() {
return Ok(Vec::new());
}
// Create a new NSImage with 64x64 size
let nsimage_class = AnyClass::get(c"NSImage")
.ok_or_else(|| Error::new(Status::GenericFailure, "NSImage class not found"))?;
let resized_image: *mut AnyObject = msg_send![nsimage_class, alloc];
let resized_image: *mut AnyObject =
msg_send![resized_image, initWithSize: NSSize { width: 64.0, height: 64.0 }];
let _: () = msg_send![resized_image, lockFocus];
// Define drawing rectangle for 64x64 image
let draw_rect = NSRect {
origin: NSPoint { x: 0.0, y: 0.0 },
size: NSSize {
width: 64.0,
height: 64.0,
},
};
// Draw the original icon into draw_rect (using NSCompositingOperationCopy = 2)
let _: () = msg_send![icon, drawInRect: draw_rect, fromRect: NSRect { origin: NSPoint { x: 0.0, y: 0.0 }, size: NSSize { width: 0.0, height: 0.0 } }, operation: 2, fraction: 1.0];
let _: () = msg_send![resized_image, unlockFocus];
// Get TIFF representation from the downsized image
let tiff_data: *mut AnyObject = msg_send![resized_image, TIFFRepresentation];
if tiff_data.is_null() {
return Ok(Vec::new());
}
// Create bitmap image rep from TIFF
let bitmap_class = AnyClass::get(c"NSBitmapImageRep")
.ok_or_else(|| Error::new(Status::GenericFailure, "NSBitmapImageRep class not found"))?;
let bitmap: *mut AnyObject = msg_send![bitmap_class, imageRepWithData: tiff_data];
if bitmap.is_null() {
return Ok(Vec::new());
}
// Create properties dictionary with compression factor
let dict_class = AnyClass::get(c"NSMutableDictionary").ok_or_else(|| {
Error::new(
Status::GenericFailure,
"NSMutableDictionary class not found",
)
})?;
let properties: *mut AnyObject = msg_send![dict_class, dictionary];
// Add compression properties
let compression_key = NSString::from_str("NSImageCompressionFactor");
let number_class = AnyClass::get(c"NSNumber")
.ok_or_else(|| Error::new(Status::GenericFailure, "NSNumber class not found"))?;
let compression_value: *mut AnyObject = msg_send![number_class, numberWithDouble: 0.8];
let _: () = msg_send![properties, setObject: compression_value, forKey: &*compression_key];
// Get PNG data with properties
let png_data: *mut AnyObject =
msg_send![bitmap, representationUsingType: 4, properties: properties]; // 4 = PNG
if png_data.is_null() {
return Ok(Vec::new());
}
// Get bytes from NSData
let bytes: *const u8 = msg_send![png_data, bytes];
let length: usize = msg_send![png_data, length];
if bytes.is_null() {
return Ok(Vec::new());
}
// Copy bytes into a Vec<u8>
let data = std::slice::from_raw_parts(bytes, length).to_vec();
Ok(data)
}
}
}
#[napi]
pub struct Application {
inner: TappableApplication,
pub(crate) object_id: AudioObjectID,
pub(crate) process_id: i32,
pub(crate) bundle_identifier: String,
pub(crate) name: String,
}
#[napi]
impl Application {
fn new(app: TappableApplication) -> Result<Self> {
let object_id = app.object_id;
let bundle_identifier = app.bundle_identifier()?;
let name = app.name()?;
let process_id = app.process_id()?;
Ok(Self {
inner: app,
object_id,
process_id,
bundle_identifier,
name,
})
}
#[napi]
pub fn tap_global_audio(
excluded_processes: Option<Vec<&Application>>,
audio_stream_callback: Arc<ThreadsafeFunction<Float32Array, (), Float32Array, true>>,
) -> Result<AudioTapStream> {
let mut device = AggregateDevice::create_global_tap_but_exclude_processes(
&excluded_processes
.unwrap_or_default()
.iter()
.map(|app| app.object_id)
.collect::<Vec<_>>(),
)?;
device.start(audio_stream_callback)
}
#[napi(getter)]
pub fn process_id(&self) -> i32 {
self.process_id
}
#[napi(getter)]
pub fn bundle_identifier(&self) -> String {
self.bundle_identifier.clone()
}
#[napi(getter)]
pub fn name(&self) -> String {
self.name.clone()
}
#[napi(getter)]
pub fn icon(&self) -> Result<Buffer> {
let icon = self.inner.icon()?;
Ok(Buffer::from(icon))
}
#[napi(getter)]
pub fn get_is_running(&self) -> Result<bool> {
Ok(get_process_property(
&self.object_id,
kAudioProcessPropertyIsRunningInput,
)?)
}
#[napi]
pub fn tap_audio(
&self,
audio_stream_callback: Arc<ThreadsafeFunction<Float32Array, (), Float32Array, true>>,
) -> Result<AudioTapStream> {
let mut device = AggregateDevice::new(self)?;
device.start(audio_stream_callback)
}
}
#[napi]
pub struct ApplicationListChangedSubscriber {
listener_block: *const Block<dyn Fn(u32, *mut c_void)>,
}
#[napi]
impl ApplicationListChangedSubscriber {
#[napi]
pub fn unsubscribe(&self) -> Result<()> {
let status = unsafe {
AudioObjectRemovePropertyListenerBlock(
kAudioObjectSystemObject,
&AudioObjectPropertyAddress {
mSelector: kAudioHardwarePropertyProcessObjectList,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain,
},
ptr::null_mut(),
self.listener_block.cast_mut().cast(),
)
};
if status != 0 {
return Err(Error::new(
Status::GenericFailure,
"Failed to remove property listener",
));
}
Ok(())
}
}
#[napi]
pub struct ApplicationStateChangedSubscriber {
id: Uuid,
object_id: AudioObjectID,
}
#[napi]
impl ApplicationStateChangedSubscriber {
#[napi]
pub fn unsubscribe(&self) {
if let Ok(mut lock) = APPLICATION_STATE_CHANGED_SUBSCRIBERS.write() {
if let Some(subscribers) = lock.get_mut(&self.object_id) {
subscribers.remove(&self.id);
if subscribers.is_empty() {
lock.remove(&self.object_id);
if let Some(listener_block) = APPLICATION_STATE_CHANGED_LISTENER_BLOCKS
.write()
.ok()
.as_mut()
.and_then(|map| map.remove(&self.object_id))
{
unsafe {
AudioObjectRemovePropertyListenerBlock(
self.object_id,
&AudioObjectPropertyAddress {
mSelector: kAudioProcessPropertyIsRunning,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain,
},
ptr::null_mut(),
listener_block.load(Ordering::Relaxed),
);
}
}
}
}
}
}
}
#[napi]
pub struct ShareableContent {
_inner: SCShareableContent,
}
#[napi]
#[derive(Default)]
pub struct RecordingPermissions {
pub audio: bool,
pub screen: bool,
}
#[napi]
impl ShareableContent {
#[napi]
pub fn on_application_list_changed(
callback: Arc<ThreadsafeFunction<(), ()>>,
) -> Result<ApplicationListChangedSubscriber> {
let callback_block: RcBlock<dyn Fn(u32, *mut c_void)> =
RcBlock::new(move |_in_number_addresses, _in_addresses: *mut c_void| {
if let Err(err) = RUNNING_APPLICATIONS
.write()
.map_err(|_| {
Error::new(
Status::GenericFailure,
"Poisoned RwLock while writing RunningApplications",
)
})
.and_then(|mut running_applications| {
audio_process_list().map_err(From::from).map(|apps| {
*running_applications = apps;
})
})
{
callback.call(Err(err), ThreadsafeFunctionCallMode::NonBlocking);
} else {
callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking);
}
});
let listener_block = &*callback_block as *const Block<dyn Fn(u32, *mut c_void)>;
let status = unsafe {
AudioObjectAddPropertyListenerBlock(
kAudioObjectSystemObject,
&AudioObjectPropertyAddress {
mSelector: kAudioHardwarePropertyProcessObjectList,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain,
},
ptr::null_mut(),
listener_block.cast_mut().cast(),
)
};
if status != 0 {
return Err(Error::new(
Status::GenericFailure,
"Failed to add property listener",
));
}
Ok(ApplicationListChangedSubscriber { listener_block })
}
#[napi]
pub fn on_app_state_changed(
app: &Application,
callback: Arc<ThreadsafeFunction<(), ()>>,
) -> Result<ApplicationStateChangedSubscriber> {
let id = Uuid::new_v4();
let mut lock = APPLICATION_STATE_CHANGED_SUBSCRIBERS.write().map_err(|_| {
Error::new(
Status::GenericFailure,
"Poisoned RwLock while writing ApplicationStateChangedSubscribers",
)
})?;
if let Some(subscribers) = lock.get_mut(&app.object_id) {
subscribers.insert(id, callback);
} else {
let object_id = app.object_id;
let list_change: RcBlock<dyn Fn(u32, *mut c_void)> =
RcBlock::new(move |in_number_addresses, in_addresses: *mut c_void| {
let addresses = unsafe {
std::slice::from_raw_parts(
in_addresses as *mut AudioObjectPropertyAddress,
in_number_addresses as usize,
)
};
for address in addresses {
if address.mSelector == kAudioProcessPropertyIsRunning {
if let Some(subscribers) = APPLICATION_STATE_CHANGED_SUBSCRIBERS
.read()
.ok()
.as_ref()
.and_then(|map| map.get(&object_id))
{
for callback in subscribers.values() {
callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking);
}
}
}
}
});
let address = AudioObjectPropertyAddress {
mSelector: kAudioProcessPropertyIsRunning,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain,
};
let listener_block = &*list_change as *const Block<dyn Fn(u32, *mut c_void)>;
let status = unsafe {
AudioObjectAddPropertyListenerBlock(
app.object_id,
&address,
ptr::null_mut(),
listener_block.cast_mut().cast(),
)
};
if status != 0 {
return Err(Error::new(
Status::GenericFailure,
"Failed to add property listener",
));
}
let subscribers = {
let mut map = HashMap::new();
map.insert(id, callback);
map
};
lock.insert(app.object_id, subscribers);
}
Ok(ApplicationStateChangedSubscriber {
id,
object_id: app.object_id,
})
}
#[napi(constructor)]
pub fn new() -> Result<Self> {
Ok(Self {
_inner: SCShareableContent::get().map_err(|err| Error::new(Status::GenericFailure, err))?,
})
}
#[napi]
pub fn applications(&self) -> Result<Vec<Application>> {
RUNNING_APPLICATIONS
.read()
.map_err(|_| {
Error::new(
Status::GenericFailure,
"Poisoned RwLock while reading RunningApplications",
)
})?
.iter()
.filter_map(|id| {
let app = TappableApplication::new(*id);
if !app.bundle_identifier().ok()?.is_empty() {
Some(Application::new(app))
} else {
None
}
})
.collect()
}
#[napi]
pub fn application_with_process_id(&self, process_id: u32) -> Result<Application> {
// Find the AudioObjectID for the given process ID
let audio_object_id = {
let running_apps = RUNNING_APPLICATIONS.read().map_err(|_| {
Error::new(
Status::GenericFailure,
"Poisoned RwLock while reading RunningApplications",
)
})?;
*running_apps
.iter()
.find(|&&id| {
let app = TappableApplication::new(id);
app
.process_id()
.map(|pid| pid as u32 == process_id)
.unwrap_or(false)
})
.ok_or_else(|| {
Error::new(
Status::GenericFailure,
format!("No application found with process ID {}", process_id),
)
})?
};
let app = TappableApplication::new(audio_object_id);
Application::new(app)
}
#[napi]
pub fn check_recording_permissions(&self) -> Result<RecordingPermissions> {
let av_capture_class = AVCAPTUREDEVICE_CLASS
.as_ref()
.ok_or_else(|| Error::new(Status::GenericFailure, "AVCaptureDevice class not found"))?;
let sc_stream_class = SCSTREAM_CLASS
.as_ref()
.ok_or_else(|| Error::new(Status::GenericFailure, "SCStream class not found"))?;
let media_type = NSString::from_str("com.apple.avfoundation.avcapturedevice.built-in_audio");
let audio_status: i32 = unsafe {
msg_send![
*av_capture_class,
authorizationStatusForMediaType: &*media_type
]
};
let screen_status: bool = unsafe { msg_send![*sc_stream_class, isScreenCaptureAuthorized] };
Ok(RecordingPermissions {
// AVAuthorizationStatusAuthorized = 3
audio: audio_status == 3,
screen: screen_status,
})
}
}

360
packages/frontend/native/media_capture/src/macos/tap_audio.rs Normal file
View File

@@ -0,0 +1,360 @@
use std::{ffi::c_void, sync::Arc};
use block2::{Block, RcBlock};
use core_foundation::{
array::CFArray,
base::{CFType, ItemRef, TCFType},
boolean::CFBoolean,
dictionary::CFDictionary,
string::CFString,
uuid::CFUUID,
};
use coreaudio::sys::{
kAudioAggregateDeviceIsPrivateKey, kAudioAggregateDeviceIsStackedKey,
kAudioAggregateDeviceMainSubDeviceKey, kAudioAggregateDeviceNameKey,
kAudioAggregateDeviceSubDeviceListKey, kAudioAggregateDeviceTapAutoStartKey,
kAudioAggregateDeviceTapListKey, kAudioAggregateDeviceUIDKey, kAudioHardwareNoError,
kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultSystemOutputDevice,
kAudioSubDeviceUIDKey, kAudioSubTapDriftCompensationKey, kAudioSubTapUIDKey,
AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID, AudioDeviceIOProcID,
AudioDeviceStart, AudioDeviceStop, AudioHardwareCreateAggregateDevice,
AudioHardwareDestroyAggregateDevice, AudioObjectID, AudioTimeStamp, OSStatus,
};
use napi::{
bindgen_prelude::Float32Array,
threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
Result,
};
use napi_derive::napi;
use objc2::{runtime::AnyObject, Encode, Encoding, RefEncode};
use crate::{
ca_tap_description::CATapDescription, device::get_device_uid, error::CoreAudioError,
queue::create_audio_tap_queue, screen_capture_kit::Application,
};
extern "C" {
fn AudioHardwareCreateProcessTap(
inDescription: *mut AnyObject,
outTapID: *mut AudioObjectID,
) -> OSStatus;
fn AudioHardwareDestroyProcessTap(tapID: AudioObjectID) -> OSStatus;
}
/// [Apple's documentation](https://developer.apple.com/documentation/coreaudiotypes/audiobuffer?language=objc)
#[repr(C)]
#[derive(Clone, Copy, Debug, PartialEq)]
#[allow(non_snake_case)]
pub struct AudioBuffer {
pub mNumberChannels: u32,
pub mDataByteSize: u32,
pub mData: *mut c_void,
}
unsafe impl Encode for AudioBuffer {
const ENCODING: Encoding = Encoding::Struct(
"AudioBuffer",
&[<u32>::ENCODING, <u32>::ENCODING, <*mut c_void>::ENCODING],
);
}
unsafe impl RefEncode for AudioBuffer {
const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);
}
#[repr(C)]
#[derive(Clone, Copy, Debug, PartialEq)]
#[allow(non_snake_case)]
pub struct AudioBufferList {
pub mNumberBuffers: u32,
pub mBuffers: [AudioBuffer; 1],
}
unsafe impl Encode for AudioBufferList {
const ENCODING: Encoding = Encoding::Struct(
"AudioBufferList",
&[<u32>::ENCODING, <[AudioBuffer; 1]>::ENCODING],
);
}
unsafe impl RefEncode for AudioBufferList {
const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);
}
pub struct AggregateDevice {
pub tap_id: AudioObjectID,
pub id: AudioObjectID,
}
impl AggregateDevice {
pub fn new(app: &Application) -> Result<Self> {
let mut tap_id: AudioObjectID = 0;
let tap_description = CATapDescription::init_stereo_mixdown_of_processes(app.object_id)?;
let status = unsafe { AudioHardwareCreateProcessTap(tap_description.inner, &mut tap_id) };
if status != 0 {
return Err(CoreAudioError::CreateProcessTapFailed(status).into());
}
let description_dict = Self::create_aggregate_description(tap_id, tap_description.get_uuid()?)?;
let mut aggregate_device_id: AudioObjectID = 0;
let status = unsafe {
AudioHardwareCreateAggregateDevice(
description_dict.as_concrete_TypeRef().cast(),
&mut aggregate_device_id,
)
};
// Check the status and return the appropriate result
if status != 0 {
return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into());
}
Ok(Self {
tap_id,
id: aggregate_device_id,
})
}
pub fn create_global_tap_but_exclude_processes(processes: &[AudioObjectID]) -> Result<Self> {
let mut tap_id: AudioObjectID = 0;
let tap_description =
CATapDescription::init_stereo_global_tap_but_exclude_processes(processes)?;
let status = unsafe { AudioHardwareCreateProcessTap(tap_description.inner, &mut tap_id) };
if status != 0 {
return Err(CoreAudioError::CreateProcessTapFailed(status).into());
}
let description_dict = Self::create_aggregate_description(tap_id, tap_description.get_uuid()?)?;
let mut aggregate_device_id: AudioObjectID = 0;
let status = unsafe {
AudioHardwareCreateAggregateDevice(
description_dict.as_concrete_TypeRef().cast(),
&mut aggregate_device_id,
)
};
// Check the status and return the appropriate result
if status != 0 {
return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into());
}
Ok(Self {
tap_id,
id: aggregate_device_id,
})
}
pub fn start(
&mut self,
audio_stream_callback: Arc<ThreadsafeFunction<Float32Array, (), Float32Array, true>>,
) -> Result<AudioTapStream> {
let queue = create_audio_tap_queue();
let mut in_proc_id: AudioDeviceIOProcID = None;
let in_io_block: RcBlock<
dyn Fn(*mut c_void, *mut c_void, *mut c_void, *mut c_void, *mut c_void) -> i32,
> = RcBlock::new(
move |_in_now: *mut c_void,
in_input_data: *mut c_void,
in_input_time: *mut c_void,
_out_output_data: *mut c_void,
_in_output_time: *mut c_void| {
let AudioTimeStamp { mSampleTime, .. } = unsafe { &*in_input_time.cast() };
// ignore pre-roll
if *mSampleTime < 0.0 {
return kAudioHardwareNoError as i32;
}
let AudioBufferList { mBuffers, .. } =
unsafe { &mut *in_input_data.cast::<AudioBufferList>() };
let [AudioBuffer {
mData,
mNumberChannels,
mDataByteSize,
}] = mBuffers;
// Only create slice if we have valid data
if !mData.is_null() && *mDataByteSize > 0 {
// Calculate total number of samples (accounting for interleaved stereo)
let total_samples = *mDataByteSize as usize / 4; // 4 bytes per f32
// Create a slice of all samples
let samples: &[f32] =
unsafe { std::slice::from_raw_parts(mData.cast::<f32>(), total_samples) };
// Convert to mono if needed
let mono_samples: Vec<f32> = if *mNumberChannels > 1 {
samples
.chunks(*mNumberChannels as usize)
.map(|chunk| chunk.iter().sum::<f32>() / *mNumberChannels as f32)
.collect()
} else {
samples.to_vec()
};
audio_stream_callback.call(
Ok(mono_samples.into()),
ThreadsafeFunctionCallMode::NonBlocking,
);
}
kAudioHardwareNoError as i32
},
);
let status = unsafe {
AudioDeviceCreateIOProcIDWithBlock(
&mut in_proc_id,
self.id,
queue.cast(),
(&*in_io_block
as *const Block<
dyn Fn(*mut c_void, *mut c_void, *mut c_void, *mut c_void, *mut c_void) -> i32,
>)
.cast_mut()
.cast(),
)
};
if status != 0 {
return Err(CoreAudioError::CreateIOProcIDWithBlockFailed(status).into());
}
let status = unsafe { AudioDeviceStart(self.id, in_proc_id) };
if status != 0 {
return Err(CoreAudioError::AudioDeviceStartFailed(status).into());
}
Ok(AudioTapStream {
device_id: self.id,
in_proc_id,
stop_called: false,
})
}
fn create_aggregate_description(
tap_id: AudioObjectID,
tap_uuid_string: ItemRef<CFString>,
) -> Result<CFDictionary<CFType, CFType>> {
let system_output_uid = get_device_uid(kAudioHardwarePropertyDefaultSystemOutputDevice)?;
let default_input_uid = get_device_uid(kAudioHardwarePropertyDefaultInputDevice)?;
let aggregate_device_name = CFString::new(&format!("Tap-{}", tap_id));
let aggregate_device_uid: uuid::Uuid = CFUUID::new().into();
let aggregate_device_uid_string = aggregate_device_uid.to_string();
// Sub-device UID key and dictionary
let sub_device_output_dict = CFDictionary::from_CFType_pairs(&[(
cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),
system_output_uid.as_CFType(),
)]);
let sub_device_input_dict = CFDictionary::from_CFType_pairs(&[(
cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),
default_input_uid.as_CFType(),
)]);
let tap_device_dict = CFDictionary::from_CFType_pairs(&[
(
cfstring_from_bytes_with_nul(kAudioSubTapDriftCompensationKey).as_CFType(),
CFBoolean::false_value().as_CFType(),
),
(
cfstring_from_bytes_with_nul(kAudioSubTapUIDKey).as_CFType(),
tap_uuid_string.as_CFType(),
),
]);
let capture_device_list = vec![sub_device_input_dict, sub_device_output_dict];
// Sub-device list
let sub_device_list = CFArray::from_CFTypes(&capture_device_list);
let tap_list = CFArray::from_CFTypes(&[tap_device_dict]);
// Create the aggregate device description dictionary
let description_dict = CFDictionary::from_CFType_pairs(&[
(
cfstring_from_bytes_with_nul(kAudioAggregateDeviceNameKey).as_CFType(),
aggregate_device_name.as_CFType(),
),
(
cfstring_from_bytes_with_nul(kAudioAggregateDeviceUIDKey).as_CFType(),
CFString::new(aggregate_device_uid_string.as_str()).as_CFType(),
),
(
cfstring_from_bytes_with_nul(kAudioAggregateDeviceMainSubDeviceKey).as_CFType(),
system_output_uid.as_CFType(),
),
(
cfstring_from_bytes_with_nul(kAudioAggregateDeviceIsPrivateKey).as_CFType(),
CFBoolean::true_value().as_CFType(),
),
(
cfstring_from_bytes_with_nul(kAudioAggregateDeviceIsStackedKey).as_CFType(),
CFBoolean::false_value().as_CFType(),
),
(
cfstring_from_bytes_with_nul(kAudioAggregateDeviceTapAutoStartKey).as_CFType(),
CFBoolean::true_value().as_CFType(),
),
(
cfstring_from_bytes_with_nul(kAudioAggregateDeviceSubDeviceListKey).as_CFType(),
sub_device_list.as_CFType(),
),
(
cfstring_from_bytes_with_nul(kAudioAggregateDeviceTapListKey).as_CFType(),
tap_list.as_CFType(),
),
]);
Ok(description_dict)
}
}
#[napi]
pub struct AudioTapStream {
device_id: AudioObjectID,
in_proc_id: AudioDeviceIOProcID,
stop_called: bool,
}
#[napi]
impl AudioTapStream {
#[napi]
pub fn stop(&mut self) -> Result<()> {
if self.stop_called {
return Ok(());
}
self.stop_called = true;
let status = unsafe { AudioDeviceStop(self.device_id, self.in_proc_id) };
if status != 0 {
return Err(CoreAudioError::AudioDeviceStopFailed(status).into());
}
let status = unsafe { AudioDeviceDestroyIOProcID(self.device_id, self.in_proc_id) };
if status != 0 {
return Err(CoreAudioError::AudioDeviceDestroyIOProcIDFailed(status).into());
}
let status = unsafe { AudioHardwareDestroyAggregateDevice(self.device_id) };
if status != 0 {
return Err(CoreAudioError::AudioHardwareDestroyAggregateDeviceFailed(status).into());
}
let status = unsafe { AudioHardwareDestroyProcessTap(self.device_id) };
if status != 0 {
return Err(CoreAudioError::AudioHardwareDestroyProcessTapFailed(status).into());
}
Ok(())
}
}
fn cfstring_from_bytes_with_nul(bytes: &'static [u8]) -> CFString {
CFString::new(
unsafe { std::ffi::CStr::from_bytes_with_nul_unchecked(bytes) }
.to_string_lossy()
.as_ref(),
)
}