From 2fb55a8b62d857062fa5ccf3d7fd13a64c7f782e Mon Sep 17 00:00:00 2001 From: galister <22305755+galister@users.noreply.github.com> Date: Mon, 5 Feb 2024 23:15:32 +0100 Subject: [PATCH] feature: osc sender --- Cargo.lock | 11 +++ Cargo.toml | 4 +- src/backend/mod.rs | 2 + src/backend/openxr/helpers.rs | 164 +++++++++++++++++++++++++++++++ src/backend/openxr/lines.rs | 9 +- src/backend/openxr/mod.rs | 179 +++------------------------------- src/backend/openxr/overlay.rs | 4 +- src/backend/osc.rs | 91 +++++++++++++++++ 8 files changed, 292 insertions(+), 172 deletions(-) create mode 100644 src/backend/openxr/helpers.rs create mode 100644 src/backend/osc.rs diff --git a/Cargo.lock b/Cargo.lock index a078769..990c3ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2885,6 +2885,16 @@ dependencies = [ "hound", ] +[[package]] +name = "rosc" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2e63d9e6b0d090be1485cf159b1e04c3973d2d3e1614963544ea2ff47a4a981" +dependencies = [ + "byteorder", + "nom", +] + [[package]] name = "roxmltree" version = "0.14.1" @@ -4295,6 +4305,7 @@ dependencies = [ "raw-window-handle 0.5.2", "regex", "rodio", + "rosc", "serde", "serde_json", "serde_yaml", diff --git a/Cargo.toml b/Cargo.toml index c69486c..d995815 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ png = "0.17.10" raw-window-handle = "0.5.2" regex = "1.9.5" rodio = { version = "0.17.1", default-features = false, features = ["wav", "hound"] } +rosc = { version = "0.10.1", optional = true } serde = { version = "1.0.188", features = ["derive", "rc"] } serde_json = "1.0.113" serde_yaml = "0.9.25" @@ -50,5 +51,6 @@ xdg = "2.5.2" [features] openvr = ["dep:ovr_overlay", "dep:json"] openxr = ["dep:openxr"] -default = ["openvr", "openxr"] +osc = ["dep:rosc"] +default = ["openvr", "openxr", "osc"] diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 0101aea..90531f4 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -4,4 +4,6 @@ pub mod input; pub mod openvr; #[cfg(feature = "openxr")] pub mod openxr; +#[cfg(feature = "osc")] +pub mod osc; pub mod overlay; diff --git a/src/backend/openxr/helpers.rs b/src/backend/openxr/helpers.rs new file mode 100644 index 0000000..376c765 --- /dev/null +++ b/src/backend/openxr/helpers.rs @@ -0,0 +1,164 @@ +use anyhow::{bail, ensure}; +use glam::{Affine3A, Quat, Vec3}; +use openxr as xr; +use xr::OverlaySessionCreateFlagsEXTX; + +pub(super) fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> { + let Ok(entry) = (unsafe { xr::Entry::load() }) else { + bail!("OpenXR Loader not found."); + }; + + let Ok(available_extensions) = entry.enumerate_extensions() else { + bail!("Failed to enumerate OpenXR extensions."); + }; + ensure!( + available_extensions.khr_vulkan_enable2, + "Missing KHR_vulkan_enable2 extension." + ); + ensure!( + available_extensions.extx_overlay, + "Missing EXTX_overlay extension." + ); + + let mut enabled_extensions = xr::ExtensionSet::default(); + enabled_extensions.khr_vulkan_enable2 = true; + enabled_extensions.extx_overlay = true; + + //#[cfg(not(debug_assertions))] + let layers = []; + //#[cfg(debug_assertions)] + //let layers = [ + // "XR_APILAYER_LUNARG_api_dump", + // "XR_APILAYER_LUNARG_standard_validation", + //]; + + let Ok(xr_instance) = entry.create_instance( + &xr::ApplicationInfo { + application_name: "wlx-overlay-s", + application_version: 0, + engine_name: "wlx-overlay-s", + engine_version: 0, + }, + &enabled_extensions, + &layers, + ) else { + bail!("Failed to create OpenXR instance."); + }; + + let Ok(instance_props) = xr_instance.properties() else { + bail!("Failed to query OpenXR instance properties."); + }; + log::info!( + "Using OpenXR runtime: {} {}", + instance_props.runtime_name, + instance_props.runtime_version + ); + + let Ok(system) = xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY) else { + bail!("Failed to access OpenXR HMD system."); + }; + + let vk_target_version_xr = xr::Version::new(1, 1, 0); + + let Ok(reqs) = xr_instance.graphics_requirements::(system) else { + bail!("Failed to query OpenXR Vulkan requirements."); + }; + + if vk_target_version_xr < reqs.min_api_version_supported + || vk_target_version_xr.major() > reqs.max_api_version_supported.major() + { + bail!( + "OpenXR runtime requires Vulkan version > {}, < {}.0.0", + reqs.min_api_version_supported, + reqs.max_api_version_supported.major() + 1 + ); + } + + Ok((xr_instance, system)) +} +pub(super) unsafe fn create_overlay_session( + instance: &xr::Instance, + system: xr::SystemId, + info: &xr::vulkan::SessionCreateInfo, +) -> Result { + let overlay = xr::sys::SessionCreateInfoOverlayEXTX { + ty: xr::sys::SessionCreateInfoOverlayEXTX::TYPE, + next: std::ptr::null(), + create_flags: OverlaySessionCreateFlagsEXTX::EMPTY, + session_layers_placement: 5, + }; + let binding = xr::sys::GraphicsBindingVulkanKHR { + ty: xr::sys::GraphicsBindingVulkanKHR::TYPE, + next: &overlay as *const _ as *const _, + instance: info.instance, + physical_device: info.physical_device, + device: info.device, + queue_family_index: info.queue_family_index, + queue_index: info.queue_index, + }; + let info = xr::sys::SessionCreateInfo { + ty: xr::sys::SessionCreateInfo::TYPE, + next: &binding as *const _ as *const _, + create_flags: Default::default(), + system_id: system, + }; + let mut out = xr::sys::Session::NULL; + let x = (instance.fp().create_session)(instance.as_raw(), &info, &mut out); + if x.into_raw() >= 0 { + Ok(out) + } else { + Err(x) + } +} + +pub(super) fn hmd_pose_from_views(views: &Vec) -> Affine3A { + let pos = { + let pos0: Vec3 = unsafe { std::mem::transmute(views[0].pose.position) }; + let pos1: Vec3 = unsafe { std::mem::transmute(views[1].pose.position) }; + (pos0 + pos1) * 0.5 + }; + let rot = { + let rot0 = unsafe { std::mem::transmute(views[0].pose.orientation) }; + let rot1 = unsafe { std::mem::transmute(views[1].pose.orientation) }; + quat_lerp(rot0, rot1, 0.5) + }; + + Affine3A::from_rotation_translation(rot, pos) +} + +fn quat_lerp(a: Quat, mut b: Quat, t: f32) -> Quat { + let l2 = a.dot(b); + if l2 < 0.0 { + b = -b; + } + + Quat::from_xyzw( + a.x - t * (a.x - b.x), + a.y - t * (a.y - b.y), + a.z - t * (a.z - b.z), + a.w - t * (a.w - b.w), + ) + .normalize() +} + +pub(super) fn transform_to_posef(transform: &Affine3A) -> xr::Posef { + let translation = transform.translation; + let norm_mat3 = transform + .matrix3 + .mul_scalar(1.0 / transform.matrix3.x_axis.length()); + let rotation = Quat::from_mat3a(&norm_mat3).normalize(); + + xr::Posef { + orientation: xr::Quaternionf { + x: rotation.x, + y: rotation.y, + z: rotation.z, + w: rotation.w, + }, + position: xr::Vector3f { + x: translation.x, + y: translation.y, + z: translation.z, + }, + } +} diff --git a/src/backend/openxr/lines.rs b/src/backend/openxr/lines.rs index 53b7212..eac21e7 100644 --- a/src/backend/openxr/lines.rs +++ b/src/backend/openxr/lines.rs @@ -11,11 +11,14 @@ use std::{ use vulkano::{command_buffer::CommandBufferUsage, format::Format, image::view::ImageView}; -use crate::graphics::{WlxCommandBuffer, WlxGraphics}; +use crate::{ + backend::openxr::helpers, + graphics::{WlxCommandBuffer, WlxGraphics}, +}; use super::{ swapchain::{create_swapchain_render_data, SwapchainRenderData}, - transform_to_posef, XrState, + XrState, }; static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(1); @@ -111,7 +114,7 @@ impl LinePool { transform = transform * rotations[closest.0]; - let posef = transform_to_posef(&transform); + let posef = helpers::transform_to_posef(&transform); line.maybe_line = Some(Line { view: self.colors[color].clone(), diff --git a/src/backend/openxr/mod.rs b/src/backend/openxr/mod.rs index 8183b17..d4b739c 100644 --- a/src/backend/openxr/mod.rs +++ b/src/backend/openxr/mod.rs @@ -7,17 +7,15 @@ use std::{ time::Duration, }; -use anyhow::{bail, ensure}; -use glam::{Affine3A, Quat, Vec3}; use openxr as xr; use vulkano::{command_buffer::CommandBufferUsage, Handle, VulkanObject}; -use xr::OverlaySessionCreateFlagsEXTX; use crate::{ backend::{ common::{OverlayContainer, TaskType}, input::interact, openxr::{input::DoubleClickCounter, lines::LinePool, overlay::OpenXrOverlayData}, + osc::OscSender, }, graphics::WlxGraphics, state::AppState, @@ -25,6 +23,7 @@ use crate::{ use super::common::BackendError; +mod helpers; mod input; mod lines; mod overlay; @@ -42,7 +41,7 @@ struct XrState { } pub fn openxr_run(running: Arc) -> Result<(), BackendError> { - let (xr_instance, system) = match init_xr() { + let (xr_instance, system) = match helpers::init_xr() { Ok((xr_instance, system)) => (xr_instance, system), Err(e) => { log::warn!("Will not use OpenXR: {}", e); @@ -63,10 +62,13 @@ pub fn openxr_run(running: Arc) -> Result<(), BackendError> { let mut overlays = OverlayContainer::::new(&mut app_state); let mut lines = LinePool::new(app_state.graphics.clone()); + #[cfg(feature = "osc")] + let mut osc_sender = OscSender::new(9000).ok(); + app_state.hid_provider.set_desktop_extent(overlays.extent); let (session, mut frame_wait, mut frame_stream) = unsafe { - let raw_session = create_overlay_session( + let raw_session = helpers::create_overlay_session( &xr_instance, system, &xr::vulkan::SessionCreateInfo { @@ -212,6 +214,11 @@ pub fn openxr_run(running: Arc) -> Result<(), BackendError> { .iter_mut() .for_each(|o| o.after_input(&mut app_state)); + #[cfg(feature = "osc")] + if let Some(ref mut sender) = osc_sender { + let _ = sender.send_params(&overlays); + }; + let (_, views) = xr_state .session .locate_views( @@ -221,7 +228,7 @@ pub fn openxr_run(running: Arc) -> Result<(), BackendError> { ) .unwrap(); - app_state.input_state.hmd = hmd_pose_from_views(&views); + app_state.input_state.hmd = helpers::hmd_pose_from_views(&views); overlays .iter_mut() @@ -296,163 +303,3 @@ pub fn openxr_run(running: Arc) -> Result<(), BackendError> { Ok(()) } - -fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> { - let Ok(entry) = (unsafe { xr::Entry::load() }) else { - bail!("OpenXR Loader not found."); - }; - - let Ok(available_extensions) = entry.enumerate_extensions() else { - bail!("Failed to enumerate OpenXR extensions."); - }; - ensure!( - available_extensions.khr_vulkan_enable2, - "Missing KHR_vulkan_enable2 extension." - ); - ensure!( - available_extensions.extx_overlay, - "Missing EXTX_overlay extension." - ); - - let mut enabled_extensions = xr::ExtensionSet::default(); - enabled_extensions.khr_vulkan_enable2 = true; - enabled_extensions.extx_overlay = true; - - //#[cfg(not(debug_assertions))] - let layers = []; - //#[cfg(debug_assertions)] - //let layers = [ - // "XR_APILAYER_LUNARG_api_dump", - // "XR_APILAYER_LUNARG_standard_validation", - //]; - - let Ok(xr_instance) = entry.create_instance( - &xr::ApplicationInfo { - application_name: "wlx-overlay-s", - application_version: 0, - engine_name: "wlx-overlay-s", - engine_version: 0, - }, - &enabled_extensions, - &layers, - ) else { - bail!("Failed to create OpenXR instance."); - }; - - let Ok(instance_props) = xr_instance.properties() else { - bail!("Failed to query OpenXR instance properties."); - }; - log::info!( - "Using OpenXR runtime: {} {}", - instance_props.runtime_name, - instance_props.runtime_version - ); - - let Ok(system) = xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY) else { - bail!("Failed to access OpenXR HMD system."); - }; - - let vk_target_version_xr = xr::Version::new(1, 1, 0); - - let Ok(reqs) = xr_instance.graphics_requirements::(system) else { - bail!("Failed to query OpenXR Vulkan requirements."); - }; - - if vk_target_version_xr < reqs.min_api_version_supported - || vk_target_version_xr.major() > reqs.max_api_version_supported.major() - { - bail!( - "OpenXR runtime requires Vulkan version > {}, < {}.0.0", - reqs.min_api_version_supported, - reqs.max_api_version_supported.major() + 1 - ); - } - - Ok((xr_instance, system)) -} -unsafe fn create_overlay_session( - instance: &xr::Instance, - system: xr::SystemId, - info: &xr::vulkan::SessionCreateInfo, -) -> Result { - let overlay = xr::sys::SessionCreateInfoOverlayEXTX { - ty: xr::sys::SessionCreateInfoOverlayEXTX::TYPE, - next: std::ptr::null(), - create_flags: OverlaySessionCreateFlagsEXTX::EMPTY, - session_layers_placement: 5, - }; - let binding = xr::sys::GraphicsBindingVulkanKHR { - ty: xr::sys::GraphicsBindingVulkanKHR::TYPE, - next: &overlay as *const _ as *const _, - instance: info.instance, - physical_device: info.physical_device, - device: info.device, - queue_family_index: info.queue_family_index, - queue_index: info.queue_index, - }; - let info = xr::sys::SessionCreateInfo { - ty: xr::sys::SessionCreateInfo::TYPE, - next: &binding as *const _ as *const _, - create_flags: Default::default(), - system_id: system, - }; - let mut out = xr::sys::Session::NULL; - let x = (instance.fp().create_session)(instance.as_raw(), &info, &mut out); - if x.into_raw() >= 0 { - Ok(out) - } else { - Err(x) - } -} - -fn hmd_pose_from_views(views: &Vec) -> Affine3A { - let pos = { - let pos0: Vec3 = unsafe { std::mem::transmute(views[0].pose.position) }; - let pos1: Vec3 = unsafe { std::mem::transmute(views[1].pose.position) }; - (pos0 + pos1) * 0.5 - }; - let rot = { - let rot0 = unsafe { std::mem::transmute(views[0].pose.orientation) }; - let rot1 = unsafe { std::mem::transmute(views[1].pose.orientation) }; - quat_lerp(rot0, rot1, 0.5) - }; - - Affine3A::from_rotation_translation(rot, pos) -} - -fn quat_lerp(a: Quat, mut b: Quat, t: f32) -> Quat { - let l2 = a.dot(b); - if l2 < 0.0 { - b = -b; - } - - Quat::from_xyzw( - a.x - t * (a.x - b.x), - a.y - t * (a.y - b.y), - a.z - t * (a.z - b.z), - a.w - t * (a.w - b.w), - ) - .normalize() -} - -fn transform_to_posef(transform: &Affine3A) -> xr::Posef { - let translation = transform.translation; - let norm_mat3 = transform - .matrix3 - .mul_scalar(1.0 / transform.matrix3.x_axis.length()); - let rotation = Quat::from_mat3a(&norm_mat3).normalize(); - - xr::Posef { - orientation: xr::Quaternionf { - x: rotation.x, - y: rotation.y, - z: rotation.z, - w: rotation.w, - }, - position: xr::Vector3f { - x: translation.x, - y: translation.y, - z: translation.z, - }, - } -} diff --git a/src/backend/openxr/overlay.rs b/src/backend/openxr/overlay.rs index 126f1d6..0b85908 100644 --- a/src/backend/openxr/overlay.rs +++ b/src/backend/openxr/overlay.rs @@ -2,7 +2,7 @@ use openxr as xr; use std::sync::Arc; use xr::{CompositionLayerFlags, EyeVisibility}; -use super::{swapchain::SwapchainRenderData, transform_to_posef, XrState}; +use super::{helpers, swapchain::SwapchainRenderData, XrState}; use crate::{ backend::{openxr::swapchain::create_swapchain_render_data, overlay::OverlayData}, graphics::WlxCommandBuffer, @@ -51,7 +51,7 @@ impl OverlayData { }); let sub_image = data.acquire_present_release(command_buffer, my_view); - let posef = transform_to_posef(&self.state.transform); + let posef = helpers::transform_to_posef(&self.state.transform); let scale_x = self.state.transform.matrix3.col(0).length(); let aspect_ratio = self.backend.extent()[1] as f32 / self.backend.extent()[0] as f32; diff --git a/src/backend/osc.rs b/src/backend/osc.rs new file mode 100644 index 0000000..9e7f456 --- /dev/null +++ b/src/backend/osc.rs @@ -0,0 +1,91 @@ +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, + time::Instant, +}; + +use anyhow::bail; +use rosc::{OscMessage, OscPacket, OscType}; + +use crate::overlays::{keyboard::KEYBOARD_NAME, watch::WATCH_NAME}; + +use super::common::OverlayContainer; + +pub struct OscSender { + last_sent: Instant, + upstream: UdpSocket, +} + +impl OscSender { + pub fn new(send_port: u16) -> anyhow::Result { + let ip = IpAddr::V4(Ipv4Addr::LOCALHOST); + + let Ok(upstream) = UdpSocket::bind("0.0.0.0:0") else { + bail!("Failed to bind UDP socket - OSC will not function."); + }; + + let Ok(_) = upstream.connect(SocketAddr::new(ip, send_port)) else { + bail!("Failed to connect UDP socket - OSC will not function."); + }; + + Ok(Self { + upstream, + last_sent: Instant::now(), + }) + } + + pub fn send_message(&self, addr: String, args: Vec) -> anyhow::Result<()> { + let packet = OscPacket::Message(OscMessage { addr, args }); + let Ok(bytes) = rosc::encoder::encode(&packet) else { + bail!("Could not encode OSC packet."); + }; + + let Ok(_) = self.upstream.send(&bytes) else { + bail!("Could not send OSC packet."); + }; + + Ok(()) + } + + pub fn send_params(&mut self, overlays: &OverlayContainer) -> anyhow::Result<()> + where + D: Default, + { + if self.last_sent.elapsed().as_millis() < 100 { + return Ok(()); + } + + let mut num_overlays = 0; + let mut has_keyboard = false; + let mut has_wrist = false; + + for o in overlays.iter() { + if !o.state.want_visible { + continue; + } + match o.state.name.as_ref() { + WATCH_NAME => has_wrist = true, + KEYBOARD_NAME => has_keyboard = true, + _ => num_overlays += 1, + } + } + + self.send_message( + "/avatar/parameters/isOverlayOpen".into(), + vec![OscType::Bool(num_overlays > 0)], + )?; + self.send_message( + "/avatar/parameters/isKeyboardOpen".into(), + vec![OscType::Bool(has_keyboard)], + )?; + self.send_message( + "/avatar/parameters/isWristVisible".into(), + vec![OscType::Bool(has_wrist)], + )?; + self.send_message( + "/avatar/parameters/openOverlayCount".into(), + vec![OscType::Int(num_overlays)], + )?; + + Ok(()) + } +}