feature: osc sender

Cargo.lock

@@ -2885,6 +2885,16 @@ dependencies = [
  "hound",
 ]
 
+[[package]]
+name = "rosc"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2e63d9e6b0d090be1485cf159b1e04c3973d2d3e1614963544ea2ff47a4a981"
+dependencies = [
+ "byteorder",
+ "nom",
+]
+
 [[package]]
 name = "roxmltree"
 version = "0.14.1"
@@ -4295,6 +4305,7 @@ dependencies = [
  "raw-window-handle 0.5.2",
  "regex",
  "rodio",
+ "rosc",
  "serde",
  "serde_json",
  "serde_yaml",

Cargo.toml

@@ -35,6 +35,7 @@ png = "0.17.10"
 raw-window-handle = "0.5.2"
 regex = "1.9.5"
 rodio = { version = "0.17.1", default-features = false, features = ["wav", "hound"] }
+rosc = { version = "0.10.1", optional = true }
 serde = { version = "1.0.188", features = ["derive", "rc"] }
 serde_json = "1.0.113"
 serde_yaml = "0.9.25"
@@ -50,5 +51,6 @@ xdg = "2.5.2"
 [features]
 openvr = ["dep:ovr_overlay", "dep:json"]
 openxr = ["dep:openxr"]
-default = ["openvr", "openxr"]
+osc = ["dep:rosc"]
+default = ["openvr", "openxr", "osc"]
 

src/backend/mod.rs

@@ -4,4 +4,6 @@ pub mod input;
 pub mod openvr;
 #[cfg(feature = "openxr")]
 pub mod openxr;
+#[cfg(feature = "osc")]
+pub mod osc;
 pub mod overlay;

src/backend/openxr/helpers.rs

@@ -0,0 +1,164 @@
+use anyhow::{bail, ensure};
+use glam::{Affine3A, Quat, Vec3};
+use openxr as xr;
+use xr::OverlaySessionCreateFlagsEXTX;
+
+pub(super) fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
+    let Ok(entry) = (unsafe { xr::Entry::load() }) else {
+        bail!("OpenXR Loader not found.");
+    };
+
+    let Ok(available_extensions) = entry.enumerate_extensions() else {
+        bail!("Failed to enumerate OpenXR extensions.");
+    };
+    ensure!(
+        available_extensions.khr_vulkan_enable2,
+        "Missing KHR_vulkan_enable2 extension."
+    );
+    ensure!(
+        available_extensions.extx_overlay,
+        "Missing EXTX_overlay extension."
+    );
+
+    let mut enabled_extensions = xr::ExtensionSet::default();
+    enabled_extensions.khr_vulkan_enable2 = true;
+    enabled_extensions.extx_overlay = true;
+
+    //#[cfg(not(debug_assertions))]
+    let layers = [];
+    //#[cfg(debug_assertions)]
+    //let layers = [
+    //    "XR_APILAYER_LUNARG_api_dump",
+    //    "XR_APILAYER_LUNARG_standard_validation",
+    //];
+
+    let Ok(xr_instance) = entry.create_instance(
+        &xr::ApplicationInfo {
+            application_name: "wlx-overlay-s",
+            application_version: 0,
+            engine_name: "wlx-overlay-s",
+            engine_version: 0,
+        },
+        &enabled_extensions,
+        &layers,
+    ) else {
+        bail!("Failed to create OpenXR instance.");
+    };
+
+    let Ok(instance_props) = xr_instance.properties() else {
+        bail!("Failed to query OpenXR instance properties.");
+    };
+    log::info!(
+        "Using OpenXR runtime: {} {}",
+        instance_props.runtime_name,
+        instance_props.runtime_version
+    );
+
+    let Ok(system) = xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY) else {
+        bail!("Failed to access OpenXR HMD system.");
+    };
+
+    let vk_target_version_xr = xr::Version::new(1, 1, 0);
+
+    let Ok(reqs) = xr_instance.graphics_requirements::<xr::Vulkan>(system) else {
+        bail!("Failed to query OpenXR Vulkan requirements.");
+    };
+
+    if vk_target_version_xr < reqs.min_api_version_supported
+        || vk_target_version_xr.major() > reqs.max_api_version_supported.major()
+    {
+        bail!(
+            "OpenXR runtime requires Vulkan version > {}, < {}.0.0",
+            reqs.min_api_version_supported,
+            reqs.max_api_version_supported.major() + 1
+        );
+    }
+
+    Ok((xr_instance, system))
+}
+pub(super) unsafe fn create_overlay_session(
+    instance: &xr::Instance,
+    system: xr::SystemId,
+    info: &xr::vulkan::SessionCreateInfo,
+) -> Result<xr::sys::Session, xr::sys::Result> {
+    let overlay = xr::sys::SessionCreateInfoOverlayEXTX {
+        ty: xr::sys::SessionCreateInfoOverlayEXTX::TYPE,
+        next: std::ptr::null(),
+        create_flags: OverlaySessionCreateFlagsEXTX::EMPTY,
+        session_layers_placement: 5,
+    };
+    let binding = xr::sys::GraphicsBindingVulkanKHR {
+        ty: xr::sys::GraphicsBindingVulkanKHR::TYPE,
+        next: &overlay as *const _ as *const _,
+        instance: info.instance,
+        physical_device: info.physical_device,
+        device: info.device,
+        queue_family_index: info.queue_family_index,
+        queue_index: info.queue_index,
+    };
+    let info = xr::sys::SessionCreateInfo {
+        ty: xr::sys::SessionCreateInfo::TYPE,
+        next: &binding as *const _ as *const _,
+        create_flags: Default::default(),
+        system_id: system,
+    };
+    let mut out = xr::sys::Session::NULL;
+    let x = (instance.fp().create_session)(instance.as_raw(), &info, &mut out);
+    if x.into_raw() >= 0 {
+        Ok(out)
+    } else {
+        Err(x)
+    }
+}
+
+pub(super) fn hmd_pose_from_views(views: &Vec<xr::View>) -> Affine3A {
+    let pos = {
+        let pos0: Vec3 = unsafe { std::mem::transmute(views[0].pose.position) };
+        let pos1: Vec3 = unsafe { std::mem::transmute(views[1].pose.position) };
+        (pos0 + pos1) * 0.5
+    };
+    let rot = {
+        let rot0 = unsafe { std::mem::transmute(views[0].pose.orientation) };
+        let rot1 = unsafe { std::mem::transmute(views[1].pose.orientation) };
+        quat_lerp(rot0, rot1, 0.5)
+    };
+
+    Affine3A::from_rotation_translation(rot, pos)
+}
+
+fn quat_lerp(a: Quat, mut b: Quat, t: f32) -> Quat {
+    let l2 = a.dot(b);
+    if l2 < 0.0 {
+        b = -b;
+    }
+
+    Quat::from_xyzw(
+        a.x - t * (a.x - b.x),
+        a.y - t * (a.y - b.y),
+        a.z - t * (a.z - b.z),
+        a.w - t * (a.w - b.w),
+    )
+    .normalize()
+}
+
+pub(super) fn transform_to_posef(transform: &Affine3A) -> xr::Posef {
+    let translation = transform.translation;
+    let norm_mat3 = transform
+        .matrix3
+        .mul_scalar(1.0 / transform.matrix3.x_axis.length());
+    let rotation = Quat::from_mat3a(&norm_mat3).normalize();
+
+    xr::Posef {
+        orientation: xr::Quaternionf {
+            x: rotation.x,
+            y: rotation.y,
+            z: rotation.z,
+            w: rotation.w,
+        },
+        position: xr::Vector3f {
+            x: translation.x,
+            y: translation.y,
+            z: translation.z,
+        },
+    }
+}

src/backend/openxr/lines.rs

@@ -11,11 +11,14 @@ use std::{
 
 use vulkano::{command_buffer::CommandBufferUsage, format::Format, image::view::ImageView};
 
-use crate::graphics::{WlxCommandBuffer, WlxGraphics};
+use crate::{
+    backend::openxr::helpers,
+    graphics::{WlxCommandBuffer, WlxGraphics},
+};
 
 use super::{
     swapchain::{create_swapchain_render_data, SwapchainRenderData},
-    transform_to_posef, XrState,
+    XrState,
 };
 
 static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(1);
@@ -111,7 +114,7 @@ impl LinePool {
 
         transform = transform * rotations[closest.0];
 
-        let posef = transform_to_posef(&transform);
+        let posef = helpers::transform_to_posef(&transform);
 
         line.maybe_line = Some(Line {
             view: self.colors[color].clone(),

src/backend/openxr/mod.rs

@@ -7,17 +7,15 @@ use std::{
     time::Duration,
 };
 
-use anyhow::{bail, ensure};
-use glam::{Affine3A, Quat, Vec3};
 use openxr as xr;
 use vulkano::{command_buffer::CommandBufferUsage, Handle, VulkanObject};
-use xr::OverlaySessionCreateFlagsEXTX;
 
 use crate::{
     backend::{
         common::{OverlayContainer, TaskType},
         input::interact,
         openxr::{input::DoubleClickCounter, lines::LinePool, overlay::OpenXrOverlayData},
+        osc::OscSender,
     },
     graphics::WlxGraphics,
     state::AppState,
@@ -25,6 +23,7 @@ use crate::{
 
 use super::common::BackendError;
 
+mod helpers;
 mod input;
 mod lines;
 mod overlay;
@@ -42,7 +41,7 @@ struct XrState {
 }
 
 pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
-    let (xr_instance, system) = match init_xr() {
+    let (xr_instance, system) = match helpers::init_xr() {
         Ok((xr_instance, system)) => (xr_instance, system),
         Err(e) => {
             log::warn!("Will not use OpenXR: {}", e);
@@ -63,10 +62,13 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
     let mut overlays = OverlayContainer::<OpenXrOverlayData>::new(&mut app_state);
     let mut lines = LinePool::new(app_state.graphics.clone());
 
+    #[cfg(feature = "osc")]
+    let mut osc_sender = OscSender::new(9000).ok();
+
     app_state.hid_provider.set_desktop_extent(overlays.extent);
 
     let (session, mut frame_wait, mut frame_stream) = unsafe {
-        let raw_session = create_overlay_session(
+        let raw_session = helpers::create_overlay_session(
             &xr_instance,
             system,
             &xr::vulkan::SessionCreateInfo {
@@ -212,6 +214,11 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
             .iter_mut()
             .for_each(|o| o.after_input(&mut app_state));
 
+        #[cfg(feature = "osc")]
+        if let Some(ref mut sender) = osc_sender {
+            let _ = sender.send_params(&overlays);
+        };
+
         let (_, views) = xr_state
             .session
             .locate_views(
@@ -221,7 +228,7 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
             )
             .unwrap();
 
-        app_state.input_state.hmd = hmd_pose_from_views(&views);
+        app_state.input_state.hmd = helpers::hmd_pose_from_views(&views);
 
         overlays
             .iter_mut()
@@ -296,163 +303,3 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
 
     Ok(())
 }
-
-fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
-    let Ok(entry) = (unsafe { xr::Entry::load() }) else {
-        bail!("OpenXR Loader not found.");
-    };
-
-    let Ok(available_extensions) = entry.enumerate_extensions() else {
-        bail!("Failed to enumerate OpenXR extensions.");
-    };
-    ensure!(
-        available_extensions.khr_vulkan_enable2,
-        "Missing KHR_vulkan_enable2 extension."
-    );
-    ensure!(
-        available_extensions.extx_overlay,
-        "Missing EXTX_overlay extension."
-    );
-
-    let mut enabled_extensions = xr::ExtensionSet::default();
-    enabled_extensions.khr_vulkan_enable2 = true;
-    enabled_extensions.extx_overlay = true;
-
-    //#[cfg(not(debug_assertions))]
-    let layers = [];
-    //#[cfg(debug_assertions)]
-    //let layers = [
-    //    "XR_APILAYER_LUNARG_api_dump",
-    //    "XR_APILAYER_LUNARG_standard_validation",
-    //];
-
-    let Ok(xr_instance) = entry.create_instance(
-        &xr::ApplicationInfo {
-            application_name: "wlx-overlay-s",
-            application_version: 0,
-            engine_name: "wlx-overlay-s",
-            engine_version: 0,
-        },
-        &enabled_extensions,
-        &layers,
-    ) else {
-        bail!("Failed to create OpenXR instance.");
-    };
-
-    let Ok(instance_props) = xr_instance.properties() else {
-        bail!("Failed to query OpenXR instance properties.");
-    };
-    log::info!(
-        "Using OpenXR runtime: {} {}",
-        instance_props.runtime_name,
-        instance_props.runtime_version
-    );
-
-    let Ok(system) = xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY) else {
-        bail!("Failed to access OpenXR HMD system.");
-    };
-
-    let vk_target_version_xr = xr::Version::new(1, 1, 0);
-
-    let Ok(reqs) = xr_instance.graphics_requirements::<xr::Vulkan>(system) else {
-        bail!("Failed to query OpenXR Vulkan requirements.");
-    };
-
-    if vk_target_version_xr < reqs.min_api_version_supported
-        || vk_target_version_xr.major() > reqs.max_api_version_supported.major()
-    {
-        bail!(
-            "OpenXR runtime requires Vulkan version > {}, < {}.0.0",
-            reqs.min_api_version_supported,
-            reqs.max_api_version_supported.major() + 1
-        );
-    }
-
-    Ok((xr_instance, system))
-}
-unsafe fn create_overlay_session(
-    instance: &xr::Instance,
-    system: xr::SystemId,
-    info: &xr::vulkan::SessionCreateInfo,
-) -> Result<xr::sys::Session, xr::sys::Result> {
-    let overlay = xr::sys::SessionCreateInfoOverlayEXTX {
-        ty: xr::sys::SessionCreateInfoOverlayEXTX::TYPE,
-        next: std::ptr::null(),
-        create_flags: OverlaySessionCreateFlagsEXTX::EMPTY,
-        session_layers_placement: 5,
-    };
-    let binding = xr::sys::GraphicsBindingVulkanKHR {
-        ty: xr::sys::GraphicsBindingVulkanKHR::TYPE,
-        next: &overlay as *const _ as *const _,
-        instance: info.instance,
-        physical_device: info.physical_device,
-        device: info.device,
-        queue_family_index: info.queue_family_index,
-        queue_index: info.queue_index,
-    };
-    let info = xr::sys::SessionCreateInfo {
-        ty: xr::sys::SessionCreateInfo::TYPE,
-        next: &binding as *const _ as *const _,
-        create_flags: Default::default(),
-        system_id: system,
-    };
-    let mut out = xr::sys::Session::NULL;
-    let x = (instance.fp().create_session)(instance.as_raw(), &info, &mut out);
-    if x.into_raw() >= 0 {
-        Ok(out)
-    } else {
-        Err(x)
-    }
-}
-
-fn hmd_pose_from_views(views: &Vec<xr::View>) -> Affine3A {
-    let pos = {
-        let pos0: Vec3 = unsafe { std::mem::transmute(views[0].pose.position) };
-        let pos1: Vec3 = unsafe { std::mem::transmute(views[1].pose.position) };
-        (pos0 + pos1) * 0.5
-    };
-    let rot = {
-        let rot0 = unsafe { std::mem::transmute(views[0].pose.orientation) };
-        let rot1 = unsafe { std::mem::transmute(views[1].pose.orientation) };
-        quat_lerp(rot0, rot1, 0.5)
-    };
-
-    Affine3A::from_rotation_translation(rot, pos)
-}
-
-fn quat_lerp(a: Quat, mut b: Quat, t: f32) -> Quat {
-    let l2 = a.dot(b);
-    if l2 < 0.0 {
-        b = -b;
-    }
-
-    Quat::from_xyzw(
-        a.x - t * (a.x - b.x),
-        a.y - t * (a.y - b.y),
-        a.z - t * (a.z - b.z),
-        a.w - t * (a.w - b.w),
-    )
-    .normalize()
-}
-
-fn transform_to_posef(transform: &Affine3A) -> xr::Posef {
-    let translation = transform.translation;
-    let norm_mat3 = transform
-        .matrix3
-        .mul_scalar(1.0 / transform.matrix3.x_axis.length());
-    let rotation = Quat::from_mat3a(&norm_mat3).normalize();
-
-    xr::Posef {
-        orientation: xr::Quaternionf {
-            x: rotation.x,
-            y: rotation.y,
-            z: rotation.z,
-            w: rotation.w,
-        },
-        position: xr::Vector3f {
-            x: translation.x,
-            y: translation.y,
-            z: translation.z,
-        },
-    }
-}

src/backend/openxr/overlay.rs

@@ -2,7 +2,7 @@ use openxr as xr;
 use std::sync::Arc;
 use xr::{CompositionLayerFlags, EyeVisibility};
 
-use super::{swapchain::SwapchainRenderData, transform_to_posef, XrState};
+use super::{helpers, swapchain::SwapchainRenderData, XrState};
 use crate::{
     backend::{openxr::swapchain::create_swapchain_render_data, overlay::OverlayData},
     graphics::WlxCommandBuffer,
@@ -51,7 +51,7 @@ impl OverlayData<OpenXrOverlayData> {
         });
 
         let sub_image = data.acquire_present_release(command_buffer, my_view);
-        let posef = transform_to_posef(&self.state.transform);
+        let posef = helpers::transform_to_posef(&self.state.transform);
 
         let scale_x = self.state.transform.matrix3.col(0).length();
         let aspect_ratio = self.backend.extent()[1] as f32 / self.backend.extent()[0] as f32;

src/backend/osc.rs

@@ -0,0 +1,91 @@
+use std::{
+    net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
+    time::Instant,
+};
+
+use anyhow::bail;
+use rosc::{OscMessage, OscPacket, OscType};
+
+use crate::overlays::{keyboard::KEYBOARD_NAME, watch::WATCH_NAME};
+
+use super::common::OverlayContainer;
+
+pub struct OscSender {
+    last_sent: Instant,
+    upstream: UdpSocket,
+}
+
+impl OscSender {
+    pub fn new(send_port: u16) -> anyhow::Result<Self> {
+        let ip = IpAddr::V4(Ipv4Addr::LOCALHOST);
+
+        let Ok(upstream) = UdpSocket::bind("0.0.0.0:0") else {
+            bail!("Failed to bind UDP socket - OSC will not function.");
+        };
+
+        let Ok(_) = upstream.connect(SocketAddr::new(ip, send_port)) else {
+            bail!("Failed to connect UDP socket - OSC will not function.");
+        };
+
+        Ok(Self {
+            upstream,
+            last_sent: Instant::now(),
+        })
+    }
+
+    pub fn send_message(&self, addr: String, args: Vec<OscType>) -> anyhow::Result<()> {
+        let packet = OscPacket::Message(OscMessage { addr, args });
+        let Ok(bytes) = rosc::encoder::encode(&packet) else {
+            bail!("Could not encode OSC packet.");
+        };
+
+        let Ok(_) = self.upstream.send(&bytes) else {
+            bail!("Could not send OSC packet.");
+        };
+
+        Ok(())
+    }
+
+    pub fn send_params<D>(&mut self, overlays: &OverlayContainer<D>) -> anyhow::Result<()>
+    where
+        D: Default,
+    {
+        if self.last_sent.elapsed().as_millis() < 100 {
+            return Ok(());
+        }
+
+        let mut num_overlays = 0;
+        let mut has_keyboard = false;
+        let mut has_wrist = false;
+
+        for o in overlays.iter() {
+            if !o.state.want_visible {
+                continue;
+            }
+            match o.state.name.as_ref() {
+                WATCH_NAME => has_wrist = true,
+                KEYBOARD_NAME => has_keyboard = true,
+                _ => num_overlays += 1,
+            }
+        }
+
+        self.send_message(
+            "/avatar/parameters/isOverlayOpen".into(),
+            vec![OscType::Bool(num_overlays > 0)],
+        )?;
+        self.send_message(
+            "/avatar/parameters/isKeyboardOpen".into(),
+            vec![OscType::Bool(has_keyboard)],
+        )?;
+        self.send_message(
+            "/avatar/parameters/isWristVisible".into(),
+            vec![OscType::Bool(has_wrist)],
+        )?;
+        self.send_message(
+            "/avatar/parameters/openOverlayCount".into(),
+            vec![OscType::Int(num_overlays)],
+        )?;
+
+        Ok(())
+    }
+}