progress commit: openxr + pipewire + refactor

src/backend/common.rs

@@ -18,6 +18,13 @@ use crate::{
 
 use super::overlay::{OverlayData, OverlayState};
 
+pub enum BackendError {
+    NotSupported,
+    Shutdown,
+    Restart,
+    Fatal,
+}
+
 pub struct OverlayContainer<T>
 where
     T: Default,
@@ -39,11 +46,13 @@ where
             get_screens_x11()
         };
 
-        //let watch = create_watch::<T>(&app, &screens);
-        //overlays.insert(watch.state.id, watch);
+        let mut watch = create_watch::<T>(&app, &screens);
+        watch.state.want_visible = false;
+        overlays.insert(watch.state.id, watch);
 
-        //let keyboard = create_keyboard(&app);
-        //overlays.insert(keyboard.state.id, keyboard);
+        let mut keyboard = create_keyboard(&app);
+        keyboard.state.want_visible = false;
+        overlays.insert(keyboard.state.id, keyboard);
 
         let mut first = true;
         for mut screen in screens {

src/backend/mod.rs

@@ -1,5 +1,7 @@
 pub mod common;
 pub mod input;
+#[cfg(feature = "openvr")]
 pub mod openvr;
+#[cfg(feature = "openxr")]
 pub mod openxr;
 pub mod overlay;

src/backend/openvr/lines.rs

@@ -29,7 +29,7 @@ impl LinePool {
 
         let buf = vec![255; 16];
 
-        let texture = command_buffer.texture2d(2, 2, Format::R8G8B8A8_UNORM, buf);
+        let texture = command_buffer.texture2d(2, 2, Format::R8G8B8A8_UNORM, &buf);
         command_buffer.build_and_execute_now();
 
         graphics

src/backend/openvr/mod.rs

@@ -1,6 +1,10 @@
 use glam::Vec4;
 use std::{
     collections::VecDeque,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
     time::{Duration, Instant},
 };
 
@@ -19,24 +23,27 @@ use crate::{
         input::interact,
         openvr::{input::OpenVrInputSource, lines::LinePool},
     },
+    graphics::WlxGraphics,
     state::AppState,
 };
 
 use self::{input::action_manifest_path, overlay::OpenVrOverlayData};
 
-use super::common::{OverlayContainer, TaskType};
+use super::common::{BackendError, OverlayContainer, TaskType};
 
 pub mod input;
 pub mod lines;
 pub mod overlay;
 
-pub fn openvr_run() {
+pub fn openvr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
     let app_type = EVRApplicationType::VRApplication_Overlay;
     let Ok(context) = ovr_overlay::Context::init(app_type) else {
-        log::error!("Failed to initialize OpenVR");
-        return;
+        log::warn!("Will not use OpenVR: Context init failed");
+        return Err(BackendError::NotSupported);
     };
 
+    log::info!("Using OpenVR runtime");
+
     let mut overlay_mngr = context.overlay_mngr();
     //let mut settings_mngr = context.settings_mngr();
     let mut input_mngr = context.input_mngr();
@@ -55,19 +62,23 @@ pub fn openvr_run() {
         InstanceExtensions::from_iter(names.iter().map(|s| s.as_str()))
     };
 
-    let mut state = AppState::new(instance_extensions, device_extensions_fn);
+    let mut state = {
+        let graphics = WlxGraphics::new(instance_extensions, device_extensions_fn);
+        AppState::from_graphics(graphics)
+    };
+
     let mut overlays = OverlayContainer::<OpenVrOverlayData>::new(&mut state);
 
     state.hid_provider.set_desktop_extent(overlays.extent);
 
     if let Err(e) = input_mngr.set_action_manifest(action_manifest_path()) {
         log::error!("Failed to set action manifest: {}", e.description());
-        return;
+        return Err(BackendError::Fatal);
     };
 
     let Ok(mut input_source) = OpenVrInputSource::new(&mut input_mngr) else {
         log::error!("Failed to initialize input");
-        return;
+        return Err(BackendError::Fatal);
     };
 
     let Ok(refresh_rate) = system_mngr.get_tracked_device_property::<f32>(
@@ -75,7 +86,7 @@ pub fn openvr_run() {
         ETrackedDeviceProperty::Prop_DisplayFrequency_Float,
     ) else {
         log::error!("Failed to get display refresh rate");
-        return;
+        return Err(BackendError::Fatal);
     };
 
     log::info!("HMD running @ {} Hz", refresh_rate);
@@ -90,12 +101,17 @@ pub fn openvr_run() {
         lines.allocate(&mut overlay_mngr, &mut state),
     ];
 
-    loop {
+    'main_loop: loop {
+        if !running.load(Ordering::Relaxed) {
+            log::warn!("Received shutdown signal.");
+            break 'main_loop;
+        }
+
         while let Some(event) = system_mngr.poll_next_event() {
             match event.event_type {
                 EVREventType::VREvent_Quit => {
-                    log::info!("Received quit event, shutting down.");
-                    return;
+                    log::warn!("Received quit event, shutting down.");
+                    break 'main_loop;
                 }
                 EVREventType::VREvent_TrackedDeviceActivated
                 | EVREventType::VREvent_TrackedDeviceDeactivated
@@ -173,4 +189,9 @@ pub fn openvr_run() {
             },
         ));
     }
+
+    log::warn!("OpenVR shutdown");
+    // context.shutdown() called by Drop
+
+    Ok(())
 }

src/backend/openxr/input.rs

@@ -0,0 +1,232 @@
+use glam::{bool, Affine3A, Quat, Vec3};
+use openxr as xr;
+
+use crate::{backend::input::Pointer, state::AppState};
+
+type XrSession = xr::Session<xr::Vulkan>;
+
+pub(super) struct OpenXrInputSource {
+    action_set: xr::ActionSet,
+    hands: [OpenXrHand; 2],
+    pub(super) stage: xr::Space,
+}
+
+pub(super) struct OpenXrHand {
+    source: OpenXrHandSource,
+    space: xr::Space,
+}
+
+pub(super) struct OpenXrHandSource {
+    action_pose: xr::Action<xr::Posef>,
+    action_click: xr::Action<bool>,
+    action_grab: xr::Action<bool>,
+    action_scroll: xr::Action<xr::Vector2f>,
+    action_alt_click: xr::Action<bool>,
+    action_show_hide: xr::Action<bool>,
+    action_click_modifier_right: xr::Action<bool>,
+    action_click_modifier_middle: xr::Action<bool>,
+    action_haptics: xr::Action<xr::Haptic>,
+}
+
+impl OpenXrInputSource {
+    pub fn new(session: XrSession) -> Self {
+        let mut action_set = session
+            .instance()
+            .create_action_set("wlx-overlay-s", "WlxOverlay-S Actions", 0)
+            .expect("Failed to create action set");
+
+        let left_source = OpenXrHandSource::new(&mut action_set, "left");
+        let right_source = OpenXrHandSource::new(&mut action_set, "right");
+
+        session.attach_action_sets(&[&action_set]).unwrap();
+
+        let stage = session
+            .create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)
+            .unwrap();
+
+        Self {
+            action_set,
+            hands: [
+                OpenXrHand::new(session.clone(), left_source),
+                OpenXrHand::new(session, right_source),
+            ],
+            stage,
+        }
+    }
+
+    pub fn update(&self, session: &XrSession, time: xr::Time, state: &mut AppState) {
+        session.sync_actions(&[(&self.action_set).into()]).unwrap();
+
+        for i in 0..2 {
+            self.hands[i].update(
+                &mut state.input_state.pointers[i],
+                &self.stage,
+                session,
+                time,
+            );
+        }
+    }
+}
+
+impl OpenXrHand {
+    pub(super) fn new(session: XrSession, source: OpenXrHandSource) -> Self {
+        let space = source
+            .action_pose
+            .create_space(session.clone(), xr::Path::NULL, xr::Posef::IDENTITY)
+            .unwrap();
+
+        Self { source, space }
+    }
+
+    pub(super) fn update(
+        &self,
+        pointer: &mut Pointer,
+        stage: &xr::Space,
+        session: &XrSession,
+        time: xr::Time,
+    ) {
+        let location = self.space.locate(stage, time).unwrap();
+        if location
+            .location_flags
+            .contains(xr::SpaceLocationFlags::ORIENTATION_VALID)
+        {
+            let quat = unsafe { std::mem::transmute::<_, Quat>(location.pose.orientation) };
+            let pos = unsafe { std::mem::transmute::<_, Vec3>(location.pose.position) };
+            pointer.pose = Affine3A::from_rotation_translation(quat, pos);
+        }
+
+        pointer.now.click = self
+            .source
+            .action_click
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+
+        pointer.now.grab = self
+            .source
+            .action_grab
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+
+        pointer.now.scroll = self
+            .source
+            .action_scroll
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state
+            .y;
+
+        pointer.now.alt_click = self
+            .source
+            .action_alt_click
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+
+        pointer.now.show_hide = self
+            .source
+            .action_show_hide
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+
+        pointer.now.click_modifier_right = self
+            .source
+            .action_click_modifier_right
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+
+        pointer.now.click_modifier_middle = self
+            .source
+            .action_click_modifier_middle
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+    }
+}
+
+// supported action types: Haptic, Posef, Vector2f, f32, bool
+impl OpenXrHandSource {
+    pub(super) fn new(action_set: &mut xr::ActionSet, side: &str) -> Self {
+        let action_pose = action_set
+            .create_action::<xr::Posef>(
+                &format!("{}_hand", side),
+                &format!("{} hand pose", side),
+                &[],
+            )
+            .unwrap();
+
+        let action_click = action_set
+            .create_action::<bool>(
+                &format!("{}_click", side),
+                &format!("{} hand click", side),
+                &[],
+            )
+            .unwrap();
+        let action_grab = action_set
+            .create_action::<bool>(
+                &format!("{}_grab", side),
+                &format!("{} hand grab", side),
+                &[],
+            )
+            .unwrap();
+        let action_scroll = action_set
+            .create_action::<xr::Vector2f>(
+                &format!("{}_scroll", side),
+                &format!("{} hand scroll", side),
+                &[],
+            )
+            .unwrap();
+        let action_alt_click = action_set
+            .create_action::<bool>(
+                &format!("{}_alt_click", side),
+                &format!("{} hand alt click", side),
+                &[],
+            )
+            .unwrap();
+        let action_show_hide = action_set
+            .create_action::<bool>(
+                &format!("{}_show_hide", side),
+                &format!("{} hand show/hide", side),
+                &[],
+            )
+            .unwrap();
+        let action_click_modifier_right = action_set
+            .create_action::<bool>(
+                &format!("{}_click_modifier_right", side),
+                &format!("{} hand right click modifier", side),
+                &[],
+            )
+            .unwrap();
+        let action_click_modifier_middle = action_set
+            .create_action::<bool>(
+                &format!("{}_click_modifier_middle", side),
+                &format!("{} hand middle click modifier", side),
+                &[],
+            )
+            .unwrap();
+        let action_haptics = action_set
+            .create_action::<xr::Haptic>(
+                &format!("{}_haptics", side),
+                &format!("{} hand haptics", side),
+                &[],
+            )
+            .unwrap();
+
+        // TODO suggest bindings
+
+        Self {
+            action_pose,
+            action_click,
+            action_grab,
+            action_scroll,
+            action_alt_click,
+            action_show_hide,
+            action_click_modifier_right,
+            action_click_modifier_middle,
+            action_haptics,
+        }
+    }
+}

src/backend/openxr/mod.rs

@@ -0,0 +1,363 @@
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+
+use anyhow::{bail, ensure};
+use ash::vk::{self};
+use glam::{Affine3A, Quat, Vec3};
+use openxr as xr;
+use vulkano::{
+    image::{view::ImageView, ImageCreateInfo, ImageUsage},
+    render_pass::{Framebuffer, FramebufferCreateInfo},
+    Handle, VulkanObject,
+};
+
+use crate::{
+    backend::{common::OverlayContainer, input::interact, openxr::overlay::OpenXrOverlayData},
+    graphics::WlxGraphics,
+    state::AppState,
+};
+
+use super::common::BackendError;
+
+mod input;
+mod lines;
+mod overlay;
+
+const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
+const VIEW_COUNT: u32 = 2;
+
+pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
+    let (xr_instance, system) = match init_xr() {
+        Ok((xr_instance, system)) => (xr_instance, system),
+        Err(e) => {
+            log::warn!("Will not use OpenXR: {}", e);
+            return Err(BackendError::NotSupported);
+        }
+    };
+
+    let environment_blend_mode = xr_instance
+        .enumerate_environment_blend_modes(system, VIEW_TYPE)
+        .unwrap()[0];
+    log::info!("Using environment blend mode: {:?}", environment_blend_mode);
+
+    let mut state = {
+        let graphics = WlxGraphics::new_xr(xr_instance.clone(), system);
+        AppState::from_graphics(graphics)
+    };
+
+    let mut overlays = OverlayContainer::<OpenXrOverlayData>::new(&mut state);
+
+    state.hid_provider.set_desktop_extent(overlays.extent);
+
+    let (session, mut frame_wait, mut frame_stream) = unsafe {
+        xr_instance
+            .create_session::<xr::Vulkan>(
+                system,
+                &xr::vulkan::SessionCreateInfo {
+                    instance: state.graphics.instance.handle().as_raw() as _,
+                    physical_device: state.graphics.device.physical_device().handle().as_raw() as _,
+                    device: state.graphics.device.handle().as_raw() as _,
+                    queue_family_index: state.graphics.queue.queue_family_index(),
+                    queue_index: 0,
+                },
+            )
+            .unwrap()
+    };
+
+    let input_source = input::OpenXrInputSource::new(session.clone());
+
+    let mut swapchain = None;
+    let mut session_running = false;
+    let mut event_storage = xr::EventDataBuffer::new();
+
+    'main_loop: loop {
+        if !running.load(Ordering::Relaxed) {
+            log::warn!("Received shutdown signal.");
+            match session.request_exit() {
+                Ok(_) => log::info!("OpenXR session exit requested."),
+                Err(xr::sys::Result::ERROR_SESSION_NOT_RUNNING) => break 'main_loop,
+                Err(e) => {
+                    log::error!("Failed to request OpenXR session exit: {}", e);
+                    break 'main_loop;
+                }
+            }
+        }
+
+        while let Some(event) = xr_instance.poll_event(&mut event_storage).unwrap() {
+            use xr::Event::*;
+            match event {
+                SessionStateChanged(e) => {
+                    // Session state change is where we can begin and end sessions, as well as
+                    // find quit messages!
+                    println!("entered state {:?}", e.state());
+                    match e.state() {
+                        xr::SessionState::READY => {
+                            session.begin(VIEW_TYPE).unwrap();
+                            session_running = true;
+                        }
+                        xr::SessionState::STOPPING => {
+                            session.end().unwrap();
+                            session_running = false;
+                        }
+                        xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => {
+                            break 'main_loop;
+                        }
+                        _ => {}
+                    }
+                }
+                InstanceLossPending(_) => {
+                    break 'main_loop;
+                }
+                EventsLost(e) => {
+                    println!("lost {} events", e.lost_event_count());
+                }
+                _ => {}
+            }
+        }
+
+        if !session_running {
+            std::thread::sleep(Duration::from_millis(100));
+            continue 'main_loop;
+        }
+
+        let xr_frame_state = frame_wait.wait().unwrap();
+        frame_stream.begin().unwrap();
+
+        if !xr_frame_state.should_render {
+            frame_stream
+                .end(
+                    xr_frame_state.predicted_display_time,
+                    environment_blend_mode,
+                    &[],
+                )
+                .unwrap();
+            continue 'main_loop;
+        }
+
+        state.input_state.pre_update();
+        input_source.update(&session, xr_frame_state.predicted_display_time, &mut state);
+        state.input_state.post_update();
+
+        let (_, views) = session
+            .locate_views(
+                VIEW_TYPE,
+                xr_frame_state.predicted_display_time,
+                &input_source.stage,
+            )
+            .unwrap();
+
+        state.input_state.hmd = hmd_pose_from_views(&views);
+
+        let _pointer_lengths = interact(&mut overlays, &mut state);
+
+        //TODO lines
+
+        overlays
+            .iter_mut()
+            .filter(|o| o.state.want_visible)
+            .for_each(|o| o.render(&mut state));
+
+        state.hid_provider.on_new_frame();
+
+        let swapchain = swapchain.get_or_insert_with(|| {
+            let views = xr_instance
+                .enumerate_view_configuration_views(system, VIEW_TYPE)
+                .unwrap();
+            debug_assert_eq!(views.len(), VIEW_COUNT as usize);
+            debug_assert_eq!(views[0], views[1]);
+
+            let resolution = vk::Extent2D {
+                width: views[0].recommended_image_rect_width,
+                height: views[0].recommended_image_rect_height,
+            };
+            log::info!(
+                "Swapchain resolution: {}x{}",
+                resolution.width,
+                resolution.height
+            );
+            let swapchain = session
+                .create_swapchain(&xr::SwapchainCreateInfo {
+                    create_flags: xr::SwapchainCreateFlags::EMPTY,
+                    usage_flags: xr::SwapchainUsageFlags::COLOR_ATTACHMENT
+                        | xr::SwapchainUsageFlags::SAMPLED,
+                    format: state.graphics.native_format as _,
+                    sample_count: 1,
+                    width: resolution.width,
+                    height: resolution.height,
+                    face_count: 1,
+                    array_size: VIEW_COUNT,
+                    mip_count: 1,
+                })
+                .unwrap();
+
+            // thanks @yshui
+            let swapchain_images = swapchain
+                .enumerate_images()
+                .unwrap()
+                .into_iter()
+                .map(|handle| {
+                    let vk_image = vk::Image::from_raw(handle);
+                    let raw_image = unsafe {
+                        vulkano::image::sys::RawImage::from_handle(
+                            state.graphics.device.clone(),
+                            vk_image,
+                            ImageCreateInfo {
+                                format: state.graphics.native_format,
+                                extent: [resolution.width * 2, resolution.height, 1],
+                                usage: ImageUsage::COLOR_ATTACHMENT | ImageUsage::TRANSFER_DST,
+                                ..Default::default()
+                            },
+                        )
+                        .unwrap()
+                    };
+                    // SAFETY: OpenXR guarantees that the image is a swapchain image, thus has memory backing it.
+                    let image = Arc::new(unsafe { raw_image.assume_bound() });
+                    let view = ImageView::new_default(image).unwrap();
+                    let fb = Framebuffer::new(
+                        todo!(),
+                        FramebufferCreateInfo {
+                            attachments: vec![view.clone()],
+                            ..Default::default()
+                        },
+                    )
+                    .unwrap();
+
+                    XrFramebuffer {
+                        framebuffer: fb,
+                        color: view,
+                    }
+                })
+                .collect();
+
+            XrSwapchain {
+                handle: swapchain,
+                buffers: swapchain_images,
+                resolution: [resolution.width, resolution.height, 1],
+            }
+        });
+
+        let image_index = swapchain.handle.acquire_image().unwrap();
+    }
+
+    Ok(())
+}
+
+fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
+    let Ok(entry) = (unsafe { xr::Entry::load() }) else {
+        bail!("OpenXR Loader not found.");
+    };
+
+    let Ok(available_extensions) = entry.enumerate_extensions() else {
+        bail!("Failed to enumerate OpenXR extensions.");
+    };
+    ensure!(
+        available_extensions.khr_vulkan_enable2,
+        "Missing KHR_vulkan_enable2 extension."
+    );
+    ensure!(
+        available_extensions.extx_overlay,
+        "Missing EXTX_overlay extension."
+    );
+
+    let mut enabled_extensions = xr::ExtensionSet::default();
+    enabled_extensions.khr_vulkan_enable2 = true;
+    enabled_extensions.extx_overlay = true;
+
+    #[cfg(not(debug_assertions))]
+    let layers = [];
+    #[cfg(debug_assertions)]
+    let layers = [
+        "XR_APILAYER_LUNARG_api_dump",
+        "XR_APILAYER_LUNARG_standard_validation",
+    ];
+
+    let Ok(xr_instance) = entry.create_instance(
+        &xr::ApplicationInfo {
+            application_name: "wlx-overlay-s",
+            application_version: 0,
+            engine_name: "wlx-overlay-s",
+            engine_version: 0,
+        },
+        &enabled_extensions,
+        &layers,
+    ) else {
+        bail!("Failed to create OpenXR instance.");
+    };
+
+    let Ok(instance_props) = xr_instance.properties() else {
+        bail!("Failed to query OpenXR instance properties.");
+    };
+    log::info!(
+        "Using OpenXR runtime: {} {}",
+        instance_props.runtime_name,
+        instance_props.runtime_version
+    );
+
+    let Ok(system) = xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY) else {
+        bail!("Failed to access OpenXR HMD system.");
+    };
+
+    let vk_target_version_xr = xr::Version::new(1, 1, 0);
+
+    let Ok(reqs) = xr_instance.graphics_requirements::<xr::Vulkan>(system) else {
+        bail!("Failed to query OpenXR Vulkan requirements.");
+    };
+
+    if vk_target_version_xr < reqs.min_api_version_supported
+        || vk_target_version_xr.major() > reqs.max_api_version_supported.major()
+    {
+        bail!(
+            "OpenXR runtime requires Vulkan version > {}, < {}.0.0",
+            reqs.min_api_version_supported,
+            reqs.max_api_version_supported.major() + 1
+        );
+    }
+
+    Ok((xr_instance, system))
+}
+
+struct XrSwapchain {
+    handle: xr::Swapchain<xr::Vulkan>,
+    buffers: Vec<XrFramebuffer>,
+    resolution: [u32; 3],
+}
+
+struct XrFramebuffer {
+    framebuffer: Arc<Framebuffer>,
+    color: Arc<ImageView>,
+}
+
+fn hmd_pose_from_views(views: &Vec<xr::View>) -> Affine3A {
+    let pos = {
+        let pos0: Vec3 = unsafe { std::mem::transmute(views[0].pose.position) };
+        let pos1: Vec3 = unsafe { std::mem::transmute(views[1].pose.position) };
+        (pos0 + pos1) * 0.5
+    };
+    let rot = {
+        let rot0 = unsafe { std::mem::transmute(views[0].pose.orientation) };
+        let rot1 = unsafe { std::mem::transmute(views[1].pose.orientation) };
+        quat_lerp(rot0, rot1, 0.5)
+    };
+
+    Affine3A::from_rotation_translation(rot, pos)
+}
+
+fn quat_lerp(a: Quat, mut b: Quat, t: f32) -> Quat {
+    let l2 = a.dot(b);
+    if l2 < 0.0 {
+        b = -b;
+    }
+
+    Quat::from_xyzw(
+        a.x - t * (a.x - b.x),
+        a.y - t * (a.y - b.y),
+        a.z - t * (a.z - b.z),
+        a.w - t * (a.w - b.w),
+    )
+    .normalize()
+}

src/backend/openxr/overlay.rs

@@ -0,0 +1,7 @@
+pub struct OpenXrOverlayData {}
+
+impl Default for OpenXrOverlayData {
+    fn default() -> Self {
+        Self {}
+    }
+}