new workspace

wlx-overlay-s/src/backend/openxr/blocker.rs

@@ -0,0 +1,73 @@
+use libmonado::{ClientState, Monado};
+use log::{info, warn};
+
+use crate::{backend::overlay::OverlayID, state::AppState};
+
+pub(super) struct InputBlocker {
+    hovered_last_frame: bool,
+}
+
+impl InputBlocker {
+    pub const fn new() -> Self {
+        Self {
+            hovered_last_frame: false,
+        }
+    }
+
+    pub fn update(&mut self, state: &AppState, watch_id: OverlayID, monado: &mut Monado) {
+        if !state.session.config.block_game_input {
+            return;
+        }
+
+        let any_hovered = state.input_state.pointers.iter().any(|p| {
+            p.interaction.hovered_id.is_some_and(|id| {
+                id != watch_id || !state.session.config.block_game_input_ignore_watch
+            })
+        });
+
+        match (any_hovered, self.hovered_last_frame) {
+            (true, false) => {
+                info!("Blocking input");
+                set_clients_io_active(monado, false);
+            }
+            (false, true) => {
+                info!("Unblocking input");
+                set_clients_io_active(monado, true);
+            }
+            _ => {}
+        }
+
+        self.hovered_last_frame = any_hovered;
+    }
+}
+
+fn set_clients_io_active(monado: &mut Monado, active: bool) {
+    match monado.clients() {
+        Ok(clients) => {
+            for mut client in clients {
+                let name = match client.name() {
+                    Ok(n) => n,
+                    Err(e) => {
+                        warn!("Failed to get client name: {e}");
+                        continue;
+                    }
+                };
+
+                let state = match client.state() {
+                    Ok(s) => s,
+                    Err(e) => {
+                        warn!("Failed to get client state: {e}");
+                        continue;
+                    }
+                };
+
+                if name != "wlx-overlay-s" && state.contains(ClientState::ClientSessionVisible) {
+                    if let Err(e) = client.set_io_active(active) {
+                        warn!("Failed to set io active for client: {e}");
+                    }
+                }
+            }
+        }
+        Err(e) => warn!("Failed to get clients from Monado: {e}"),
+    }
+}

wlx-overlay-s/src/backend/openxr/helpers.rs

@@ -0,0 +1,190 @@
+use anyhow::{bail, ensure};
+use glam::{Affine3A, Quat, Vec3, Vec3A};
+use openxr::{self as xr, SessionCreateFlags, Version};
+use xr::OverlaySessionCreateFlagsEXTX;
+
+pub(super) fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
+    let entry = xr::Entry::linked();
+
+    let Ok(available_extensions) = entry.enumerate_extensions() else {
+        bail!("Failed to enumerate OpenXR extensions.");
+    };
+    ensure!(
+        available_extensions.khr_vulkan_enable2,
+        "Missing KHR_vulkan_enable2 extension."
+    );
+    ensure!(
+        available_extensions.extx_overlay,
+        "Missing EXTX_overlay extension."
+    );
+
+    let mut enabled_extensions = xr::ExtensionSet::default();
+    enabled_extensions.khr_vulkan_enable2 = true;
+    enabled_extensions.extx_overlay = true;
+    if available_extensions.khr_binding_modification && available_extensions.ext_dpad_binding {
+        enabled_extensions.khr_binding_modification = true;
+        enabled_extensions.ext_dpad_binding = true;
+    } else {
+        log::warn!("Missing EXT_dpad_binding extension.");
+    }
+    if available_extensions.ext_hp_mixed_reality_controller {
+        enabled_extensions.ext_hp_mixed_reality_controller = true;
+    } else {
+        log::warn!("Missing EXT_hp_mixed_reality_controller extension.");
+    }
+    if available_extensions.khr_composition_layer_cylinder {
+        enabled_extensions.khr_composition_layer_cylinder = true;
+    } else {
+        log::warn!("Missing EXT_composition_layer_cylinder extension.");
+    }
+    if available_extensions.khr_composition_layer_equirect2 {
+        enabled_extensions.khr_composition_layer_equirect2 = true;
+    } else {
+        log::warn!("Missing EXT_composition_layer_equirect2 extension.");
+    }
+    if available_extensions
+        .other
+        .contains(&"XR_MNDX_system_buttons".to_owned())
+    {
+        enabled_extensions
+            .other
+            .push("XR_MNDX_system_buttons".to_owned());
+    }
+
+    //#[cfg(not(debug_assertions))]
+    let layers = [];
+    //#[cfg(debug_assertions)]
+    //let layers = [
+    //    "XR_APILAYER_LUNARG_api_dump",
+    //    "XR_APILAYER_LUNARG_standard_validation",
+    //];
+
+    let Ok(xr_instance) = entry.create_instance(
+        &xr::ApplicationInfo {
+            api_version: Version::new(1, 1, 37),
+            application_name: "wlx-overlay-s",
+            application_version: 0,
+            engine_name: "wlx-overlay-s",
+            engine_version: 0,
+        },
+        &enabled_extensions,
+        &layers,
+    ) else {
+        bail!("Failed to create OpenXR instance.");
+    };
+
+    let Ok(instance_props) = xr_instance.properties() else {
+        bail!("Failed to query OpenXR instance properties.");
+    };
+    log::info!(
+        "Using OpenXR runtime: {} {}",
+        instance_props.runtime_name,
+        instance_props.runtime_version
+    );
+
+    let Ok(system) = xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY) else {
+        bail!("Failed to access OpenXR HMD system.");
+    };
+
+    let vk_target_version_xr = xr::Version::new(1, 1, 0);
+
+    let Ok(reqs) = xr_instance.graphics_requirements::<xr::Vulkan>(system) else {
+        bail!("Failed to query OpenXR Vulkan requirements.");
+    };
+
+    if vk_target_version_xr < reqs.min_api_version_supported
+        || vk_target_version_xr.major() > reqs.max_api_version_supported.major()
+    {
+        bail!(
+            "OpenXR runtime requires Vulkan version > {}, < {}.0.0",
+            reqs.min_api_version_supported,
+            reqs.max_api_version_supported.major() + 1
+        );
+    }
+
+    Ok((xr_instance, system))
+}
+
+pub(super) unsafe fn create_overlay_session(
+    instance: &xr::Instance,
+    system: xr::SystemId,
+    info: &xr::vulkan::SessionCreateInfo,
+) -> Result<xr::sys::Session, xr::sys::Result> {
+    let overlay = xr::sys::SessionCreateInfoOverlayEXTX {
+        ty: xr::sys::SessionCreateInfoOverlayEXTX::TYPE,
+        next: std::ptr::null(),
+        create_flags: OverlaySessionCreateFlagsEXTX::EMPTY,
+        session_layers_placement: 5,
+    };
+    let binding = xr::sys::GraphicsBindingVulkanKHR {
+        ty: xr::sys::GraphicsBindingVulkanKHR::TYPE,
+        next: (&raw const overlay).cast(),
+        instance: info.instance,
+        physical_device: info.physical_device,
+        device: info.device,
+        queue_family_index: info.queue_family_index,
+        queue_index: info.queue_index,
+    };
+    let info = xr::sys::SessionCreateInfo {
+        ty: xr::sys::SessionCreateInfo::TYPE,
+        next: (&raw const binding).cast(),
+        create_flags: SessionCreateFlags::default(),
+        system_id: system,
+    };
+    let mut out = xr::sys::Session::NULL;
+    let x = (instance.fp().create_session)(instance.as_raw(), &info, &mut out);
+    if x.into_raw() >= 0 {
+        Ok(out)
+    } else {
+        Err(x)
+    }
+}
+
+type Vec3M = mint::Vector3<f32>;
+type QuatM = mint::Quaternion<f32>;
+
+pub(super) fn ipd_from_views(views: &[xr::View]) -> f32 {
+    let p0: Vec3 = Vec3M::from(views[0].pose.position).into();
+    let p1: Vec3 = Vec3M::from(views[1].pose.position).into();
+
+    (p0.distance(p1) * 10000.0).round() * 0.1
+}
+
+pub(super) fn transform_to_norm_quat(transform: &Affine3A) -> Quat {
+    let norm_mat3 = transform
+        .matrix3
+        .mul_scalar(1.0 / transform.matrix3.x_axis.length());
+    Quat::from_mat3a(&norm_mat3).normalize()
+}
+
+pub(super) fn translation_rotation_to_posef(translation: Vec3A, mut rotation: Quat) -> xr::Posef {
+    if !rotation.is_finite() {
+        rotation = Quat::IDENTITY;
+    }
+
+    xr::Posef {
+        orientation: xr::Quaternionf {
+            x: rotation.x,
+            y: rotation.y,
+            z: rotation.z,
+            w: rotation.w,
+        },
+        position: xr::Vector3f {
+            x: translation.x,
+            y: translation.y,
+            z: translation.z,
+        },
+    }
+}
+
+pub(super) fn transform_to_posef(transform: &Affine3A) -> xr::Posef {
+    let translation = transform.translation;
+    let rotation = transform_to_norm_quat(transform);
+    translation_rotation_to_posef(translation, rotation)
+}
+
+pub(super) fn posef_to_transform(pose: &xr::Posef) -> Affine3A {
+    let rotation = QuatM::from(pose.orientation).into();
+    let translation = Vec3M::from(pose.position).into();
+    Affine3A::from_rotation_translation(rotation, translation)
+}

wlx-overlay-s/src/backend/openxr/input.rs

@@ -0,0 +1,692 @@
+use std::{
+    array::from_fn,
+    mem::transmute,
+    time::{Duration, Instant},
+};
+
+use glam::{bool, Affine3A, Quat, Vec3};
+use libmonado as mnd;
+use openxr::{self as xr, Quaternionf, Vector2f, Vector3f};
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    backend::input::{Haptics, Pointer, TrackedDevice, TrackedDeviceRole},
+    config_io,
+    state::{AppSession, AppState},
+};
+
+use super::{helpers::posef_to_transform, XrState};
+
+static CLICK_TIMES: [Duration; 3] = [
+    Duration::ZERO,
+    Duration::from_millis(500),
+    Duration::from_millis(750),
+];
+
+pub(super) struct OpenXrInputSource {
+    action_set: xr::ActionSet,
+    hands: [OpenXrHand; 2],
+}
+
+pub(super) struct OpenXrHand {
+    source: OpenXrHandSource,
+    space: xr::Space,
+}
+
+pub struct MultiClickHandler<const COUNT: usize> {
+    name: String,
+    action_f32: xr::Action<f32>,
+    action_bool: xr::Action<bool>,
+    previous: [Instant; COUNT],
+    held_active: bool,
+    held_inactive: bool,
+}
+
+impl<const COUNT: usize> MultiClickHandler<COUNT> {
+    fn new(action_set: &xr::ActionSet, action_name: &str, side: &str) -> anyhow::Result<Self> {
+        let name = format!("{side}_{COUNT}-{action_name}");
+        let name_f32 = format!("{}_value", &name);
+
+        let action_bool = action_set.create_action::<bool>(&name, &name, &[])?;
+        let action_f32 = action_set.create_action::<f32>(&name_f32, &name_f32, &[])?;
+
+        Ok(Self {
+            name,
+            action_f32,
+            action_bool,
+            previous: from_fn(|_| Instant::now()),
+            held_active: false,
+            held_inactive: false,
+        })
+    }
+    fn check<G>(&mut self, session: &xr::Session<G>, threshold: f32) -> anyhow::Result<bool> {
+        let res = self.action_bool.state(session, xr::Path::NULL)?;
+        let mut state = res.is_active && res.current_state;
+
+        if !state {
+            let res = self.action_f32.state(session, xr::Path::NULL)?;
+            state = res.is_active && res.current_state > threshold;
+        }
+
+        if !state {
+            self.held_active = false;
+            self.held_inactive = false;
+            return Ok(false);
+        }
+
+        if self.held_active {
+            return Ok(true);
+        }
+
+        if self.held_inactive {
+            return Ok(false);
+        }
+
+        let passed = self
+            .previous
+            .iter()
+            .all(|instant| instant.elapsed() < CLICK_TIMES[COUNT]);
+
+        if passed {
+            log::trace!("{}: passed", self.name);
+            self.held_active = true;
+            self.held_inactive = false;
+
+            // reset to no prior clicks
+            let long_ago = Instant::now().checked_sub(Duration::from_secs(10)).unwrap();
+            self.previous
+                .iter_mut()
+                .for_each(|instant| *instant = long_ago);
+        } else if COUNT > 0 {
+            log::trace!("{}: rotate", self.name);
+            self.previous.rotate_right(1);
+            self.previous[0] = Instant::now();
+            self.held_inactive = true;
+        }
+
+        Ok(passed)
+    }
+}
+
+pub struct CustomClickAction {
+    single: MultiClickHandler<0>,
+    double: MultiClickHandler<1>,
+    triple: MultiClickHandler<2>,
+}
+
+impl CustomClickAction {
+    pub fn new(action_set: &xr::ActionSet, name: &str, side: &str) -> anyhow::Result<Self> {
+        let single = MultiClickHandler::new(action_set, name, side)?;
+        let double = MultiClickHandler::new(action_set, name, side)?;
+        let triple = MultiClickHandler::new(action_set, name, side)?;
+
+        Ok(Self {
+            single,
+            double,
+            triple,
+        })
+    }
+    pub fn state(
+        &mut self,
+        before: bool,
+        state: &XrState,
+        session: &AppSession,
+    ) -> anyhow::Result<bool> {
+        let threshold = if before {
+            session.config.xr_click_sensitivity_release
+        } else {
+            session.config.xr_click_sensitivity
+        };
+
+        Ok(self.single.check(&state.session, threshold)?
+            || self.double.check(&state.session, threshold)?
+            || self.triple.check(&state.session, threshold)?)
+    }
+}
+
+pub(super) struct OpenXrHandSource {
+    pose: xr::Action<xr::Posef>,
+    click: CustomClickAction,
+    grab: CustomClickAction,
+    alt_click: CustomClickAction,
+    show_hide: CustomClickAction,
+    toggle_dashboard: CustomClickAction,
+    space_drag: CustomClickAction,
+    space_rotate: CustomClickAction,
+    space_reset: CustomClickAction,
+    modifier_right: CustomClickAction,
+    modifier_middle: CustomClickAction,
+    move_mouse: CustomClickAction,
+    scroll: xr::Action<Vector2f>,
+    haptics: xr::Action<xr::Haptic>,
+}
+
+impl OpenXrInputSource {
+    pub fn new(xr: &XrState) -> anyhow::Result<Self> {
+        let mut action_set =
+            xr.session
+                .instance()
+                .create_action_set("wlx-overlay-s", "WlxOverlay-S Actions", 0)?;
+
+        let left_source = OpenXrHandSource::new(&mut action_set, "left")?;
+        let right_source = OpenXrHandSource::new(&mut action_set, "right")?;
+
+        suggest_bindings(&xr.instance, &[&left_source, &right_source]);
+
+        xr.session.attach_action_sets(&[&action_set])?;
+
+        Ok(Self {
+            action_set,
+            hands: [
+                OpenXrHand::new(xr, left_source)?,
+                OpenXrHand::new(xr, right_source)?,
+            ],
+        })
+    }
+
+    pub fn haptics(&self, xr: &XrState, hand: usize, haptics: &Haptics) {
+        let action = &self.hands[hand].source.haptics;
+
+        let duration_nanos = f64::from(haptics.duration) * 1_000_000_000.0;
+
+        let _ = action.apply_feedback(
+            &xr.session,
+            xr::Path::NULL,
+            &xr::HapticVibration::new()
+                .amplitude(haptics.intensity)
+                .frequency(haptics.frequency)
+                .duration(xr::Duration::from_nanos(duration_nanos as _)),
+        );
+    }
+
+    pub fn update(&mut self, xr: &XrState, state: &mut AppState) -> anyhow::Result<()> {
+        xr.session.sync_actions(&[(&self.action_set).into()])?;
+
+        let loc = xr.view.locate(&xr.stage, xr.predicted_display_time)?;
+        let hmd = posef_to_transform(&loc.pose);
+        if loc
+            .location_flags
+            .contains(xr::SpaceLocationFlags::ORIENTATION_VALID)
+        {
+            state.input_state.hmd.matrix3 = hmd.matrix3;
+        }
+
+        if loc
+            .location_flags
+            .contains(xr::SpaceLocationFlags::POSITION_VALID)
+        {
+            state.input_state.hmd.translation = hmd.translation;
+        }
+
+        for i in 0..2 {
+            self.hands[i].update(&mut state.input_state.pointers[i], xr, &state.session)?;
+        }
+        Ok(())
+    }
+
+    fn update_device_battery_status(
+        device: &mut mnd::Device,
+        role: TrackedDeviceRole,
+        app: &mut AppState,
+    ) {
+        if let Ok(status) = device.battery_status() {
+            if status.present {
+                app.input_state.devices.push(TrackedDevice {
+                    soc: Some(status.charge),
+                    charging: status.charging,
+                    role,
+                });
+                log::debug!(
+                    "Device {} role {:#?}: {:.0}% (charging {})",
+                    device.index,
+                    role,
+                    status.charge * 100.0f32,
+                    status.charging
+                );
+            }
+        }
+    }
+
+    pub fn update_devices(app: &mut AppState, monado: &mut mnd::Monado) {
+        app.input_state.devices.clear();
+
+        let roles = [
+            (mnd::DeviceRole::Head, TrackedDeviceRole::Hmd),
+            (mnd::DeviceRole::Eyes, TrackedDeviceRole::None),
+            (mnd::DeviceRole::Left, TrackedDeviceRole::LeftHand),
+            (mnd::DeviceRole::Right, TrackedDeviceRole::RightHand),
+            (mnd::DeviceRole::Gamepad, TrackedDeviceRole::None),
+            (
+                mnd::DeviceRole::HandTrackingLeft,
+                TrackedDeviceRole::LeftHand,
+            ),
+            (
+                mnd::DeviceRole::HandTrackingRight,
+                TrackedDeviceRole::RightHand,
+            ),
+        ];
+        let mut seen = Vec::<u32>::with_capacity(32);
+        for (mnd_role, wlx_role) in roles {
+            let device = monado.device_from_role(mnd_role);
+            if let Ok(mut device) = device {
+                if !seen.contains(&device.index) {
+                    seen.push(device.index);
+                    Self::update_device_battery_status(&mut device, wlx_role, app);
+                }
+            }
+        }
+        if let Ok(devices) = monado.devices() {
+            for mut device in devices {
+                if !seen.contains(&device.index) {
+                    let role = if device.name_id >= 4 && device.name_id <= 8 {
+                        TrackedDeviceRole::Tracker
+                    } else {
+                        TrackedDeviceRole::None
+                    };
+                    Self::update_device_battery_status(&mut device, role, app);
+                }
+            }
+        }
+
+        app.input_state.devices.sort_by(|a, b| {
+            u8::from(a.soc.is_none())
+                .cmp(&u8::from(b.soc.is_none()))
+                .then((a.role as u8).cmp(&(b.role as u8)))
+                .then(a.soc.unwrap_or(999.).total_cmp(&b.soc.unwrap_or(999.)))
+        });
+    }
+}
+
+impl OpenXrHand {
+    pub(super) fn new(xr: &XrState, source: OpenXrHandSource) -> Result<Self, xr::sys::Result> {
+        let space = source
+            .pose
+            .create_space(&xr.session, xr::Path::NULL, xr::Posef::IDENTITY)?;
+
+        Ok(Self { source, space })
+    }
+
+    pub(super) fn update(
+        &mut self,
+        pointer: &mut Pointer,
+        xr: &XrState,
+        session: &AppSession,
+    ) -> anyhow::Result<()> {
+        let location = self.space.locate(&xr.stage, xr.predicted_display_time)?;
+        if location
+            .location_flags
+            .contains(xr::SpaceLocationFlags::ORIENTATION_VALID)
+        {
+            let (cur_quat, cur_pos) = (Quat::from_affine3(&pointer.pose), pointer.pose.translation);
+
+            let (new_quat, new_pos) = unsafe {
+                (
+                    transmute::<Quaternionf, Quat>(location.pose.orientation),
+                    transmute::<Vector3f, Vec3>(location.pose.position),
+                )
+            };
+            let lerp_factor =
+                (1.0 / (xr.fps / 100.0) * session.config.pointer_lerp_factor).clamp(0.1, 1.0);
+            pointer.raw_pose = Affine3A::from_rotation_translation(new_quat, new_pos);
+            pointer.pose = Affine3A::from_rotation_translation(
+                cur_quat.lerp(new_quat, lerp_factor),
+                cur_pos.lerp(new_pos.into(), lerp_factor).into(),
+            );
+        }
+
+        pointer.now.click = self.source.click.state(pointer.before.click, xr, session)?;
+
+        pointer.now.grab = self.source.grab.state(pointer.before.grab, xr, session)?;
+
+        let scroll = self
+            .source
+            .scroll
+            .state(&xr.session, xr::Path::NULL)?
+            .current_state;
+
+        pointer.now.scroll_x = scroll.x;
+        pointer.now.scroll_y = scroll.y;
+
+        pointer.now.alt_click =
+            self.source
+                .alt_click
+                .state(pointer.before.alt_click, xr, session)?;
+
+        pointer.now.show_hide =
+            self.source
+                .show_hide
+                .state(pointer.before.show_hide, xr, session)?;
+
+        pointer.now.click_modifier_right =
+            self.source
+                .modifier_right
+                .state(pointer.before.click_modifier_right, xr, session)?;
+
+        pointer.now.toggle_dashboard =
+            self.source
+                .toggle_dashboard
+                .state(pointer.before.toggle_dashboard, xr, session)?;
+
+        pointer.now.click_modifier_middle =
+            self.source
+                .modifier_middle
+                .state(pointer.before.click_modifier_middle, xr, session)?;
+
+        pointer.now.move_mouse =
+            self.source
+                .move_mouse
+                .state(pointer.before.move_mouse, xr, session)?;
+
+        pointer.now.space_drag =
+            self.source
+                .space_drag
+                .state(pointer.before.space_drag, xr, session)?;
+
+        pointer.now.space_rotate =
+            self.source
+                .space_rotate
+                .state(pointer.before.space_rotate, xr, session)?;
+
+        pointer.now.space_reset =
+            self.source
+                .space_reset
+                .state(pointer.before.space_reset, xr, session)?;
+
+        Ok(())
+    }
+}
+
+// supported action types: Haptic, Posef, Vector2f, f32, bool
+impl OpenXrHandSource {
+    pub(super) fn new(action_set: &mut xr::ActionSet, side: &str) -> anyhow::Result<Self> {
+        let action_pose = action_set.create_action::<xr::Posef>(
+            &format!("{side}_hand"),
+            &format!("{side} hand pose"),
+            &[],
+        )?;
+
+        let action_scroll = action_set.create_action::<Vector2f>(
+            &format!("{side}_scroll"),
+            &format!("{side} hand scroll"),
+            &[],
+        )?;
+        let action_haptics = action_set.create_action::<xr::Haptic>(
+            &format!("{side}_haptics"),
+            &format!("{side} hand haptics"),
+            &[],
+        )?;
+
+        Ok(Self {
+            pose: action_pose,
+            click: CustomClickAction::new(action_set, "click", side)?,
+            grab: CustomClickAction::new(action_set, "grab", side)?,
+            scroll: action_scroll,
+            alt_click: CustomClickAction::new(action_set, "alt_click", side)?,
+            show_hide: CustomClickAction::new(action_set, "show_hide", side)?,
+            toggle_dashboard: CustomClickAction::new(action_set, "toggle_dashboard", side)?,
+            space_drag: CustomClickAction::new(action_set, "space_drag", side)?,
+            space_rotate: CustomClickAction::new(action_set, "space_rotate", side)?,
+            space_reset: CustomClickAction::new(action_set, "space_reset", side)?,
+            modifier_right: CustomClickAction::new(action_set, "click_modifier_right", side)?,
+            modifier_middle: CustomClickAction::new(action_set, "click_modifier_middle", side)?,
+            move_mouse: CustomClickAction::new(action_set, "move_mouse", side)?,
+            haptics: action_haptics,
+        })
+    }
+}
+
+fn to_path(maybe_path_str: Option<&String>, instance: &xr::Instance) -> Option<xr::Path> {
+    maybe_path_str.as_ref().and_then(|s| {
+        instance
+            .string_to_path(s)
+            .inspect_err(|_| {
+                log::warn!("Invalid binding path: {s}");
+            })
+            .ok()
+    })
+}
+
+fn is_bool(maybe_type_str: Option<&String>) -> bool {
+    maybe_type_str
+        .as_ref()
+        .unwrap() // want panic
+        .split('/')
+        .next_back()
+        .is_some_and(|last| matches!(last, "click" | "touch") || last.starts_with("dpad_"))
+}
+
+macro_rules! add_custom {
+    ($action:expr, $left:expr, $right:expr, $bindings:expr, $instance:expr) => {
+        if let Some(action) = $action.as_ref() {
+            if let Some(p) = to_path(action.left.as_ref(), $instance) {
+                if is_bool(action.left.as_ref()) {
+                    if action.triple_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$left.triple.action_bool, p));
+                    } else if action.double_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$left.double.action_bool, p));
+                    } else {
+                        $bindings.push(xr::Binding::new(&$left.single.action_bool, p));
+                    }
+                } else {
+                    if action.triple_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$left.triple.action_f32, p));
+                    } else if action.double_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$left.double.action_f32, p));
+                    } else {
+                        $bindings.push(xr::Binding::new(&$left.single.action_f32, p));
+                    }
+                }
+            }
+            if let Some(p) = to_path(action.right.as_ref(), $instance) {
+                if is_bool(action.right.as_ref()) {
+                    if action.triple_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$right.triple.action_bool, p));
+                    } else if action.double_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$right.double.action_bool, p));
+                    } else {
+                        $bindings.push(xr::Binding::new(&$right.single.action_bool, p));
+                    }
+                } else {
+                    if action.triple_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$right.triple.action_f32, p));
+                    } else if action.double_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$right.double.action_f32, p));
+                    } else {
+                        $bindings.push(xr::Binding::new(&$right.single.action_f32, p));
+                    }
+                }
+            }
+        }
+    };
+}
+
+#[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
+fn suggest_bindings(instance: &xr::Instance, hands: &[&OpenXrHandSource; 2]) {
+    let profiles = load_action_profiles();
+
+    for profile in profiles {
+        let Ok(profile_path) = instance.string_to_path(&profile.profile) else {
+            log::debug!("Profile not supported: {}", profile.profile);
+            continue;
+        };
+
+        let mut bindings: Vec<xr::Binding> = vec![];
+
+        if let Some(action) = profile.pose {
+            if let Some(p) = to_path(action.left.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[0].pose, p));
+            }
+            if let Some(p) = to_path(action.right.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[1].pose, p));
+            }
+        }
+
+        if let Some(action) = profile.haptic {
+            if let Some(p) = to_path(action.left.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[0].haptics, p));
+            }
+            if let Some(p) = to_path(action.right.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[1].haptics, p));
+            }
+        }
+
+        if let Some(action) = profile.scroll {
+            if let Some(p) = to_path(action.left.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[0].scroll, p));
+            }
+            if let Some(p) = to_path(action.right.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[1].scroll, p));
+            }
+        }
+
+        add_custom!(
+            profile.click,
+            hands[0].click,
+            hands[1].click,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.alt_click,
+            &hands[0].alt_click,
+            &hands[1].alt_click,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.grab,
+            &hands[0].grab,
+            &hands[1].grab,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.show_hide,
+            &hands[0].show_hide,
+            &hands[1].show_hide,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.toggle_dashboard,
+            &hands[0].toggle_dashboard,
+            &hands[1].toggle_dashboard,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.space_drag,
+            &hands[0].space_drag,
+            &hands[1].space_drag,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.space_rotate,
+            &hands[0].space_rotate,
+            &hands[1].space_rotate,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.space_reset,
+            &hands[0].space_reset,
+            &hands[1].space_reset,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.click_modifier_right,
+            &hands[0].modifier_right,
+            &hands[1].modifier_right,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.click_modifier_middle,
+            &hands[0].modifier_middle,
+            &hands[1].modifier_middle,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.move_mouse,
+            &hands[0].move_mouse,
+            &hands[1].move_mouse,
+            bindings,
+            instance
+        );
+
+        if instance
+            .suggest_interaction_profile_bindings(profile_path, &bindings)
+            .is_err()
+        {
+            log::error!("Bad bindings for {}", &profile.profile[22..]);
+            log::error!("Verify config: ~/.config/wlxoverlay/openxr_actions.json5");
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct OpenXrActionConfAction {
+    left: Option<String>,
+    right: Option<String>,
+    threshold: Option<[f32; 2]>,
+    double_click: Option<bool>,
+    triple_click: Option<bool>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct OpenXrActionConfProfile {
+    profile: String,
+    pose: Option<OpenXrActionConfAction>,
+    click: Option<OpenXrActionConfAction>,
+    grab: Option<OpenXrActionConfAction>,
+    alt_click: Option<OpenXrActionConfAction>,
+    show_hide: Option<OpenXrActionConfAction>,
+    toggle_dashboard: Option<OpenXrActionConfAction>,
+    space_drag: Option<OpenXrActionConfAction>,
+    space_rotate: Option<OpenXrActionConfAction>,
+    space_reset: Option<OpenXrActionConfAction>,
+    click_modifier_right: Option<OpenXrActionConfAction>,
+    click_modifier_middle: Option<OpenXrActionConfAction>,
+    move_mouse: Option<OpenXrActionConfAction>,
+    scroll: Option<OpenXrActionConfAction>,
+    haptic: Option<OpenXrActionConfAction>,
+}
+
+const DEFAULT_PROFILES: &str = include_str!("openxr_actions.json5");
+
+fn load_action_profiles() -> Vec<OpenXrActionConfProfile> {
+    let mut profiles: Vec<OpenXrActionConfProfile> =
+        serde_json5::from_str(DEFAULT_PROFILES).unwrap(); // want panic
+
+    let Some(conf) = config_io::load("openxr_actions.json5") else {
+        return profiles;
+    };
+
+    match serde_json5::from_str::<Vec<OpenXrActionConfProfile>>(&conf) {
+        Ok(override_profiles) => {
+            for new in override_profiles {
+                if let Some(i) = profiles.iter().position(|old| old.profile == new.profile) {
+                    profiles[i] = new;
+                }
+            }
+        }
+        Err(e) => {
+            log::error!("Failed to load openxr_actions.json5: {e}");
+        }
+    }
+
+    profiles
+}

wlx-overlay-s/src/backend/openxr/lines.rs

@@ -0,0 +1,201 @@
+use glam::{Affine3A, Vec3, Vec3A};
+use idmap::IdMap;
+use openxr as xr;
+use std::{
+    f32::consts::PI,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+};
+
+use wgui::gfx::{pipeline::WGfxPipeline, WGfx};
+
+use crate::{
+    backend::openxr::helpers,
+    graphics::{CommandBuffers, ExtentExt, Vert2Uv},
+    state::AppState,
+};
+use vulkano::{
+    command_buffer::CommandBufferUsage, pipeline::graphics::input_assembly::PrimitiveTopology,
+};
+
+use super::{
+    swapchain::{create_swapchain, SwapchainOpts, WlxSwapchain},
+    CompositionLayer, XrState,
+};
+
+static LINE_AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(1);
+pub(super) const LINE_WIDTH: f32 = 0.002;
+
+// TODO customizable colors
+static COLORS: [[f32; 6]; 5] = {
+    [
+        [1., 1., 1., 1., 0., 0.],
+        [0., 0.375, 0.5, 1., 0., 0.],
+        [0.69, 0.188, 0., 1., 0., 0.],
+        [0.375, 0., 0.5, 1., 0., 0.],
+        [1., 0., 0., 1., 0., 0.],
+    ]
+};
+
+pub(super) struct LinePool {
+    lines: IdMap<usize, LineContainer>,
+    pipeline: Arc<WGfxPipeline<Vert2Uv>>,
+}
+
+impl LinePool {
+    pub(super) fn new(app: &AppState) -> anyhow::Result<Self> {
+        let pipeline = app.gfx.create_pipeline(
+            app.gfx_extras.shaders.get("vert_quad").unwrap().clone(), // want panic
+            app.gfx_extras.shaders.get("frag_color").unwrap().clone(), // want panic
+            app.gfx.surface_format,
+            None,
+            PrimitiveTopology::TriangleStrip,
+            false,
+        )?;
+
+        Ok(Self {
+            lines: IdMap::new(),
+            pipeline,
+        })
+    }
+
+    pub(super) fn allocate(&mut self, xr: &XrState, gfx: Arc<WGfx>) -> anyhow::Result<usize> {
+        let id = LINE_AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed);
+
+        let srd = create_swapchain(xr, gfx, [1, 1, 1], SwapchainOpts::new())?;
+        self.lines.insert(
+            id,
+            LineContainer {
+                swapchain: srd,
+                maybe_line: None,
+            },
+        );
+        Ok(id)
+    }
+
+    pub(super) fn draw_from(
+        &mut self,
+        id: usize,
+        mut from: Affine3A,
+        len: f32,
+        color: usize,
+        hmd: &Affine3A,
+    ) {
+        if len < 0.01 {
+            return;
+        }
+
+        debug_assert!(color < COLORS.len());
+
+        let Some(line) = self.lines.get_mut(id) else {
+            log::warn!("Line {id} not found");
+            return;
+        };
+
+        let rotation = Affine3A::from_axis_angle(Vec3::X, PI * 1.5);
+
+        from.translation += from.transform_vector3a(Vec3A::NEG_Z) * (len * 0.5);
+        let mut transform = from * rotation;
+
+        let to_hmd = hmd.translation - from.translation;
+        let sides = [Vec3A::Z, Vec3A::X, Vec3A::NEG_Z, Vec3A::NEG_X];
+        let rotations = [
+            Affine3A::IDENTITY,
+            Affine3A::from_axis_angle(Vec3::Y, PI * 0.5),
+            Affine3A::from_axis_angle(Vec3::Y, PI * -1.0),
+            Affine3A::from_axis_angle(Vec3::Y, PI * 1.5),
+        ];
+        let mut closest = (0, 0.0);
+        for (i, &side) in sides.iter().enumerate() {
+            let dot = to_hmd.dot(transform.transform_vector3a(side));
+            if i == 0 || dot > closest.1 {
+                closest = (i, dot);
+            }
+        }
+
+        transform *= rotations[closest.0];
+
+        let posef = helpers::transform_to_posef(&transform);
+
+        line.maybe_line = Some(Line {
+            color,
+            pose: posef,
+            length: len,
+        });
+    }
+
+    pub(super) fn render(
+        &mut self,
+        app: &AppState,
+        buf: &mut CommandBuffers,
+    ) -> anyhow::Result<()> {
+        for line in self.lines.values_mut() {
+            if let Some(inner) = line.maybe_line.as_mut() {
+                let tgt = line.swapchain.acquire_wait_image()?;
+
+                let set0 = self
+                    .pipeline
+                    .uniform_buffer_upload(0, COLORS[inner.color].to_vec())?;
+
+                let pass = self.pipeline.create_pass(
+                    tgt.extent_f32(),
+                    app.gfx_extras.quad_verts.clone(),
+                    0..4,
+                    0..1,
+                    vec![set0],
+                )?;
+
+                let mut cmd_buffer = app
+                    .gfx
+                    .create_gfx_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+                cmd_buffer.begin_rendering(tgt)?;
+                cmd_buffer.run_ref(&pass)?;
+                cmd_buffer.end_rendering()?;
+
+                buf.push(cmd_buffer.build()?);
+            }
+        }
+
+        Ok(())
+    }
+
+    pub(super) fn present<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+    ) -> anyhow::Result<Vec<CompositionLayer<'a>>> {
+        let mut quads = Vec::new();
+
+        for line in self.lines.values_mut() {
+            line.swapchain.ensure_image_released()?;
+
+            if let Some(inner) = line.maybe_line.take() {
+                let quad = xr::CompositionLayerQuad::new()
+                    .pose(inner.pose)
+                    .sub_image(line.swapchain.get_subimage())
+                    .eye_visibility(xr::EyeVisibility::BOTH)
+                    .space(&xr.stage)
+                    .size(xr::Extent2Df {
+                        width: LINE_WIDTH,
+                        height: inner.length,
+                    });
+
+                quads.push(CompositionLayer::Quad(quad));
+            }
+        }
+
+        Ok(quads)
+    }
+}
+
+pub(super) struct Line {
+    pub(super) color: usize,
+    pub(super) pose: xr::Posef,
+    pub(super) length: f32,
+}
+
+struct LineContainer {
+    swapchain: WlxSwapchain,
+    maybe_line: Option<Line>,
+}

wlx-overlay-s/src/backend/openxr/mod.rs

@@ -0,0 +1,577 @@
+use std::{
+    collections::VecDeque,
+    ops::Add,
+    sync::{
+        atomic::{AtomicBool, AtomicUsize, Ordering},
+        Arc,
+    },
+    time::{Duration, Instant},
+};
+
+use glam::{Affine3A, Vec3};
+use input::OpenXrInputSource;
+use libmonado::Monado;
+use openxr as xr;
+use skybox::create_skybox;
+use vulkano::{Handle, VulkanObject};
+
+use crate::{
+    backend::{
+        common::{BackendError, OverlayContainer},
+        input::interact,
+        notifications::NotificationManager,
+        openxr::{lines::LinePool, overlay::OpenXrOverlayData},
+        overlay::{OverlayData, ShouldRender},
+        task::{SystemTask, TaskType},
+    },
+    graphics::{init_openxr_graphics, CommandBuffers},
+    overlays::{
+        toast::{Toast, ToastTopic},
+        watch::{watch_fade, WATCH_NAME},
+    },
+    state::AppState,
+};
+
+#[cfg(feature = "wayvr")]
+use crate::{backend::wayvr::WayVRAction, overlays::wayvr::wayvr_action};
+
+mod blocker;
+mod helpers;
+mod input;
+mod lines;
+mod overlay;
+mod playspace;
+mod skybox;
+mod swapchain;
+
+const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
+static FRAME_COUNTER: AtomicUsize = AtomicUsize::new(0);
+
+struct XrState {
+    instance: xr::Instance,
+    session: xr::Session<xr::Vulkan>,
+    predicted_display_time: xr::Time,
+    fps: f32,
+    stage: Arc<xr::Space>,
+    view: Arc<xr::Space>,
+}
+
+#[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
+pub fn openxr_run(
+    running: Arc<AtomicBool>,
+    show_by_default: bool,
+    headless: bool,
+) -> Result<(), BackendError> {
+    let (xr_instance, system) = match helpers::init_xr() {
+        Ok((xr_instance, system)) => (xr_instance, system),
+        Err(e) => {
+            log::warn!("Will not use OpenXR: {e}");
+            return Err(BackendError::NotSupported);
+        }
+    };
+
+    let mut app = {
+        let (gfx, gfx_extras) = init_openxr_graphics(xr_instance.clone(), system)?;
+        AppState::from_graphics(gfx, gfx_extras)?
+    };
+
+    let environment_blend_mode = {
+        let modes = xr_instance.enumerate_environment_blend_modes(system, VIEW_TYPE)?;
+        if modes.contains(&xr::EnvironmentBlendMode::ALPHA_BLEND)
+            && app.session.config.use_passthrough
+        {
+            xr::EnvironmentBlendMode::ALPHA_BLEND
+        } else {
+            modes[0]
+        }
+    };
+    log::info!("Using environment blend mode: {environment_blend_mode:?}");
+
+    if show_by_default {
+        app.tasks.enqueue_at(
+            TaskType::System(SystemTask::ShowHide),
+            Instant::now().add(Duration::from_secs(1)),
+        );
+    }
+
+    let mut overlays = OverlayContainer::<OpenXrOverlayData>::new(&mut app, headless)?;
+    let mut lines = LinePool::new(&app)?;
+
+    let mut notifications = NotificationManager::new();
+    notifications.run_dbus();
+    notifications.run_udp();
+
+    let mut delete_queue = vec![];
+
+    let mut monado = Monado::auto_connect()
+        .map_err(|e| log::warn!("Will not use libmonado: {e}"))
+        .ok();
+
+    let mut playspace = monado.as_mut().and_then(|m| {
+        playspace::PlayspaceMover::new(m)
+            .map_err(|e| log::warn!("Will not use Monado playspace mover: {e}"))
+            .ok()
+    });
+
+    let mut blocker = monado.is_some().then(blocker::InputBlocker::new);
+
+    let (session, mut frame_wait, mut frame_stream) = unsafe {
+        let raw_session = helpers::create_overlay_session(
+            &xr_instance,
+            system,
+            &xr::vulkan::SessionCreateInfo {
+                instance: app.gfx.instance.handle().as_raw() as _,
+                physical_device: app.gfx.device.physical_device().handle().as_raw() as _,
+                device: app.gfx.device.handle().as_raw() as _,
+                queue_family_index: app.gfx.queue_gfx.queue_family_index(),
+                queue_index: 0,
+            },
+        )?;
+        xr::Session::from_raw(xr_instance.clone(), raw_session, Box::new(()))
+    };
+
+    let stage =
+        session.create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)?;
+
+    let view = session.create_reference_space(xr::ReferenceSpaceType::VIEW, xr::Posef::IDENTITY)?;
+
+    let mut xr_state = XrState {
+        instance: xr_instance,
+        session,
+        predicted_display_time: xr::Time::from_nanos(0),
+        fps: 30.0,
+        stage: Arc::new(stage),
+        view: Arc::new(view),
+    };
+
+    let mut skybox = if environment_blend_mode == xr::EnvironmentBlendMode::OPAQUE {
+        create_skybox(&xr_state, &app)
+    } else {
+        None
+    };
+
+    let pointer_lines = [
+        lines.allocate(&xr_state, app.gfx.clone())?,
+        lines.allocate(&xr_state, app.gfx.clone())?,
+    ];
+
+    let watch_id = overlays.get_by_name(WATCH_NAME).unwrap().state.id; // want panic
+
+    let mut input_source = input::OpenXrInputSource::new(&xr_state)?;
+
+    let mut session_running = false;
+    let mut event_storage = xr::EventDataBuffer::new();
+
+    let mut next_device_update = Instant::now();
+    let mut due_tasks = VecDeque::with_capacity(4);
+
+    let mut fps_counter: VecDeque<Instant> = VecDeque::new();
+
+    let mut main_session_visible = false;
+
+    'main_loop: loop {
+        let cur_frame = FRAME_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+        if !running.load(Ordering::Relaxed) {
+            log::warn!("Received shutdown signal.");
+            match xr_state.session.request_exit() {
+                Ok(()) => log::info!("OpenXR session exit requested."),
+                Err(xr::sys::Result::ERROR_SESSION_NOT_RUNNING) => break 'main_loop,
+                Err(e) => {
+                    log::error!("Failed to request OpenXR session exit: {e}");
+                    break 'main_loop;
+                }
+            }
+        }
+
+        while let Some(event) = xr_state.instance.poll_event(&mut event_storage)? {
+            match event {
+                xr::Event::SessionStateChanged(e) => {
+                    // Session state change is where we can begin and end sessions, as well as
+                    // find quit messages!
+                    log::info!("entered state {:?}", e.state());
+                    match e.state() {
+                        xr::SessionState::READY => {
+                            xr_state.session.begin(VIEW_TYPE)?;
+                            session_running = true;
+                        }
+                        xr::SessionState::STOPPING => {
+                            xr_state.session.end()?;
+                            session_running = false;
+                        }
+                        xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => {
+                            break 'main_loop;
+                        }
+                        _ => {}
+                    }
+                }
+                xr::Event::InstanceLossPending(_) => {
+                    break 'main_loop;
+                }
+                xr::Event::EventsLost(e) => {
+                    log::warn!("lost {} events", e.lost_event_count());
+                }
+                xr::Event::MainSessionVisibilityChangedEXTX(e) => {
+                    if main_session_visible != e.visible() {
+                        main_session_visible = e.visible();
+                        log::info!("Main session visible: {main_session_visible}");
+                        if main_session_visible {
+                            log::debug!("Destroying skybox.");
+                            skybox = None;
+                        } else if environment_blend_mode == xr::EnvironmentBlendMode::OPAQUE {
+                            log::debug!("Allocating skybox.");
+                            skybox = create_skybox(&xr_state, &app);
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        if next_device_update <= Instant::now() {
+            if let Some(monado) = &mut monado {
+                OpenXrInputSource::update_devices(&mut app, monado);
+                next_device_update = Instant::now() + Duration::from_secs(30);
+            }
+        }
+
+        if !session_running {
+            std::thread::sleep(Duration::from_millis(100));
+            continue 'main_loop;
+        }
+
+        let xr_frame_state = frame_wait.wait()?;
+        frame_stream.begin()?;
+
+        xr_state.predicted_display_time = xr_frame_state.predicted_display_time;
+        xr_state.fps = {
+            fps_counter.push_back(Instant::now());
+
+            while let Some(time) = fps_counter.front() {
+                if time.elapsed().as_secs_f32() > 1. {
+                    fps_counter.pop_front();
+                } else {
+                    break;
+                }
+            }
+
+            let total_elapsed = fps_counter
+                .front()
+                .map_or(0f32, |time| time.elapsed().as_secs_f32());
+
+            fps_counter.len() as f32 / total_elapsed
+        };
+
+        if !xr_frame_state.should_render {
+            frame_stream.end(
+                xr_frame_state.predicted_display_time,
+                environment_blend_mode,
+                &[],
+            )?;
+            continue 'main_loop;
+        }
+
+        app.input_state.pre_update();
+        input_source.update(&xr_state, &mut app)?;
+        app.input_state.post_update(&app.session);
+
+        if let Some(ref mut blocker) = blocker {
+            blocker.update(
+                &app,
+                watch_id,
+                monado.as_mut().unwrap(), // safe
+            );
+        }
+
+        if app
+            .input_state
+            .pointers
+            .iter()
+            .any(|p| p.now.show_hide && !p.before.show_hide)
+        {
+            overlays.show_hide(&mut app);
+        }
+
+        #[cfg(feature = "wayvr")]
+        if app
+            .input_state
+            .pointers
+            .iter()
+            .any(|p| p.now.toggle_dashboard && !p.before.toggle_dashboard)
+        {
+            wayvr_action(&mut app, &mut overlays, &WayVRAction::ToggleDashboard);
+        }
+
+        watch_fade(&mut app, overlays.mut_by_id(watch_id).unwrap()); // want panic
+        if let Some(ref mut space_mover) = playspace {
+            space_mover.update(
+                &mut overlays,
+                &app,
+                monado.as_mut().unwrap(), // safe
+            );
+        }
+
+        for o in overlays.iter_mut() {
+            o.after_input(&mut app)?;
+        }
+
+        #[cfg(feature = "osc")]
+        if let Some(ref mut sender) = app.osc_sender {
+            let _ = sender.send_params(&overlays, &app.input_state.devices);
+        }
+
+        let (_, views) = xr_state.session.locate_views(
+            VIEW_TYPE,
+            xr_frame_state.predicted_display_time,
+            &xr_state.stage,
+        )?;
+
+        let ipd = helpers::ipd_from_views(&views);
+        if (app.input_state.ipd - ipd).abs() > 0.01 {
+            log::info!("IPD changed: {} -> {}", app.input_state.ipd, ipd);
+            app.input_state.ipd = ipd;
+            Toast::new(
+                ToastTopic::IpdChange,
+                "IPD".into(),
+                format!("{ipd:.1} mm").into(),
+            )
+            .submit(&mut app);
+        }
+
+        overlays
+            .iter_mut()
+            .for_each(|o| o.state.auto_movement(&mut app));
+
+        let lengths_haptics = interact(&mut overlays, &mut app);
+        for (idx, (len, haptics)) in lengths_haptics.iter().enumerate() {
+            lines.draw_from(
+                pointer_lines[idx],
+                app.input_state.pointers[idx].pose,
+                *len,
+                app.input_state.pointers[idx].interaction.mode as usize + 1,
+                &app.input_state.hmd,
+            );
+            if let Some(haptics) = haptics {
+                input_source.haptics(&xr_state, idx, haptics);
+            }
+        }
+
+        app.hid_provider.commit();
+
+        let watch = overlays.mut_by_id(watch_id).unwrap(); // want panic
+        let watch_transform = watch.state.transform;
+        if !watch.state.want_visible {
+            watch.state.want_visible = true;
+            watch.state.transform = Affine3A::from_scale(Vec3 {
+                x: 0.001,
+                y: 0.001,
+                z: 0.001,
+            });
+        }
+
+        #[cfg(feature = "wayvr")]
+        if let Err(e) =
+            crate::overlays::wayvr::tick_events::<OpenXrOverlayData>(&mut app, &mut overlays)
+        {
+            log::error!("WayVR tick_events failed: {e:?}");
+        }
+
+        // Begin rendering
+        let mut buffers = CommandBuffers::default();
+
+        if !main_session_visible {
+            if let Some(skybox) = skybox.as_mut() {
+                skybox.render(&xr_state, &app, &mut buffers)?;
+            }
+        }
+
+        for o in overlays.iter_mut() {
+            o.data.cur_visible = false;
+            if !o.state.want_visible {
+                continue;
+            }
+
+            if !o.data.init {
+                o.init(&mut app)?;
+                o.data.init = true;
+            }
+
+            let should_render = match o.should_render(&mut app)? {
+                ShouldRender::Should => true,
+                ShouldRender::Can => (o.data.last_alpha - o.state.alpha).abs() > f32::EPSILON,
+                ShouldRender::Unable => false, //try show old image if exists
+            };
+
+            if should_render {
+                if !o.ensure_swapchain(&app, &xr_state)? {
+                    continue;
+                }
+                let tgt = o.data.swapchain.as_mut().unwrap().acquire_wait_image()?; // want
+                if !o.render(&mut app, tgt, &mut buffers, o.state.alpha)? {
+                    o.data.swapchain.as_mut().unwrap().ensure_image_released()?; // want
+                    continue;
+                }
+                o.data.last_alpha = o.state.alpha;
+            } else if o.data.swapchain.is_none() {
+                continue;
+            }
+            o.data.cur_visible = true;
+        }
+
+        lines.render(&app, &mut buffers)?;
+
+        let future = buffers.execute_now(app.gfx.queue_gfx.clone())?;
+        if let Some(mut future) = future {
+            if let Err(e) = future.flush() {
+                return Err(BackendError::Fatal(e.into()));
+            }
+            future.cleanup_finished();
+        }
+        // End rendering
+
+        // Layer composition
+        let mut layers = vec![];
+        if !main_session_visible {
+            if let Some(skybox) = skybox.as_mut() {
+                for (idx, layer) in skybox.present(&xr_state, &app)?.into_iter().enumerate() {
+                    layers.push(((idx as f32).mul_add(-50.0, 200.0), layer));
+                }
+            }
+        }
+
+        for o in overlays.iter_mut() {
+            if !o.data.cur_visible {
+                continue;
+            }
+            let dist_sq = (app.input_state.hmd.translation - o.state.transform.translation)
+                .length_squared()
+                + (100f32 - o.state.z_order as f32);
+            if !dist_sq.is_normal() {
+                o.data.swapchain.as_mut().unwrap().ensure_image_released()?;
+                continue;
+            }
+            let maybe_layer = o.present(&xr_state)?;
+            if matches!(maybe_layer, CompositionLayer::None) {
+                continue;
+            }
+            layers.push((dist_sq, maybe_layer));
+        }
+
+        for maybe_layer in lines.present(&xr_state)? {
+            if matches!(maybe_layer, CompositionLayer::None) {
+                continue;
+            }
+            layers.push((0.0, maybe_layer));
+        }
+        // End layer composition
+
+        #[cfg(feature = "wayvr")]
+        if let Some(wayvr) = &app.wayvr {
+            wayvr.borrow_mut().data.tick_finish()?;
+        }
+
+        // Begin layer submit
+        layers.sort_by(|a, b| b.0.total_cmp(&a.0));
+
+        let frame_ref = layers
+            .iter()
+            .map(|f| match f.1 {
+                CompositionLayer::Quad(ref l) => l as &xr::CompositionLayerBase<xr::Vulkan>,
+                CompositionLayer::Cylinder(ref l) => l as &xr::CompositionLayerBase<xr::Vulkan>,
+                CompositionLayer::Equirect2(ref l) => l as &xr::CompositionLayerBase<xr::Vulkan>,
+                CompositionLayer::None => unreachable!(),
+            })
+            .collect::<Vec<_>>();
+
+        frame_stream.end(
+            xr_state.predicted_display_time,
+            environment_blend_mode,
+            &frame_ref,
+        )?;
+        // End layer submit
+
+        let removed_overlays = overlays.update(&mut app)?;
+        for o in removed_overlays {
+            delete_queue.push((o, cur_frame + 5));
+        }
+
+        notifications.submit_pending(&mut app);
+
+        app.tasks.retrieve_due(&mut due_tasks);
+        while let Some(task) = due_tasks.pop_front() {
+            match task {
+                TaskType::Overlay(sel, f) => {
+                    if let Some(o) = overlays.mut_by_selector(&sel) {
+                        f(&mut app, &mut o.state);
+                    } else {
+                        log::warn!("Overlay not found for task: {sel:?}");
+                    }
+                }
+                TaskType::CreateOverlay(sel, f) => {
+                    let None = overlays.mut_by_selector(&sel) else {
+                        continue;
+                    };
+
+                    let Some((mut overlay_state, overlay_backend)) = f(&mut app) else {
+                        continue;
+                    };
+                    overlay_state.birthframe = cur_frame;
+
+                    overlays.add(OverlayData {
+                        state: overlay_state,
+                        backend: overlay_backend,
+                        ..Default::default()
+                    });
+                }
+                TaskType::DropOverlay(sel) => {
+                    if let Some(o) = overlays.mut_by_selector(&sel) {
+                        if o.state.birthframe < cur_frame {
+                            log::debug!("{}: destroy", o.state.name);
+                            if let Some(o) = overlays.remove_by_selector(&sel) {
+                                // set for deletion after all images are done showing
+                                delete_queue.push((o, cur_frame + 5));
+                            }
+                        }
+                    }
+                }
+                TaskType::System(task) => match task {
+                    SystemTask::FixFloor => {
+                        if let Some(ref mut playspace) = playspace {
+                            playspace.fix_floor(
+                                &app.input_state,
+                                monado.as_mut().unwrap(), // safe
+                            );
+                        }
+                    }
+                    SystemTask::ResetPlayspace => {
+                        if let Some(ref mut playspace) = playspace {
+                            playspace.reset_offset(monado.as_mut().unwrap()); // safe
+                        }
+                    }
+                    SystemTask::ShowHide => {
+                        overlays.show_hide(&mut app);
+                    }
+                    _ => {}
+                },
+                #[cfg(feature = "wayvr")]
+                TaskType::WayVR(action) => {
+                    wayvr_action(&mut app, &mut overlays, &action);
+                }
+            }
+        }
+
+        delete_queue.retain(|(_, frame)| *frame > cur_frame);
+
+        let watch = overlays.mut_by_id(watch_id).unwrap(); // want panic
+        watch.state.transform = watch_transform;
+    }
+
+    Ok(())
+}
+
+pub(super) enum CompositionLayer<'a> {
+    None,
+    Quad(xr::CompositionLayerQuad<'a, xr::Vulkan>),
+    Cylinder(xr::CompositionLayerCylinderKHR<'a, xr::Vulkan>),
+    Equirect2(xr::CompositionLayerEquirect2KHR<'a, xr::Vulkan>),
+}

wlx-overlay-s/src/backend/openxr/openxr_actions.json5

@@ -0,0 +1,301 @@
+// Available bindings:
+//
+// -- click --
+// primary click to interact with the watch or overlays. required
+//
+// -- grab --
+// used to manipulate position, size, orientation of overlays in 3D space
+//
+// -- show_hide --
+// used to quickly hide and show your last selection of screens + keyboard
+//
+// -- space_drag --
+// move your stage (playspace drag)
+//
+// -- toggle_dashboard --
+// run or toggle visibility of a previously configured WayVR-compatible dashboard
+//
+// -- space_rotate --
+// rotate your stage (playspace rotate, WIP)
+//
+// -- space_reset --
+// reset your stage (reset the offset from playspace drag)
+//
+// -- click_modifier_right --
+// while this is held, your pointer will turn ORANGE and your mouse clicks will be RIGHT clicks
+//
+// -- click_modifier_middle --
+// while this is held, your pointer will turn PURPLE and your mouse clicks will be MIDDLE clicks
+//
+// -- move_mouse --
+// when using `focus_follows_mouse_mode`, you need to hold this for the mouse to move
+//
+// -- pose, haptic --
+// do not mess with these, unless you know what you're doing
+
+[
+  // Fallback controller, intended for testing
+  {
+    profile: "/interaction_profiles/khr/simple_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    },
+    click: {
+      // left trigger is click
+      left: "/user/hand/left/input/select/click",
+    },
+    grab: {
+      // right trigger is grab
+      right: "/user/hand/right/input/select/click"
+    },
+    show_hide: {
+      left: "/user/hand/left/input/menu/click"
+    }
+  },
+
+  // Oculus Touch Controller. Compatible with Quest 2, Quest 3, Quest Pro
+  {
+    profile: "/interaction_profiles/oculus/touch_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    },
+    click: {
+      left: "/user/hand/left/input/trigger/value",
+      right: "/user/hand/right/input/trigger/value"
+    },
+    grab: {
+      left: "/user/hand/left/input/squeeze/value",
+      right: "/user/hand/right/input/squeeze/value"
+    },
+    scroll: {
+      left: "/user/hand/left/input/thumbstick/y",
+      right: "/user/hand/right/input/thumbstick/y"
+    },
+    scroll_horizontal: {
+      left: "/user/hand/left/input/thumbstick/x",
+      right: "/user/hand/right/input/thumbstick/x"
+    },
+    show_hide: {
+      double_click: true,
+      left: "/user/hand/left/input/y/click",
+    },
+    space_drag: {
+      left: "/user/hand/left/input/menu/click",
+    },
+    space_reset: {
+      double_click: true,
+      left: "/user/hand/left/input/menu/click",
+    },
+    click_modifier_right: {
+      left: "/user/hand/left/input/y/touch",
+      right: "/user/hand/right/input/b/touch"
+    },
+    click_modifier_middle: {
+      left: "/user/hand/left/input/x/touch",
+      right: "/user/hand/right/input/a/touch"
+    },
+    move_mouse: {
+      // used with focus_follows_mouse_mode
+      left: "/user/hand/left/input/trigger/touch",
+      right: "/user/hand/right/input/trigger/touch"
+    }
+  },
+
+  // Index controller
+  {
+    profile: "/interaction_profiles/valve/index_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    },
+    click: {
+      left: "/user/hand/left/input/trigger/value",
+      right: "/user/hand/right/input/trigger/value"
+    },
+    alt_click: {
+      // left trackpad is space_drag
+      right: "/user/hand/right/input/trackpad/force",
+    },
+    grab: {
+      left: "/user/hand/left/input/squeeze/force",
+      right: "/user/hand/right/input/squeeze/force"
+    },
+    scroll: {
+      left: "/user/hand/left/input/thumbstick/y",
+      right: "/user/hand/right/input/thumbstick/y"
+    },
+    scroll_horizontal: {
+      left: "/user/hand/left/input/thumbstick/x",
+      right: "/user/hand/right/input/thumbstick/x"
+    },
+    toggle_dashboard: {
+      double_click: false,
+      right: "/user/hand/right/input/system/click",
+    },
+    show_hide: {
+      double_click: true,
+      left: "/user/hand/left/input/b/click",
+    },
+    space_drag: {
+      left: "/user/hand/left/input/trackpad/force",
+      // right trackpad is alt_click
+    },
+    space_reset: {
+      left: "/user/hand/left/input/trackpad/force",
+      double_click: true,
+    },
+    click_modifier_right: {
+      left: "/user/hand/left/input/b/touch",
+      right: "/user/hand/right/input/b/touch"
+    },
+    click_modifier_middle: {
+      left: "/user/hand/left/input/a/touch",
+      right: "/user/hand/right/input/a/touch"
+    },
+    move_mouse: {
+      // used with focus_follows_mouse_mode
+      left: "/user/hand/left/input/trigger/touch",
+      right: "/user/hand/right/input/trigger/touch"
+    }
+  },
+
+  // Vive controller
+  {
+    profile: "/interaction_profiles/htc/vive_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    click: {
+      left: "/user/hand/left/input/trigger/value",
+      right: "/user/hand/right/input/trigger/value"
+    },
+    grab: {
+      left: "/user/hand/left/input/squeeze/click",
+      right: "/user/hand/right/input/squeeze/click"
+    },
+    scroll: {
+      left: "/user/hand/left/input/trackpad/y",
+      right: "/user/hand/right/input/trackpad/y"
+    },
+    scroll_horizontal: {
+      left: "/user/hand/left/input/trackpad/x",
+      right: "/user/hand/right/input/trackpad/x"
+    },
+    show_hide: {
+      left: "/user/hand/left/input/menu/click",
+    },
+    space_drag: {
+      right: "/user/hand/right/input/menu/click",
+    },
+    space_reset: {
+      double_click: true,
+      right: "/user/hand/right/input/menu/click",
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    }
+  },
+
+  // Windows Mixed Reality controller
+  {
+    profile: "/interaction_profiles/microsoft/motion_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    },
+    click: {
+      left: "/user/hand/left/input/trigger/value",
+      right: "/user/hand/right/input/trigger/value"
+    },
+    grab: {
+      left: "/user/hand/left/input/squeeze/click",
+      right: "/user/hand/right/input/squeeze/click"
+    },
+    scroll: {
+      left: "/user/hand/left/input/thumbstick/y",
+      right: "/user/hand/right/input/thumbstick/y"
+    },
+    scroll_horizontal: {
+      left: "/user/hand/left/input/thumbstick/x",
+      right: "/user/hand/right/input/thumbstick/x"
+    },
+    show_hide: {
+      left: "/user/hand/left/input/system/click",
+    },
+    space_drag: {
+      right: "/user/hand/right/input/system/click",
+    },
+    space_reset: {
+      double_click: true,
+      right: "/user/hand/right/input/system/click",
+    },
+    click_modifier_right: {
+      left: "/user/hand/left/input/trackpad/dpad_up",
+      right: "/user/hand/right/input/trackpad/dpad_up"
+    },
+    click_modifier_middle: {
+      left: "/user/hand/left/input/trackpad/dpad_down",
+      right: "/user/hand/right/input/trackpad/dpad_down"
+    },
+  },
+
+  // HP Reverb G2 controller
+  {
+    profile: "/interaction_profiles/hp/mixed_reality_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    },
+    click: {
+      left: "/user/hand/left/input/trigger/value",
+      right: "/user/hand/right/input/trigger/value"
+    },
+    grab: {
+      left: "/user/hand/left/input/squeeze/value",
+      right: "/user/hand/right/input/squeeze/value"
+    },
+    scroll: {
+      left: "/user/hand/left/input/thumbstick/y",
+      right: "/user/hand/right/input/thumbstick/y"
+    },
+    scroll_horizontal: {
+      left: "/user/hand/left/input/thumbstick/x",
+      right: "/user/hand/right/input/thumbstick/x"
+    },
+    show_hide: {
+      left: "/user/hand/left/input/system/click",
+    },
+    space_drag: {
+      right: "/user/hand/right/input/system/click",
+    },
+    space_reset: {
+      double_click: true,
+      right: "/user/hand/right/input/system/click",
+    },
+  },
+
+]

wlx-overlay-s/src/backend/openxr/overlay.rs

@@ -0,0 +1,123 @@
+use glam::Vec3A;
+use openxr::{self as xr, CompositionLayerFlags};
+use std::f32::consts::PI;
+use xr::EyeVisibility;
+
+use super::{helpers, swapchain::WlxSwapchain, CompositionLayer, XrState};
+use crate::{
+    backend::{
+        openxr::swapchain::{create_swapchain, SwapchainOpts},
+        overlay::OverlayData,
+    },
+    state::AppState,
+};
+
+#[derive(Default)]
+pub struct OpenXrOverlayData {
+    last_visible: bool,
+    pub(super) swapchain: Option<WlxSwapchain>,
+    pub(super) init: bool,
+    pub(super) cur_visible: bool,
+    pub(super) last_alpha: f32,
+}
+
+impl OverlayData<OpenXrOverlayData> {
+    pub(super) fn ensure_swapchain<'a>(
+        &'a mut self,
+        app: &AppState,
+        xr: &'a XrState,
+    ) -> anyhow::Result<bool> {
+        if self.data.swapchain.is_some() {
+            return Ok(true);
+        }
+
+        let Some(meta) = self.frame_meta() else {
+            log::warn!(
+                "{}: swapchain cannot be created due to missing metadata",
+                self.state.name
+            );
+            return Ok(false);
+        };
+
+        let extent = meta.extent;
+        self.data.swapchain = Some(create_swapchain(
+            xr,
+            app.gfx.clone(),
+            extent,
+            SwapchainOpts::new(),
+        )?);
+        Ok(true)
+    }
+
+    pub(super) fn present<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+    ) -> anyhow::Result<CompositionLayer<'a>> {
+        let Some(swapchain) = self.data.swapchain.as_mut() else {
+            log::warn!("{}: swapchain not ready", self.state.name);
+            return Ok(CompositionLayer::None);
+        };
+        if !swapchain.ever_acquired {
+            log::warn!("{}: swapchain not rendered", self.state.name);
+            return Ok(CompositionLayer::None);
+        }
+        swapchain.ensure_image_released()?;
+
+        let sub_image = swapchain.get_subimage();
+        let transform = self.state.transform * self.backend.frame_meta().unwrap().transform; // contract
+
+        let aspect_ratio = swapchain.extent[1] as f32 / swapchain.extent[0] as f32;
+        let (scale_x, scale_y) = if aspect_ratio < 1.0 {
+            let major = transform.matrix3.col(0).length();
+            (major, major * aspect_ratio)
+        } else {
+            let major = transform.matrix3.col(1).length();
+            (major / aspect_ratio, major)
+        };
+
+        if let Some(curvature) = self.state.curvature {
+            let radius = scale_x / (2.0 * PI * curvature);
+            let quat = helpers::transform_to_norm_quat(&transform);
+            let center_point = transform.translation + quat.mul_vec3a(Vec3A::Z * radius);
+
+            let posef = helpers::translation_rotation_to_posef(center_point, quat);
+            let angle = 2.0 * (scale_x / (2.0 * radius));
+
+            let cylinder = xr::CompositionLayerCylinderKHR::new()
+                .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA)
+                .pose(posef)
+                .sub_image(sub_image)
+                .eye_visibility(EyeVisibility::BOTH)
+                .space(&xr.stage)
+                .radius(radius)
+                .central_angle(angle)
+                .aspect_ratio(aspect_ratio);
+            Ok(CompositionLayer::Cylinder(cylinder))
+        } else {
+            let posef = helpers::transform_to_posef(&transform);
+            let quad = xr::CompositionLayerQuad::new()
+                .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA)
+                .pose(posef)
+                .sub_image(sub_image)
+                .eye_visibility(EyeVisibility::BOTH)
+                .space(&xr.stage)
+                .size(xr::Extent2Df {
+                    width: scale_x,
+                    height: scale_y,
+                });
+            Ok(CompositionLayer::Quad(quad))
+        }
+    }
+
+    pub(super) fn after_input(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        if self.data.last_visible != self.state.want_visible {
+            if self.state.want_visible {
+                self.backend.resume(app)?;
+            } else {
+                self.backend.pause(app)?;
+            }
+        }
+        self.data.last_visible = self.state.want_visible;
+        Ok(())
+    }
+}

wlx-overlay-s/src/backend/openxr/playspace.rs

@@ -0,0 +1,199 @@
+use glam::{Affine3A, Quat, Vec3A};
+use libmonado::{Monado, Pose, ReferenceSpaceType};
+
+use crate::{
+    backend::{common::OverlayContainer, input::InputState},
+    state::AppState,
+};
+
+use super::overlay::OpenXrOverlayData;
+
+struct MoverData<T> {
+    pose: Affine3A,
+    hand: usize,
+    hand_pose: T,
+}
+
+pub(super) struct PlayspaceMover {
+    last_transform: Affine3A,
+    drag: Option<MoverData<Vec3A>>,
+    rotate: Option<MoverData<Quat>>,
+}
+
+impl PlayspaceMover {
+    pub fn new(monado: &mut Monado) -> anyhow::Result<Self> {
+        log::info!("Monado: using space offset API");
+
+        let Ok(stage) = monado.get_reference_space_offset(ReferenceSpaceType::Stage) else {
+            anyhow::bail!("Space offsets not supported.");
+        };
+
+        log::debug!("STAGE is at {:?}, {:?}", stage.position, stage.orientation);
+
+        // initial offset
+        let last_transform =
+            Affine3A::from_rotation_translation(stage.orientation.into(), stage.position.into());
+
+        Ok(Self {
+            last_transform,
+
+            drag: None,
+            rotate: None,
+        })
+    }
+
+    pub fn update(
+        &mut self,
+        overlays: &mut OverlayContainer<OpenXrOverlayData>,
+        state: &AppState,
+        monado: &mut Monado,
+    ) {
+        for pointer in &state.input_state.pointers {
+            if pointer.now.space_reset {
+                if !pointer.before.space_reset {
+                    log::info!("Space reset");
+                    self.reset_offset(monado);
+                }
+                return;
+            }
+        }
+
+        if let Some(mut data) = self.rotate.take() {
+            let pointer = &state.input_state.pointers[data.hand];
+            if !pointer.now.space_rotate {
+                self.last_transform = data.pose;
+                log::info!("End space rotate");
+                return;
+            }
+
+            let new_hand =
+                Quat::from_affine3(&(data.pose * state.input_state.pointers[data.hand].raw_pose));
+
+            let dq = new_hand * data.hand_pose.conjugate();
+            let mut space_transform = if state.session.config.space_rotate_unlocked {
+                Affine3A::from_quat(dq)
+            } else {
+                let rel_y = f32::atan2(
+                    2.0 * dq.y.mul_add(dq.w, dq.x * dq.z),
+                    2.0f32.mul_add(dq.w.mul_add(dq.w, dq.x * dq.x), -1.0),
+                );
+
+                Affine3A::from_rotation_y(rel_y)
+            };
+            let offset = (space_transform.transform_vector3a(state.input_state.hmd.translation)
+                - state.input_state.hmd.translation)
+                * -1.0;
+
+            space_transform.translation = offset;
+
+            data.pose *= space_transform;
+            data.hand_pose = new_hand;
+
+            apply_offset(data.pose, monado);
+            self.rotate = Some(data);
+        } else {
+            for (i, pointer) in state.input_state.pointers.iter().enumerate() {
+                if pointer.now.space_rotate {
+                    let hand_pose = Quat::from_affine3(&(self.last_transform * pointer.raw_pose));
+                    self.rotate = Some(MoverData {
+                        pose: self.last_transform,
+                        hand: i,
+                        hand_pose,
+                    });
+                    self.drag = None;
+                    log::info!("Start space rotate");
+                    return;
+                }
+            }
+        }
+
+        if let Some(mut data) = self.drag.take() {
+            let pointer = &state.input_state.pointers[data.hand];
+            if !pointer.now.space_drag {
+                self.last_transform = data.pose;
+                log::info!("End space drag");
+                return;
+            }
+
+            let new_hand = data
+                .pose
+                .transform_point3a(state.input_state.pointers[data.hand].raw_pose.translation);
+            let relative_pos =
+                (new_hand - data.hand_pose) * state.session.config.space_drag_multiplier;
+
+            if relative_pos.length_squared() > 1000.0 {
+                log::warn!("Space drag too fast, ignoring");
+                return;
+            }
+
+            let overlay_offset = data.pose.inverse().transform_vector3a(relative_pos) * -1.0;
+
+            overlays.iter_mut().for_each(|overlay| {
+                if overlay.state.grabbable {
+                    overlay.state.dirty = true;
+                    overlay.state.transform.translation += overlay_offset;
+                }
+            });
+
+            data.pose.translation += relative_pos;
+            data.hand_pose = new_hand;
+
+            apply_offset(data.pose, monado);
+            self.drag = Some(data);
+        } else {
+            for (i, pointer) in state.input_state.pointers.iter().enumerate() {
+                if pointer.now.space_drag {
+                    let hand_pos = self
+                        .last_transform
+                        .transform_point3a(pointer.raw_pose.translation);
+                    self.drag = Some(MoverData {
+                        pose: self.last_transform,
+                        hand: i,
+                        hand_pose: hand_pos,
+                    });
+                    log::info!("Start space drag");
+                    return;
+                }
+            }
+        }
+    }
+
+    pub fn reset_offset(&mut self, monado: &mut Monado) {
+        if self.drag.is_some() {
+            log::info!("Space drag interrupted by manual reset");
+            self.drag = None;
+        }
+        if self.rotate.is_some() {
+            log::info!("Space rotate interrupted by manual reset");
+            self.rotate = None;
+        }
+
+        self.last_transform = Affine3A::IDENTITY;
+        apply_offset(self.last_transform, monado);
+    }
+
+    pub fn fix_floor(&mut self, input: &InputState, monado: &mut Monado) {
+        if self.drag.is_some() {
+            log::info!("Space drag interrupted by fix floor");
+            self.drag = None;
+        }
+        if self.rotate.is_some() {
+            log::info!("Space rotate interrupted by fix floor");
+            self.rotate = None;
+        }
+
+        let y1 = input.pointers[0].raw_pose.translation.y;
+        let y2 = input.pointers[1].raw_pose.translation.y;
+        let delta = y1.min(y2) - 0.03;
+        self.last_transform.translation.y += delta;
+        apply_offset(self.last_transform, monado);
+    }
+}
+
+fn apply_offset(transform: Affine3A, monado: &mut Monado) {
+    let pose = Pose {
+        position: transform.translation.into(),
+        orientation: Quat::from_affine3(&transform).into(),
+    };
+    let _ = monado.set_reference_space_offset(ReferenceSpaceType::Stage, pose);
+}

wlx-overlay-s/src/backend/openxr/skybox.rs

@@ -0,0 +1,252 @@
+use std::{
+    f32::consts::PI,
+    fs::File,
+    sync::{Arc, LazyLock},
+};
+
+use glam::{Quat, Vec3A};
+use openxr as xr;
+use vulkano::{
+    command_buffer::CommandBufferUsage,
+    image::view::ImageView,
+    pipeline::graphics::{color_blend::AttachmentBlend, input_assembly::PrimitiveTopology},
+};
+
+use crate::{
+    backend::openxr::{helpers::translation_rotation_to_posef, swapchain::SwapchainOpts},
+    config_io,
+    graphics::{dds::WlxCommandBufferDds, CommandBuffers, ExtentExt},
+    state::AppState,
+};
+
+use super::{
+    swapchain::{create_swapchain, WlxSwapchain},
+    CompositionLayer, XrState,
+};
+
+pub(super) struct Skybox {
+    view: Arc<ImageView>,
+    sky: Option<WlxSwapchain>,
+    grid: Option<WlxSwapchain>,
+}
+
+impl Skybox {
+    pub fn new(app: &AppState) -> anyhow::Result<Self> {
+        let mut command_buffer = app
+            .gfx
+            .create_xfer_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+
+        let mut maybe_image = None;
+
+        'custom_tex: {
+            if app.session.config.skybox_texture.is_empty() {
+                break 'custom_tex;
+            }
+
+            let real_path = config_io::get_config_root().join(&*app.session.config.skybox_texture);
+            let Ok(f) = File::open(real_path) else {
+                log::warn!(
+                    "Could not open custom skybox texture at: {}",
+                    app.session.config.skybox_texture
+                );
+                break 'custom_tex;
+            };
+            match command_buffer.upload_image_dds(f) {
+                Ok(image) => {
+                    maybe_image = Some(image);
+                }
+                Err(e) => {
+                    log::warn!(
+                        "Could not use custom skybox texture at: {}",
+                        app.session.config.skybox_texture
+                    );
+                    log::warn!("{e:?}");
+                }
+            }
+        }
+
+        if maybe_image.is_none() {
+            let p = include_bytes!("../../res/table_mountain_2.dds");
+            maybe_image = Some(command_buffer.upload_image_dds(p.as_slice())?);
+        }
+
+        command_buffer.build_and_execute_now()?;
+
+        let view = ImageView::new_default(maybe_image.unwrap())?; // safe unwrap
+
+        Ok(Self {
+            view,
+            sky: None,
+            grid: None,
+        })
+    }
+
+    fn prepare_sky<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+        app: &AppState,
+        buf: &mut CommandBuffers,
+    ) -> anyhow::Result<()> {
+        if self.sky.is_some() {
+            return Ok(());
+        }
+        let opts = SwapchainOpts::new().immutable();
+
+        let extent = self.view.image().extent();
+        let mut swapchain = create_swapchain(xr, app.gfx.clone(), extent, opts)?;
+        let tgt = swapchain.acquire_wait_image()?;
+        let pipeline = app.gfx.create_pipeline(
+            app.gfx_extras.shaders.get("vert_quad").unwrap().clone(), // want panic
+            app.gfx_extras.shaders.get("frag_srgb").unwrap().clone(), // want panic
+            app.gfx.surface_format,
+            None,
+            PrimitiveTopology::TriangleStrip,
+            false,
+        )?;
+
+        let set0 = pipeline.uniform_sampler(0, self.view.clone(), app.gfx.texture_filter)?;
+        let set1 = pipeline.uniform_buffer_upload(1, vec![1f32])?;
+        let pass = pipeline.create_pass(
+            tgt.extent_f32(),
+            app.gfx_extras.quad_verts.clone(),
+            0..4,
+            0..1,
+            vec![set0, set1],
+        )?;
+
+        let mut cmd_buffer = app
+            .gfx
+            .create_gfx_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+        cmd_buffer.begin_rendering(tgt)?;
+        cmd_buffer.run_ref(&pass)?;
+        cmd_buffer.end_rendering()?;
+
+        buf.push(cmd_buffer.build()?);
+
+        self.sky = Some(swapchain);
+        Ok(())
+    }
+
+    fn prepare_grid<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+        app: &AppState,
+        buf: &mut CommandBuffers,
+    ) -> anyhow::Result<()> {
+        if self.grid.is_some() {
+            return Ok(());
+        }
+
+        let extent = [1024, 1024, 1];
+        let mut swapchain = create_swapchain(
+            xr,
+            app.gfx.clone(),
+            extent,
+            SwapchainOpts::new().immutable(),
+        )?;
+        let pipeline = app.gfx.create_pipeline(
+            app.gfx_extras.shaders.get("vert_quad").unwrap().clone(), // want panic
+            app.gfx_extras.shaders.get("frag_grid").unwrap().clone(), // want panic
+            app.gfx.surface_format,
+            Some(AttachmentBlend::alpha()),
+            PrimitiveTopology::TriangleStrip,
+            false,
+        )?;
+
+        let tgt = swapchain.acquire_wait_image()?;
+        let pass = pipeline.create_pass(
+            tgt.extent_f32(),
+            app.gfx_extras.quad_verts.clone(),
+            0..4,
+            0..1,
+            vec![],
+        )?;
+
+        let mut cmd_buffer = app
+            .gfx
+            .create_gfx_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+        cmd_buffer.begin_rendering(tgt)?;
+        cmd_buffer.run_ref(&pass)?;
+        cmd_buffer.end_rendering()?;
+
+        buf.push(cmd_buffer.build()?);
+
+        self.grid = Some(swapchain);
+        Ok(())
+    }
+
+    pub(super) fn render(
+        &mut self,
+        xr: &XrState,
+        app: &AppState,
+        buf: &mut CommandBuffers,
+    ) -> anyhow::Result<()> {
+        self.prepare_sky(xr, app, buf)?;
+        self.prepare_grid(xr, app, buf)?;
+        Ok(())
+    }
+
+    pub(super) fn present<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+        app: &AppState,
+    ) -> anyhow::Result<Vec<CompositionLayer<'a>>> {
+        // cover the entire sphere
+        const HORIZ_ANGLE: f32 = 2.0 * PI;
+        const HI_VERT_ANGLE: f32 = 0.5 * PI;
+        const LO_VERT_ANGLE: f32 = -0.5 * PI;
+
+        static GRID_POSE: LazyLock<xr::Posef> = LazyLock::new(|| {
+            translation_rotation_to_posef(Vec3A::ZERO, Quat::from_rotation_x(PI * -0.5))
+        });
+
+        let pose = xr::Posef {
+            orientation: xr::Quaternionf::IDENTITY,
+            position: xr::Vector3f {
+                x: app.input_state.hmd.translation.x,
+                y: app.input_state.hmd.translation.y,
+                z: app.input_state.hmd.translation.z,
+            },
+        };
+
+        self.sky.as_mut().unwrap().ensure_image_released()?;
+
+        let sky = xr::CompositionLayerEquirect2KHR::new()
+            .layer_flags(xr::CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA)
+            .pose(pose)
+            .radius(10.0)
+            .sub_image(self.sky.as_ref().unwrap().get_subimage())
+            .eye_visibility(xr::EyeVisibility::BOTH)
+            .space(&xr.stage)
+            .central_horizontal_angle(HORIZ_ANGLE)
+            .upper_vertical_angle(HI_VERT_ANGLE)
+            .lower_vertical_angle(LO_VERT_ANGLE);
+
+        self.grid.as_mut().unwrap().ensure_image_released()?;
+        let grid = xr::CompositionLayerQuad::new()
+            .layer_flags(xr::CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA)
+            .pose(*GRID_POSE)
+            .size(xr::Extent2Df {
+                width: 10.0,
+                height: 10.0,
+            })
+            .sub_image(self.grid.as_ref().unwrap().get_subimage())
+            .eye_visibility(xr::EyeVisibility::BOTH)
+            .space(&xr.stage);
+
+        Ok(vec![
+            CompositionLayer::Equirect2(sky),
+            CompositionLayer::Quad(grid),
+        ])
+    }
+}
+
+pub(super) fn create_skybox(xr: &XrState, app: &AppState) -> Option<Skybox> {
+    if !app.session.config.use_skybox {
+        return None;
+    }
+    xr.instance
+        .exts()
+        .khr_composition_layer_equirect2
+        .and_then(|_| Skybox::new(app).ok())
+}

wlx-overlay-s/src/backend/openxr/swapchain.rs

@@ -0,0 +1,125 @@
+use std::sync::Arc;
+
+use ash::vk;
+use openxr as xr;
+
+use smallvec::SmallVec;
+use vulkano::{
+    image::{sys::RawImage, view::ImageView, ImageCreateInfo, ImageUsage},
+    Handle,
+};
+use wgui::gfx::WGfx;
+
+use super::XrState;
+
+#[derive(Default)]
+pub(super) struct SwapchainOpts {
+    pub immutable: bool,
+}
+
+impl SwapchainOpts {
+    pub fn new() -> Self {
+        Self::default()
+    }
+    pub const fn immutable(mut self) -> Self {
+        self.immutable = true;
+        self
+    }
+}
+
+pub(super) fn create_swapchain(
+    xr: &XrState,
+    gfx: Arc<WGfx>,
+    extent: [u32; 3],
+    opts: SwapchainOpts,
+) -> anyhow::Result<WlxSwapchain> {
+    let create_flags = if opts.immutable {
+        xr::SwapchainCreateFlags::STATIC_IMAGE
+    } else {
+        xr::SwapchainCreateFlags::EMPTY
+    };
+
+    let swapchain = xr.session.create_swapchain(&xr::SwapchainCreateInfo {
+        create_flags,
+        usage_flags: xr::SwapchainUsageFlags::COLOR_ATTACHMENT | xr::SwapchainUsageFlags::SAMPLED,
+        format: gfx.surface_format as _,
+        sample_count: 1,
+        width: extent[0],
+        height: extent[1],
+        face_count: 1,
+        array_size: 1,
+        mip_count: 1,
+    })?;
+
+    let images = swapchain
+        .enumerate_images()?
+        .into_iter()
+        .map(|handle| {
+            let vk_image = vk::Image::from_raw(handle);
+            // thanks @yshui
+            let raw_image = unsafe {
+                RawImage::from_handle_borrowed(
+                    gfx.device.clone(),
+                    vk_image,
+                    ImageCreateInfo {
+                        format: gfx.surface_format as _,
+                        extent,
+                        usage: ImageUsage::COLOR_ATTACHMENT,
+                        ..Default::default()
+                    },
+                )?
+            };
+            // SAFETY: OpenXR guarantees that the image is a swapchain image, thus has memory backing it.
+            let image = Arc::new(unsafe { raw_image.assume_bound() });
+            Ok(ImageView::new_default(image)?)
+        })
+        .collect::<anyhow::Result<SmallVec<[Arc<ImageView>; 4]>>>()?;
+
+    Ok(WlxSwapchain {
+        acquired: false,
+        ever_acquired: false,
+        swapchain,
+        images,
+        extent,
+    })
+}
+
+pub(super) struct WlxSwapchain {
+    acquired: bool,
+    pub(super) ever_acquired: bool,
+    pub(super) swapchain: xr::Swapchain<xr::Vulkan>,
+    pub(super) extent: [u32; 3],
+    pub(super) images: SmallVec<[Arc<ImageView>; 4]>,
+}
+
+impl WlxSwapchain {
+    pub(super) fn acquire_wait_image(&mut self) -> anyhow::Result<Arc<ImageView>> {
+        let idx = self.swapchain.acquire_image()? as usize;
+        self.swapchain.wait_image(xr::Duration::INFINITE)?;
+        self.ever_acquired = true;
+        self.acquired = true;
+        Ok(self.images[idx].clone())
+    }
+
+    pub(super) fn ensure_image_released(&mut self) -> anyhow::Result<()> {
+        if self.acquired {
+            self.swapchain.release_image()?;
+            self.acquired = false;
+        }
+        Ok(())
+    }
+
+    pub(super) fn get_subimage(&self) -> xr::SwapchainSubImage<xr::Vulkan> {
+        debug_assert!(self.ever_acquired, "swapchain was never acquired!");
+        xr::SwapchainSubImage::new()
+            .swapchain(&self.swapchain)
+            .image_rect(xr::Rect2Di {
+                offset: xr::Offset2Di { x: 0, y: 0 },
+                extent: xr::Extent2Di {
+                    width: self.extent[0] as _,
+                    height: self.extent[1] as _,
+                },
+            })
+            .image_array_index(0)
+    }
+}