handsfree mode

dash-frontend/assets/lang/en.json

@@ -67,6 +67,9 @@
 		"NOTIFICATIONS_SOUND_ENABLED": "Notification sounds",
 		"OPAQUE_BACKGROUND": "Opaque background",
 		"OPTION": {
+			"NONE": "None",
+			"HMD_PINCH": "HMD + pinch",
+			"EYE_PINCH": "Eye + pinch",
 			"AUTO": "Automatic",
 			"AUTO_HELP": "ScreenCopy GPU if supported,\notherwise PipeWire GPU.",
 			"PIPEWIRE_HELP": "Fast GPU capture,\nstandard on all desktops.",
@@ -97,6 +100,8 @@
 		"XR_CLICK_SENSITIVITY_RELEASE": "XR release sensitivity",
 		"XR_CLICK_SENSITIVITY_RELEASE_HELP": "Must be lower than click",
 		"XWAYLAND_BY_DEFAULT": "Run apps in Compatibility mode by default",
+		"HANDSFREE_POINTER": "Handsfree mode",
+		"HANDSFREE_POINTER_HELP": "Input to use when motion\ncontrollers are unavailable",
 		"AUTOSTART_APPS": "Apps to run on startup"
 	},
 	"APPLICATION_LAUNCHER": "Application launcher",

dash-frontend/src/tab/settings.rs

@@ -224,6 +224,7 @@ enum SettingType {
 	XwaylandByDefault,
 	CaptureMethod,
 	KeyboardMiddleClick,
+	HandsfreePointer,
 }
 
 impl SettingType {
@@ -286,6 +287,9 @@ impl SettingType {
 				config.keyboard_middle_click_mode =
 					wlx_common::config::AltModifier::from_str(value).expect("Invalid enum value!")
 			}
+			Self::HandsfreePointer => {
+				config.handsfree_pointer = wlx_common::config::HandsfreePointer::from_str(value).expect("Invalid enum value!")
+			}
 			_ => panic!("Requested enum for non-enum SettingType"),
 		}
 	}
@@ -294,6 +298,7 @@ impl SettingType {
 		match self {
 			Self::CaptureMethod => Self::get_enum_title_inner(config.capture_method),
 			Self::KeyboardMiddleClick => Self::get_enum_title_inner(config.keyboard_middle_click_mode),
+			Self::HandsfreePointer => Self::get_enum_title_inner(config.handsfree_pointer),
 			_ => panic!("Requested enum for non-enum SettingType"),
 		}
 	}
@@ -353,6 +358,7 @@ impl SettingType {
 			Self::XwaylandByDefault => Ok("APP_SETTINGS.XWAYLAND_BY_DEFAULT"),
 			Self::CaptureMethod => Ok("APP_SETTINGS.CAPTURE_METHOD"),
 			Self::KeyboardMiddleClick => Ok("APP_SETTINGS.KEYBOARD_MIDDLE_CLICK"),
+			Self::HandsfreePointer => Ok("APP_SETTINGS.HANDSFREE_POINTER"),
 		}
 	}
 
@@ -371,6 +377,7 @@ impl SettingType {
 			Self::ScreenRenderDown => Some("APP_SETTINGS.SCREEN_RENDER_DOWN_HELP"),
 			Self::CaptureMethod => Some("APP_SETTINGS.CAPTURE_METHOD_HELP"),
 			Self::KeyboardMiddleClick => Some("APP_SETTINGS.KEYBOARD_MIDDLE_CLICK_HELP"),
+			Self::HandsfreePointer => Some("APP_SETTINGS.HANDSFREE_POINTER_HELP"),
 			_ => None,
 		}
 	}
@@ -382,10 +389,10 @@ impl SettingType {
 			| Self::RoundMultiplier
 			| Self::UprightScreenFix
 			| Self::DoubleCursorFix
-			| Self::SetsOnWatch
 			| Self::UseSkybox
 			| Self::UsePassthrough
-			| Self::ScreenRenderDown => true,
+			| Self::ScreenRenderDown
+			| Self::CaptureMethod => true,
 			_ => false,
 		}
 	}
@@ -711,6 +718,12 @@ impl<T> TabSettings<T> {
 					SettingType::KeyboardMiddleClick,
 					wlx_common::config::AltModifier::VARIANTS
 				);
+				dropdown!(
+					mp,
+					c,
+					SettingType::HandsfreePointer,
+					wlx_common::config::HandsfreePointer::VARIANTS
+				);
 				checkbox!(mp, c, SettingType::FocusFollowsMouseMode);
 				checkbox!(mp, c, SettingType::LeftHandedMouse);
 				checkbox!(mp, c, SettingType::AllowSliding);

dash-frontend/src/views/app_launcher.rs

@@ -156,7 +156,7 @@ impl View {
 		} else {
 			CompositorMode::Native
 		};
-		radio_compositor.set_value(compositor_mode.as_ref())?;
+		radio_compositor.set_value_simple(compositor_mode.as_ref())?;
 		tasks.push(Task::SetCompositor(compositor_mode));
 
 		let res_mode = ResMode::Res1080;

wayvr/src/assets/gui/keyboard.xml

@@ -163,12 +163,21 @@
     </Button>
   </template>
 
+  <blueprint name="menu_handsfree">
+    <context_menu >
+      <cell translation="BAR.HANDSFREE.NONE" _press="::HandsfreeMode None" />
+      <cell translation="BAR.HANDSFREE.HMD" _press="::HandsfreeMode Hmd" />
+      <cell translation="BAR.HANDSFREE.EYE_TRACKING" _press="::HandsfreeMode EyeTracking" />
+    </context_menu>
+  </blueprint>
+
   <blueprint name="menu_burger">
     <context_menu >
       <cell translation="BAR.ADD_MIRROR" _press="::NewMirror" />
       <cell translation="BAR.EDIT_MODE_TOGGLE" _press="::EditToggle" />
       <cell translation="BAR.ADD_NEW_SET" _press="::AddSet" />
       <cell translation="BAR.DELETE_CURRENT_SET" _press="::DeleteSet" />
+      <cell translation="BAR.HANDSFREE.TITLE" _press="::ContextMenuOpen menu_handsfree" />
     </context_menu>
   </blueprint>
 

wayvr/src/assets/lang/en.json

@@ -12,7 +12,13 @@
 		"RELOAD_FROM_DISK": "Reload XML from disk",
 		"CLOSE_MIRROR": "Close mirror",
 		"CLOSE_APP": "Close app",
-		"FORCE_CLOSE_APP": "Force close app"
+		"FORCE_CLOSE_APP": "Force close app",
+		"HANDSFREE": {
+			"TITLE": "Handsfree mode",
+			"NONE": "Off",
+			"HMD": "HMD + pinch",
+			"EYE_TRACKING": "Eye + pinch"
+		}
 	},
 	"DEFAULT": "Default",
 	"DISABLED": "Disabled",

wayvr/src/backend/input.rs

@@ -3,7 +3,7 @@ use std::process::{Child, Command};
 use std::sync::Arc;
 use std::time::Instant;
 
-use glam::{Affine3A, Vec2, Vec3A, Vec3Swizzles};
+use glam::{Affine3A, Mat3A, Vec2, Vec3, Vec3A, Vec3Swizzles};
 
 use idmap_derive::IntegerId;
 use smallvec::{SmallVec, smallvec};
@@ -233,6 +233,8 @@ pub struct Pointer {
     pub last_click: Instant,
     pub pending_haptics: Option<Haptics>,
     pub(super) interaction: InteractionState,
+    pub tracked: bool,
+    pub handsfree: bool,
 }
 
 impl Pointer {
@@ -247,6 +249,8 @@ impl Pointer {
             last_click: Instant::now(),
             pending_haptics: None,
             interaction: InteractionState::default(),
+            tracked: false,
+            handsfree: false,
         }
     }
 
@@ -318,6 +322,47 @@ pub enum PointerMode {
     Special,
 }
 
+pub struct PointerLine {
+    pub mode: PointerMode,
+    pub a: Vec3A,
+    pub b: Vec3A,
+}
+
+fn populate_lines(
+    lines: &mut Vec<PointerLine>,
+    pointer: &Pointer,
+    maybe_hit: Option<(PointerHit, RayHit)>,
+    hmd: &Affine3A,
+) {
+    let Some((hit, raw_hit)) = maybe_hit else {
+        return;
+    };
+
+    if pointer.handsfree {
+        const HALF_SIZE: f32 = 0.01;
+
+        // horizontal arm
+        lines.push(PointerLine {
+            mode: PointerMode::Left,
+            a: raw_hit.global_pos - (hmd.x_axis * HALF_SIZE),
+            b: raw_hit.global_pos + (hmd.x_axis * HALF_SIZE),
+        });
+
+        // vertical arm
+        lines.push(PointerLine {
+            mode: PointerMode::Special,
+            a: raw_hit.global_pos - (hmd.y_axis * HALF_SIZE),
+            b: raw_hit.global_pos + (hmd.y_axis * HALF_SIZE),
+        });
+    } else {
+        lines.push(PointerLine {
+            mode: hit.mode,
+            a: pointer.pose.translation,
+            b: raw_hit.global_pos,
+        });
+    }
+}
+
 fn update_focus(focus: &mut KeyboardFocus, overlay_keyboard_focus: Option<KeyboardFocus>) {
     if let Some(f) = &overlay_keyboard_focus
         && *focus != *f
@@ -330,11 +375,12 @@ fn update_focus(focus: &mut KeyboardFocus, overlay_keyboard_focus: Option<Keyboa
 pub fn interact<O>(
     overlays: &mut OverlayWindowManager<O>,
     app: &mut AppState,
-) -> [(f32, Option<Haptics>); 2]
+    lines: &mut Vec<PointerLine>,
+) -> [Option<Haptics>; 2]
 where
     O: Default,
 {
-    if app.input_state.pointers[1].last_click > app.input_state.pointers[0].last_click {
+    let hits = if app.input_state.pointers[1].last_click > app.input_state.pointers[0].last_click {
         let right = interact_hand(1, overlays, app);
         let left = interact_hand(0, overlays, app);
         [left, right]
@@ -342,21 +388,36 @@ where
         let left = interact_hand(0, overlays, app);
         let right = interact_hand(1, overlays, app);
         [left, right]
+    };
+
+    for (idx, hit) in hits.iter().enumerate() {
+        populate_lines(
+            lines,
+            &mut app.input_state.pointers[idx],
+            hit.0,
+            &app.input_state.hmd,
+        );
     }
+
+    [hits[0].1, hits[1].1]
 }
 
 fn interact_hand<O>(
     idx: usize,
     overlays: &mut OverlayWindowManager<O>,
     app: &mut AppState,
-) -> (f32, Option<Haptics>)
+) -> (Option<(PointerHit, RayHit)>, Option<Haptics>)
 where
     O: Default,
 {
-    // already grabbing, ignore everything else
     let mut pointer = &mut app.input_state.pointers[idx];
     let pending_haptics = pointer.pending_haptics.take();
 
+    if !pointer.tracked {
+        return (None, pending_haptics); // no hit
+    }
+
+    // already grabbing, ignore everything else
     if let Some(grab_data) = pointer.interaction.grabbed {
         if let Some(grabbed) = overlays.mut_by_id(grab_data.grabbed_id) {
             handle_grabbed(idx, grabbed, app);
@@ -364,13 +425,13 @@ where
             log::warn!("Grabbed overlay {:?} does not exist", grab_data.grabbed_id);
             pointer.interaction.grabbed = None;
         }
-        return (0.1, pending_haptics);
+        return (None, pending_haptics);
     }
 
     let hovered_id = pointer.interaction.hovered_id.take();
-    let (Some(mut hit), haptics) = get_nearest_hit(idx, overlays, app) else {
+    let (Some((mut hit, raw_hit)), haptics) = get_nearest_hit(idx, overlays, app) else {
         handle_no_hit(idx, hovered_id, overlays, app);
-        return (0.0, pending_haptics); // no hit
+        return (None, pending_haptics); // no hit
     };
 
     // focus change
@@ -394,7 +455,7 @@ where
 
     let Some(hovered) = overlays.mut_by_id(hit.overlay) else {
         log::warn!("Hit overlay {:?} does not exist", hit.overlay);
-        return (0.0, pending_haptics); // no hit
+        return (None, pending_haptics); // no hit
     };
     pointer = &mut app.input_state.pointers[idx];
     pointer.interaction.hovered_id = Some(hit.overlay);
@@ -432,7 +493,7 @@ where
         );
         log::debug!("Hand {}: grabbed {}", hit.pointer, hovered.config.name);
         return (
-            hit.dist,
+            Some((hit, raw_hit)),
             Some(Haptics {
                 intensity: 0.25,
                 duration: 0.1,
@@ -463,7 +524,7 @@ where
         }
     }
 
-    (hit.dist, haptics.or(pending_haptics))
+    (Some((hit, raw_hit)), haptics.or(pending_haptics))
 }
 
 fn handle_no_hit<O>(
@@ -562,7 +623,7 @@ fn get_nearest_hit<O>(
     pointer_idx: usize,
     overlays: &mut OverlayWindowManager<O>,
     app: &mut AppState,
-) -> (Option<PointerHit>, Option<Haptics>)
+) -> (Option<(PointerHit, RayHit)>, Option<Haptics>)
 where
     O: Default,
 {
@@ -594,7 +655,7 @@ where
 
     hits.sort_by(|a, b| a.dist.total_cmp(&b.dist));
 
-    for hit in &hits {
+    for hit in hits {
         let overlay = overlays.mut_by_id(hit.overlay).unwrap(); // safe because we just got the id from the overlay
 
         let Some(uv) = overlay
@@ -611,7 +672,7 @@ where
             continue;
         }
 
-        let hit = PointerHit {
+        let pointer_hit = PointerHit {
             pointer: pointer_idx,
             overlay: hit.overlay,
             mode,
@@ -620,9 +681,9 @@ where
             dist: hit.dist,
         };
 
-        let result = overlay.config.backend.on_hover(app, &hit);
+        let result = overlay.config.backend.on_hover(app, &pointer_hit);
         if result.consume || overlay.config.editing {
-            return (Some(hit), result.haptics);
+            return (Some((pointer_hit, hit)), result.haptics);
         }
     }
 
@@ -673,6 +734,7 @@ fn start_grab(
 
     if let Some(hand) = pointer.hand()
         && !app.session.config.hide_grab_help
+        && !pointer.handsfree
     {
         let pos = state.positioning;
         app.tasks.enqueue(TaskType::Overlay(OverlayTask::Modify(

wayvr/src/backend/openvr/input.rs

@@ -175,6 +175,9 @@ impl OpenVrInputSource {
                 app_hand.raw_pose = devices[device.0 as usize]
                     .mDeviceToAbsoluteTracking
                     .to_affine();
+                app_hand.tracked = devices[device.0 as usize].bPoseIsValid;
+            } else {
+                app_hand.tracked = false;
             }
 
             hand.has_pose = false;

wayvr/src/backend/openvr/lines.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 use ash::vk::SubmitInfo;
-use glam::{Affine3A, Vec3, Vec3A, Vec4};
+use glam::{Affine3A, Quat, Vec3, Vec3A, Vec4};
 use idmap::IdMap;
 use ovr_overlay::overlay::OverlayManager;
 use ovr_overlay::sys::ETrackingUniverseOrigin;
@@ -103,27 +103,44 @@ impl LinePool {
         id
     }
 
-    pub fn draw_from(
+    pub(super) fn draw_between(
         &mut self,
         id: usize,
-        mut from: Affine3A,
+        start: Vec3A,
-        len: f32,
+        end: Vec3A,
         color: usize,
         hmd: &Affine3A,
     ) {
-        let rotation = Affine3A::from_axis_angle(Vec3::X, -PI * 0.5);
+        let dir = end - start;
+        let len = dir.length();
 
-        from.translation += from.transform_vector3a(Vec3A::NEG_Z) * (len * 0.5);
+        if len < 0.01 {
-        let mut transform = from * rotation * Affine3A::from_scale(Vec3::new(1., len / 0.002, 1.));
+            return;
+        }
 
-        let to_hmd = hmd.translation - from.translation;
+        debug_assert!(color < self.colors.len());
+
+        let center = (start + end) * 0.5;
+        let dir_norm = dir / len;
+
+        let xform = Affine3A::from_rotation_translation(
+            Quat::from_rotation_arc(Vec3::Z, dir_norm.into()),
+            center.into(),
+        );
+
+        let rotation = Affine3A::from_axis_angle(Vec3::X, PI * 1.5);
+        let mut transform = xform * rotation;
+        let to_hmd = hmd.translation - center;
         let sides = [Vec3A::Z, Vec3A::X, Vec3A::NEG_Z, Vec3A::NEG_X];
+
+        #[allow(clippy::neg_multiply)]
         let rotations = [
             Affine3A::IDENTITY,
             Affine3A::from_axis_angle(Vec3::Y, PI * 0.5),
-            Affine3A::from_axis_angle(Vec3::Y, PI * 1.0),
+            Affine3A::from_axis_angle(Vec3::Y, PI * -1.0),
             Affine3A::from_axis_angle(Vec3::Y, PI * 1.5),
         ];
+
         let mut closest = (0, 0.0);
         for (i, &side) in sides.iter().enumerate() {
             let dot = to_hmd.dot(transform.transform_vector3a(side));
@@ -135,7 +152,6 @@ impl LinePool {
         transform *= rotations[closest.0];
 
         debug_assert!(color < self.colors.len());
-
         self.draw_transform(id, transform, self.colors[color]);
     }
 

wayvr/src/backend/openvr/mod.rs

@@ -144,6 +144,7 @@ pub fn openvr_run(show_by_default: bool, headless: bool) -> Result<(), BackendEr
 
     let mut lines = LinePool::new(app.gfx.clone())?;
     let pointer_lines = [lines.allocate(), lines.allocate()];
+    let mut current_lines = Vec::with_capacity(2);
 
     'main_loop: loop {
         let _ = overlay_mgr.wait_frame_sync(frame_timeout);
@@ -276,19 +277,23 @@ pub fn openvr_run(show_by_default: bool, headless: bool) -> Result<(), BackendEr
         watch_fade(&mut app, overlays.mut_by_id(watch_id).unwrap()); // want panic
         playspace.update(&mut chaperone_mgr, &mut overlays, &app);
 
-        let lengths_haptics = interact(&mut overlays, &mut app);
+        current_lines.clear();
-        for (idx, (len, haptics)) in lengths_haptics.iter().enumerate() {
+
-            lines.draw_from(
+        let haptics = interact(&mut overlays, &mut app, &mut current_lines);
-                pointer_lines[idx],
+        for (idx, haptics) in haptics.iter().enumerate() {
-                app.input_state.pointers[idx].pose,
-                *len,
-                app.input_state.pointers[idx].interaction.mode as usize + 1,
-                &app.input_state.hmd,
-            );
             if let Some(haptics) = haptics {
                 input_source.haptics(&mut input_mgr, idx, haptics);
             }
         }
+        for (idx, line) in current_lines.iter().enumerate() {
+            lines.draw_between(
+                pointer_lines[idx],
+                line.a,
+                line.b,
+                line.mode as usize + 1,
+                &app.input_state.hmd,
+            );
+        }
 
         app.hid_provider.inner.commit();
         let mut futures = GpuFutures::default();

wayvr/src/backend/openxr/helpers.rs

@@ -37,6 +37,11 @@ pub(super) fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
     } else {
         log::warn!("Missing EXT_hand_interaction extension.");
     }
+    if available_extensions.ext_eye_gaze_interaction {
+        enabled_extensions.ext_eye_gaze_interaction = true;
+    } else {
+        log::warn!("Missing EXT_eye_gaze_interaction extension.");
+    }
     if available_extensions.khr_composition_layer_cylinder {
         enabled_extensions.khr_composition_layer_cylinder = true;
     } else {

wayvr/src/backend/openxr/input.rs

@@ -8,7 +8,7 @@ use glam::{Affine3A, Quat, Vec3, bool};
 use libmonado as mnd;
 use openxr::{self as xr, Quaternionf, Vector2f, Vector3f};
 use serde::{Deserialize, Serialize};
-use wlx_common::config_io;
+use wlx_common::{config::HandsfreePointer, config_io};
 
 use crate::{
     backend::input::{Haptics, InputState, Pointer, TrackedDevice, TrackedDeviceRole},
@@ -25,10 +25,11 @@ static CLICK_TIMES: [Duration; 3] = [
 
 pub(super) struct OpenXrInputSource {
     action_set: xr::ActionSet,
-    hands: [OpenXrHand; 2],
+    pointers: [OpenXrPointer; 2],
+    handsfree_pointer: OpenXrPointer,
 }
 
-pub(super) struct OpenXrHand {
+pub(super) struct OpenXrPointer {
     source: OpenXrHandSource,
     space: xr::Space,
 }
@@ -170,22 +171,27 @@ impl OpenXrInputSource {
 
         let left_source = OpenXrHandSource::new(&mut action_set, "left")?;
         let right_source = OpenXrHandSource::new(&mut action_set, "right")?;
+        let fallback_source = OpenXrHandSource::new(&mut action_set, "handsfree")?;
 
-        suggest_bindings(&xr.instance, &[&left_source, &right_source]);
+        suggest_bindings(
+            &xr.instance,
+            &[&left_source, &right_source, &fallback_source],
+        );
 
         xr.session.attach_action_sets(&[&action_set])?;
 
         Ok(Self {
             action_set,
-            hands: [
+            pointers: [
-                OpenXrHand::new(xr, left_source)?,
+                OpenXrPointer::new(xr, left_source)?,
-                OpenXrHand::new(xr, right_source)?,
+                OpenXrPointer::new(xr, right_source)?,
             ],
+            handsfree_pointer: OpenXrPointer::new(xr, fallback_source)?,
         })
     }
 
     pub fn haptics(&self, xr: &XrState, hand: usize, haptics: &Haptics) {
-        let action = &self.hands[hand].source.haptics;
+        let action = &self.pointers[hand].source.haptics;
 
         let duration_nanos = f64::from(haptics.duration) * 1_000_000_000.0;
 
@@ -204,11 +210,14 @@ impl OpenXrInputSource {
 
         let loc = xr.view.locate(&xr.stage, xr.predicted_display_time)?;
         let hmd = posef_to_transform(&loc.pose);
+        let mut hmd_tracked = true;
         if loc
             .location_flags
             .contains(xr::SpaceLocationFlags::ORIENTATION_VALID)
         {
             state.input_state.hmd.matrix3 = hmd.matrix3;
+        } else {
+            hmd_tracked = false;
         }
 
         if loc
@@ -216,11 +225,26 @@ impl OpenXrInputSource {
             .contains(xr::SpaceLocationFlags::POSITION_VALID)
         {
             state.input_state.hmd.translation = hmd.translation;
+        } else {
+            hmd_tracked = false;
         }
 
+        let mut any_tracked = false;
         for i in 0..2 {
-            self.hands[i].update(&mut state.input_state.pointers[i], xr, &state.session)?;
+            let pointer = &mut state.input_state.pointers[i];
+            self.pointers[i].update(pointer, xr, &state.session)?;
+            any_tracked |= pointer.tracked;
         }
+        if !any_tracked {
+            self.handsfree_pointer.update_handsfree(
+                &mut state.input_state.pointers[0],
+                xr,
+                &state.session,
+                hmd,
+                hmd_tracked,
+            )?;
+        }
+
         Ok(())
     }
 
@@ -305,7 +329,7 @@ impl OpenXrInputSource {
     }
 }
 
-impl OpenXrHand {
+impl OpenXrPointer {
     pub(super) fn new(xr: &XrState, source: OpenXrHandSource) -> Result<Self, xr::sys::Result> {
         let space = source
             .pose
@@ -314,11 +338,62 @@ impl OpenXrHand {
         Ok(Self { source, space })
     }
 
+    pub(super) fn update_handsfree(
+        &mut self,
+        pointer: &mut Pointer,
+        xr: &XrState,
+        session: &AppSession,
+        hmd: Affine3A,
+        hmd_tracked: bool,
+    ) -> anyhow::Result<()> {
+        match session.config.handsfree_pointer {
+            HandsfreePointer::None => return Ok(()),
+            HandsfreePointer::Hmd => {
+                pointer.tracked = hmd_tracked;
+                pointer.raw_pose = hmd;
+                pointer.pose = hmd;
+                let (cur_quat, cur_pos) =
+                    (Quat::from_affine3(&pointer.pose), pointer.pose.translation);
+
+                let (new_quat, new_pos) = (Quat::from_affine3(&hmd), Vec3::from(hmd.translation));
+                let lerp_factor =
+                    (1.0 / (xr.fps / 100.0) * session.config.pointer_lerp_factor).clamp(0.1, 1.0);
+                pointer.raw_pose = Affine3A::from_rotation_translation(new_quat, new_pos);
+                pointer.pose = Affine3A::from_rotation_translation(
+                    cur_quat.lerp(new_quat, lerp_factor),
+                    cur_pos.lerp(new_pos.into(), lerp_factor).into(),
+                );
+            }
+            HandsfreePointer::EyeTracking => {
+                // more aggressive smoothing for eye
+                self.pointer_load_pose(pointer, xr, session.config.pointer_lerp_factor * 0.5)?;
+            }
+        }
+
+        pointer.handsfree = pointer.tracked;
+        self.pointer_load_actions(pointer, xr, session)?;
+
+        Ok(())
+    }
+
     pub(super) fn update(
         &mut self,
         pointer: &mut Pointer,
         xr: &XrState,
         session: &AppSession,
+    ) -> anyhow::Result<()> {
+        pointer.handsfree = false;
+        self.pointer_load_pose(pointer, xr, session.config.pointer_lerp_factor)?;
+        self.pointer_load_actions(pointer, xr, session)?;
+
+        Ok(())
+    }
+
+    fn pointer_load_pose(
+        &mut self,
+        pointer: &mut Pointer,
+        xr: &XrState,
+        lerp_factor: f32,
     ) -> anyhow::Result<()> {
         let location = self.space.locate(&xr.stage, xr.predicted_display_time)?;
         if location
@@ -333,15 +408,25 @@ impl OpenXrHand {
                     transmute::<Vector3f, Vec3>(location.pose.position),
                 )
             };
-            let lerp_factor =
+            let lerp_factor = (1.0 / (xr.fps / 100.0) * lerp_factor).clamp(0.1, 1.0);
-                (1.0 / (xr.fps / 100.0) * session.config.pointer_lerp_factor).clamp(0.1, 1.0);
             pointer.raw_pose = Affine3A::from_rotation_translation(new_quat, new_pos);
             pointer.pose = Affine3A::from_rotation_translation(
                 cur_quat.lerp(new_quat, lerp_factor),
                 cur_pos.lerp(new_pos.into(), lerp_factor).into(),
             );
+            pointer.tracked = true;
+        } else {
+            pointer.tracked = false;
+        }
+        Ok(())
     }
 
+    fn pointer_load_actions(
+        &mut self,
+        pointer: &mut Pointer,
+        xr: &XrState,
+        session: &AppSession,
+    ) -> anyhow::Result<()> {
         pointer.now.click = self.source.click.state(pointer.before.click, xr, session)?;
 
         pointer.now.grab = self.source.grab.state(pointer.before.grab, xr, session)?;
@@ -464,11 +549,12 @@ fn is_bool(path_str: &str) -> bool {
 macro_rules! add_custom {
     ($action:expr, $field:ident, $hands:expr, $bindings:expr, $instance:expr) => {
         if let Some(action) = $action.as_ref() {
-            for i in 0..2 {
+            for i in 0..3 {
-                let spec = if i == 0 {
+                let spec = match i {
-                    action.left.as_ref()
+                    0 => action.left.as_ref(),
-                } else {
+                    1 => action.right.as_ref(),
-                    action.right.as_ref()
+                    2 => action.handsfree.as_ref(),
+                    _ => unreachable!(),
                 };
 
                 if let Some(spec) = spec {
@@ -526,11 +612,12 @@ macro_rules! add_custom {
 macro_rules! add_custom_lr {
     ($action:expr, $field:ident, $hands:expr, $bindings:expr, $instance:expr) => {
         if let Some(action) = $action {
-            for i in 0..2 {
+            for i in 0..3 {
-                let spec = if i == 0 {
+                let spec = match i {
-                    action.left.as_ref()
+                    0 => action.left.as_ref(),
-                } else {
+                    1 => action.right.as_ref(),
-                    action.right.as_ref()
+                    2 => action.handsfree.as_ref(),
+                    _ => unreachable!(),
                 };
 
                 if let Some(spec) = spec {
@@ -551,12 +638,14 @@ macro_rules! add_custom_lr {
 }
 
 #[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
-fn suggest_bindings(instance: &xr::Instance, hands: &[&OpenXrHandSource; 2]) {
+fn suggest_bindings(instance: &xr::Instance, hands: &[&OpenXrHandSource; 3]) {
     let profiles = load_action_profiles();
 
     for profile in profiles {
+        log::warn!("Loading profile {}", &profile.profile);
+
         let Ok(profile_path) = instance.string_to_path(&profile.profile) else {
-            log::debug!("Profile not supported: {}", profile.profile);
+            log::warn!("Profile not supported: {}", profile.profile);
             continue;
         };
 
@@ -618,6 +707,11 @@ fn suggest_bindings(instance: &xr::Instance, hands: &[&OpenXrHandSource; 2]) {
         {
             log::error!("Bad bindings for {}", &profile.profile[22..]);
             log::error!("Verify config: ~/.config/wayvr/openxr_actions.json5");
+        } else {
+            log::debug!(
+                "Bindings for {} bound successfully.",
+                &profile.profile[22..]
+            )
         }
     }
 }
@@ -633,6 +727,7 @@ enum OneOrMany<T> {
 struct OpenXrActionConfAction {
     left: Option<OneOrMany<String>>,
     right: Option<OneOrMany<String>>,
+    handsfree: Option<OneOrMany<String>>,
     threshold: Option<[f32; 2]>,
     double_click: Option<bool>,
     triple_click: Option<bool>,
@@ -672,6 +767,8 @@ fn load_action_profiles() -> Vec<OpenXrActionConfProfile> {
             for new in override_profiles {
                 if let Some(i) = profiles.iter().position(|old| old.profile == new.profile) {
                     profiles[i] = new;
+                } else {
+                    profiles.push(new);
                 }
             }
         }

wayvr/src/backend/openxr/lines.rs

@@ -1,4 +1,4 @@
-use glam::{Affine3A, Vec3, Vec3A};
+use glam::{Affine3A, Quat, Vec3, Vec3A};
 use idmap::IdMap;
 use openxr as xr;
 use smallvec::SmallVec;
@@ -96,14 +96,17 @@ impl LinePool {
         Ok(id)
     }
 
-    pub(super) fn draw_from(
+    pub(super) fn draw_between(
         &mut self,
         id: usize,
-        mut from: Affine3A,
+        start: Vec3A,
-        len: f32,
+        end: Vec3A,
         color: usize,
-        hmd: &Affine3A,
+        hmd: Affine3A,
     ) {
+        let dir = end - start;
+        let len = dir.length();
+
         if len < 0.01 {
             return;
         }
@@ -115,13 +118,19 @@ impl LinePool {
             return;
         };
 
+        let center = (start + end) * 0.5;
+        let dir_norm = dir / len;
+
+        let xform = Affine3A::from_rotation_translation(
+            Quat::from_rotation_arc(Vec3::Z, dir_norm.into()),
+            center.into(),
+        );
+
         let rotation = Affine3A::from_axis_angle(Vec3::X, PI * 1.5);
-
+        let mut transform = xform * rotation;
-        from.translation += from.transform_vector3a(Vec3A::NEG_Z) * (len * 0.5);
+        let to_hmd = hmd.translation - center;
-        let mut transform = from * rotation;
-
-        let to_hmd = hmd.translation - from.translation;
         let sides = [Vec3A::Z, Vec3A::X, Vec3A::NEG_Z, Vec3A::NEG_X];
+
         #[allow(clippy::neg_multiply)]
         let rotations = [
             Affine3A::IDENTITY,
@@ -129,6 +138,7 @@ impl LinePool {
             Affine3A::from_axis_angle(Vec3::Y, PI * -1.0),
             Affine3A::from_axis_angle(Vec3::Y, PI * 1.5),
         ];
+
         let mut closest = (0, 0.0);
         for (i, &side) in sides.iter().enumerate() {
             let dot = to_hmd.dot(transform.transform_vector3a(side));

wayvr/src/backend/openxr/mod.rs

@@ -90,6 +90,7 @@ pub fn openxr_run(show_by_default: bool, headless: bool) -> Result<(), BackendEr
 
     let mut overlays = OverlayWindowManager::<OpenXrOverlayData>::new(&mut app, headless)?;
     let mut lines = LinePool::new(&app)?;
+    let mut current_lines = Vec::with_capacity(2);
 
     let mut notifications = NotificationManager::new();
     notifications.run_dbus(&mut app.dbus);
@@ -327,19 +328,23 @@ pub fn openxr_run(show_by_default: bool, headless: bool) -> Result<(), BackendEr
             .values_mut()
             .for_each(|o| o.config.auto_movement(&mut app));
 
-        let lengths_haptics = interact(&mut overlays, &mut app);
+        current_lines.clear();
-        for (idx, (len, haptics)) in lengths_haptics.iter().enumerate() {
+
-            lines.draw_from(
+        let haptics = interact(&mut overlays, &mut app, &mut current_lines);
-                pointer_lines[idx],
+        for (idx, haptics) in haptics.iter().enumerate() {
-                app.input_state.pointers[idx].pose,
-                *len,
-                app.input_state.pointers[idx].interaction.mode as usize + 1,
-                &app.input_state.hmd,
-            );
             if let Some(haptics) = haptics {
                 input_source.haptics(&xr_state, idx, haptics);
             }
         }
+        for (idx, line) in current_lines.iter().enumerate() {
+            lines.draw_between(
+                pointer_lines[idx],
+                line.a,
+                line.b,
+                line.mode as usize + 1,
+                app.input_state.hmd,
+            );
+        }
 
         app.hid_provider.inner.commit();
 

wayvr/src/backend/openxr/openxr_actions.json5

@@ -34,22 +34,21 @@
 // do not mess with these, unless you know what you're doing
 
 [
-  // Hand tracking
+  {
-  // {
+    profile: "/interaction_profiles/ext/eye_gaze_interaction",
-  //   profile: "/interaction_profiles/ext/hand_interaction_ext",
+    pose: {
-  //   pose: {
+      fallback: "/user/eyes_ext/input/gaze_ext/pose",
-  //     left: "/user/hand/left/input/aim/pose",
+    },
-  //     right: "/user/hand/right/input/aim/pose"
+  },
-  //   },
+  {
-  //   click: {
+    profile: "/interaction_profiles/ext/hand_interaction_ext",
-  //     left: "/user/hand/left/input/pinch_ext/value",
+    click: {
-  //     right: "/user/hand/right/input/pinch_ext/value"
+      fallback: "/user/hand/right/input/pinch_ext/value"
-  //   },
+    },
-  //   grab: {
+    grab: {
-  //     left: "/user/hand/left/input/grasp_ext/value",
+      fallback: "/user/hand/left/input/pinch_ext/value",
-  //     right: "/user/hand/right/input/grasp_ext/value"
+    },
-  //   },
+  },
-  // },
 
   // Fallback controller, intended for testing
   {

wayvr/src/backend/task.rs

@@ -66,6 +66,8 @@ pub enum ModifyPanelCommand {
     SetColor(String),
     SetImage(String),
     SetVisible(bool),
+    GetValue,
+    SetValue(String),
     SetStickyState(bool),
 }
 

wayvr/src/backend/wayvr/comp.rs

@@ -36,7 +36,6 @@ use std::fs::File;
 use std::io::Write;
 use std::os::fd::OwnedFd;
 use std::sync::{Arc, Mutex};
-use wayland_client::WEnum;
 
 use smithay::utils::Serial;
 use smithay::wayland::compositor::{self, BufferAssignment, SurfaceAttributes, send_surface_state};

wayvr/src/config.rs

@@ -6,7 +6,8 @@ use wayvr_ipc::packet_client::WvrProcessLaunchParams;
 use wlx_common::{
     astr_containers::AStrMap,
     config::{
-        AltModifier, CaptureMethod, GeneralConfig, SerializedWindowSet, SerializedWindowStates,
+        AltModifier, CaptureMethod, GeneralConfig, HandsfreePointer, SerializedWindowSet,
+        SerializedWindowStates,
     },
     config_io,
     overlays::BackendAttribValue,
@@ -139,6 +140,7 @@ pub struct AutoSettings {
     pub capture_method: CaptureMethod,
     pub keyboard_middle_click_mode: AltModifier,
     pub autostart_apps: Vec<WvrProcessLaunchParams>,
+    pub handsfree_pointer: HandsfreePointer,
 }
 
 fn get_settings_path() -> PathBuf {
@@ -185,6 +187,7 @@ pub fn save_settings(config: &GeneralConfig) -> anyhow::Result<()> {
         capture_method: config.capture_method,
         keyboard_middle_click_mode: config.keyboard_middle_click_mode,
         autostart_apps: config.autostart_apps.clone(),
+        handsfree_pointer: config.handsfree_pointer,
     };
 
     let json = serde_json::to_string_pretty(&conf).unwrap(); // want panic

wayvr/src/gui/panel/button.rs

@@ -22,7 +22,7 @@ use wgui::{
     widget::EventResult,
     windowing::context_menu::{Blueprint, ContextMenu, OpenParams},
 };
-use wlx_common::overlays::ToastTopic;
+use wlx_common::{config::HandsfreePointer, overlays::ToastTopic};
 
 use crate::{
     RESTART, RUNNING,
@@ -600,6 +600,26 @@ pub(super) fn setup_custom_button<S: 'static>(
 
                 Ok(EventResult::Consumed)
             }),
+            "::HandsfreeMode" => {
+                let Some(arg) = args.next() else {
+                    log_cmd_missing_arg(parser_state, TAG, name, command);
+                    return;
+                };
+
+                let Ok(val) = HandsfreePointer::from_str(arg) else {
+                    let msg = format!("expected HandsfreePointer, found \"{arg}\"");
+                    log_cmd_invalid_arg(parser_state, TAG, name, command, &msg);
+                    return;
+                };
+
+                Box::new(move |_common, data, app, _| {
+                    if !test_button(data) || !test_duration(&button, app) {
+                        return Ok(EventResult::Pass);
+                    }
+                    app.session.config.handsfree_pointer = val;
+                    Ok(EventResult::Consumed)
+                })
+            }
             "::SendKey" => {
                 let Some(arg) = args.next() else {
                     log_cmd_missing_arg(parser_state, TAG, name, command);

wayvr/src/gui/panel/mod.rs

@@ -7,7 +7,10 @@ use idmap::IdMap;
 use label::setup_custom_label;
 use wgui::{
     assets::AssetPath,
-    components::button::ComponentButton,
+    components::{
+        button::ComponentButton, checkbox::ComponentCheckbox, radio_group::ComponentRadioGroup,
+        slider::ComponentSlider,
+    },
     drawing,
     event::{
         CallbackDataCommon, Event as WguiEvent, EventAlterables, EventCallback, EventListenerID,
@@ -20,8 +23,7 @@ use wgui::{
     parser::{
         self, CustomAttribsInfoOwned, Fetchable, ParseDocumentExtra, ParserState, parse_color_hex,
     },
-    renderer_vk::context::Context as WguiContext,
+    renderer_vk::{context::Context as WguiContext, text::custom_glyph::CustomGlyphData},
-    renderer_vk::text::custom_glyph::CustomGlyphData,
     taffy,
     widget::{
         EventResult, image::WidgetImage, label::WidgetLabel, rectangle::WidgetRectangle,
@@ -574,6 +576,33 @@ pub fn apply_custom_command<T>(
                 .set_style(wid, wgui::event::StyleSetRequest::Display(display));
             com.alterables.mark_redraw();
         }
+        ModifyPanelCommand::SetValue(value_str) => {
+            if let Ok(cb) = panel
+                .parser_state
+                .fetch_component_as::<ComponentCheckbox>(element)
+            {
+                let value_b = match value_str.to_lowercase().as_str() {
+                    "0" | "false" => false,
+                    "1" | "true" => true,
+                    _ => anyhow::bail!(
+                        "Not a valid bool. Supported values: '0', '1', 'false', 'true'"
+                    ),
+                };
+                cb.set_checked(&mut com, value_b);
+            } else if let Ok(slider) = panel
+                .parser_state
+                .fetch_component_as::<ComponentSlider>(element)
+            {
+                let value_f32 = value_str.parse::<f32>().context("Not a valid number")?;
+                slider.set_value(&mut com, value_f32);
+            } else if let Ok(radio) = panel
+                .parser_state
+                .fetch_component_as::<ComponentRadioGroup>(element)
+            {
+                radio.set_value(&mut com, &value_str)?;
+            }
+        }
+        ModifyPanelCommand::GetValue => todo!(),
         ModifyPanelCommand::SetStickyState(sticky_down) => {
             let button = panel
                 .parser_state

wayvr/src/main.rs

@@ -279,6 +279,7 @@ fn logging_init(args: &mut Args) {
                 .with_default_directive(LevelFilter::INFO.into())
                 .from_env_lossy()
                 .add_directive("symphonia_core::probe=warn".parse().unwrap())
+                .add_directive("symphonia_bundle_mp3=warn".parse().unwrap())
                 .add_directive("zbus=warn".parse().unwrap())
                 .add_directive("usvg=error".parse().unwrap())
                 .add_directive("resvg=error".parse().unwrap())

wgui/src/components/checkbox.rs

@@ -123,6 +123,11 @@ impl ComponentCheckbox {
 		common.alterables.mark_redraw();
 	}
 
+	pub fn get_checked(&self) -> bool {
+		let state = self.state.borrow_mut();
+		state.checked
+	}
+
 	pub fn get_value(&self) -> Option<Rc<str>> {
 		self.data.value.clone()
 	}

wgui/src/components/radio_group.rs

@@ -86,7 +86,7 @@ impl ComponentRadioGroup {
 		self.state.borrow().selected.as_ref().and_then(|b| b.get_value())
 	}
 
-	pub fn set_value(&self, value: &str) -> anyhow::Result<()> {
+	pub fn set_value_simple(&self, value: &str) -> anyhow::Result<()> {
 		let mut state = self.state.borrow_mut();
 		for radio_box in &state.radio_boxes {
 			if radio_box.get_value().is_some_and(|box_val| &*box_val == value) {
@@ -97,6 +97,26 @@ impl ComponentRadioGroup {
 		anyhow::bail!("No RadioBox found with value '{value}'")
 	}
 
+	pub fn set_value(&self, common: &mut CallbackDataCommon, value: &str) -> anyhow::Result<()> {
+		let mut state = self.state.borrow_mut();
+		let mut selected = None;
+		for radio_box in &state.radio_boxes {
+			if radio_box.get_value().is_some_and(|box_val| &*box_val == value) {
+				selected = Some(radio_box.clone());
+				radio_box.set_checked(common, true);
+			} else {
+				radio_box.set_checked(common, false);
+			}
+		}
+
+		if selected.is_some() {
+			state.selected = selected;
+			Ok(())
+		} else {
+			anyhow::bail!("No RadioBox found with value '{value}'")
+		}
+	}
+
 	pub fn on_value_changed(&self, callback: RadioValueChangeCallback) {
 		self.state.borrow_mut().on_value_changed = Some(callback);
 	}

wlx-common/src/config.rs

@@ -41,6 +41,7 @@ pub enum CaptureMethod {
 #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, AsRefStr, EnumString, EnumProperty, VariantArray)]
 pub enum AltModifier {
 	#[default]
+	#[strum(props(Translation = "APP_SETTINGS.OPTION.NONE"))]
 	None,
 	Shift,
 	Ctrl,
@@ -49,6 +50,17 @@ pub enum AltModifier {
 	Meta,
 }
 
+#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, AsRefStr, EnumString, EnumProperty, VariantArray)]
+pub enum HandsfreePointer {
+	#[strum(props(Translation = "APP_SETTINGS.OPTION.NONE"))]
+	None,
+	#[strum(props(Translation = "APP_SETTINGS.OPTION.HMD_PINCH"))]
+	#[default]
+	Hmd,
+	#[strum(props(Translation = "APP_SETTINGS.OPTION.EYE_PINCH"))]
+	EyeTracking,
+}
+
 #[derive(Clone, Serialize, Deserialize)]
 pub struct SerializedWindowSet {
 	pub name: Arc<str>,
@@ -295,4 +307,7 @@ pub struct GeneralConfig {
 
 	#[serde(default)]
 	pub keyboard_middle_click_mode: AltModifier,
+
+	#[serde(default)]
+	pub handsfree_pointer: HandsfreePointer,
 }