Jayden/wayvr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

panel: per-element interactibility

Browse Source
This commit is contained in:
galister
2025-10-31 17:32:10 +09:00
parent 01d11e8485
commit fa562f7b12
19 changed files with 423 additions and 361 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
wlx-overlay-s/src/assets/gui/watch.xml
View File

@@ -35,7 +35,7 @@
</template>
<elements>
<div width="400" height="200">
<div width="460" height="260" padding="30" interactable="0">
<rectangle width="100%" height="100%" padding="4" box_sizing="content_box" flex_wrap="wrap" flex_direction="column" gap="4" color="~bg_color">
<div width="100%" flex_direction="row">
<Device src="watch/hmd.svg" size="40" device="0" />
@@ -64,13 +64,13 @@
</div>
</div>
<div width="100%" flex_direction="row">
<Button macro="button_style" _press="::DashToggle" tooltip="Toggle dashboard" tooltip_side="top">
<Button macro="button_style" _press="::DashToggle" tooltip="Dashboard" tooltip_side="top">
<sprite color="~set_color" width="40" height="40" src="watch/home.svg" />
</Button>
<div id="sets">
<!-- Will populate <Set> tags at runtime -->
</div>
<Button macro="button_style" _press="::EditToggle" tooltip="Edit sets" tooltip_side="top">
<Button macro="button_style" _press="::EditToggle" tooltip="Edit mode" tooltip_side="top">
<sprite color="~set_color" width="40" height="40" src="watch/edit.svg" />
</Button>
</div>

298
wlx-overlay-s/src/backend/input.rs
View File

@@ -15,6 +15,13 @@ use crate::windowing::{OverlayID, OverlaySelector};
use super::task::{TaskContainer, TaskType};
#[derive(Clone, Default)]
pub struct HoverResult {
pub haptics: Option<Haptics>,
/// If true, the laster shows at this position and no further raycasting will be done.
pub consume: bool,
}
pub struct TrackedDevice {
pub soc: Option<f32>,
pub charging: bool,
@@ -307,7 +314,6 @@ where
}
}
#[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
fn interact_hand<O>(
idx: usize,
overlays: &mut OverlayWindowManager<O>,
@@ -316,6 +322,7 @@ fn interact_hand<O>(
where
O: Default,
{
// already grabbing, ignore everything else
let mut pointer = &mut app.input_state.pointers[idx];
if let Some(grab_data) = pointer.interaction.grabbed {
if let Some(grabbed) = overlays.mut_by_id(grab_data.grabbed_id) {
@@ -327,45 +334,29 @@ where
return (0.1, None);
}
let Some(mut hit) = pointer.get_nearest_hit(overlays) else {
if let Some(hovered_id) = pointer.interaction.hovered_id.take() {
if let Some(hovered) = overlays.mut_by_id(hovered_id) {
hovered.config.backend.on_left(app, idx);
}
pointer = &mut app.input_state.pointers[idx];
pointer.interaction.hovered_id = None;
}
if !pointer.now.click
&& pointer.before.click
&& let Some(clicked_id) = pointer.interaction.clicked_id.take()
&& let Some(clicked) = overlays.mut_by_id(clicked_id)
{
let hit = PointerHit {
pointer: pointer.idx,
overlay: clicked_id,
mode: pointer.interaction.mode,
..Default::default()
};
clicked.config.backend.on_pointer(app, &hit, false);
}
let hovered_id = pointer.interaction.hovered_id.take();
let (Some(mut hit), haptics) = get_nearest_hit(idx, overlays, app) else {
handle_no_hit(idx, hovered_id, overlays, app);
return (0.0, None); // no hit
};
if let Some(hovered_id) = pointer.interaction.hovered_id
// focus change
if let Some(hovered_id) = hovered_id
&& hovered_id != hit.overlay
&& let Some(old_hovered) = overlays.mut_by_id(hovered_id)
{
if Some(pointer.idx) == old_hovered.primary_pointer {
if old_hovered.primary_pointer.is_some_and(|i| i == idx) {
old_hovered.primary_pointer = None;
}
log::debug!("{} on_left (focus changed)", old_hovered.config.name);
old_hovered.config.backend.on_left(app, idx);
pointer = &mut app.input_state.pointers[idx];
}
let Some(hovered) = overlays.mut_by_id(hit.overlay) else {
log::warn!("Hit overlay {:?} does not exist", hit.overlay);
return (0.0, None); // no hit
};
pointer = &mut app.input_state.pointers[idx];
pointer.interaction.hovered_id = Some(hit.overlay);
if let Some(primary_pointer) = hovered.primary_pointer {
@@ -383,6 +374,7 @@ where
let hovered_state = hovered.config.active_state.as_mut().unwrap();
// grab
if pointer.now.grab && !pointer.before.grab && hovered_state.grabbable {
update_focus(
&mut app.hid_provider.keyboard_focus,
@@ -400,16 +392,68 @@ where
);
}
// Pass mouse motion events only if not scrolling
// (allows scrolling on all Chromium-based applications)
let haptics = hovered.config.backend.on_hover(app, &hit);
handle_scroll(&hit, hovered, app);
pointer = &mut app.input_state.pointers[idx];
// click / release
let pointer = &mut app.input_state.pointers[hit.pointer];
if pointer.now.click && !pointer.before.click {
pointer.interaction.clicked_id = Some(hit.overlay);
update_focus(
&mut app.hid_provider.keyboard_focus,
&hovered.config.keyboard_focus,
);
hovered.config.backend.on_pointer(app, &hit, true);
} else if !pointer.now.click && pointer.before.click {
// send release event to overlay that was originally clicked
if let Some(clicked_id) = pointer.interaction.clicked_id.take() {
if let Some(clicked) = overlays.mut_by_id(clicked_id) {
clicked.config.backend.on_pointer(app, &hit, false);
}
} else {
hovered.config.backend.on_pointer(app, &hit, false);
}
}
(hit.dist, haptics)
}
fn handle_no_hit<O>(
pointer_idx: usize,
hovered_id: Option<OverlayID>,
overlays: &mut OverlayWindowManager<O>,
app: &mut AppState,
) {
if let Some(hovered_id) = hovered_id
&& let Some(hovered) = overlays.mut_by_id(hovered_id)
{
log::debug!("{} on_left (no hit)", hovered.config.name);
hovered.config.backend.on_left(app, pointer_idx);
}
// in case click released while not aiming at anything
// send release event to overlay that was originally clicked
let pointer = &mut app.input_state.pointers[pointer_idx];
if !pointer.now.click
&& pointer.before.click
&& let Some(clicked_id) = pointer.interaction.clicked_id.take()
&& let Some(clicked) = overlays.mut_by_id(clicked_id)
{
let hit = PointerHit {
pointer: pointer.idx,
overlay: clicked_id,
mode: pointer.interaction.mode,
..Default::default()
};
clicked.config.backend.on_pointer(app, &hit, false);
}
}
fn handle_scroll<O>(hit: &PointerHit, hovered: &mut OverlayWindowData<O>, app: &mut AppState) {
let pointer = &mut app.input_state.pointers[hit.pointer];
if pointer.now.scroll_x.abs() > 0.1 || pointer.now.scroll_y.abs() > 0.1 {
let scroll_x = pointer.now.scroll_x;
let scroll_y = pointer.now.scroll_y;
if app.input_state.pointers[1 - idx]
if app.input_state.pointers[1 - hit.pointer]
.interaction
.grabbed
.is_some_and(|x| x.grabbed_id == hit.overlay)
@@ -437,87 +481,81 @@ where
.backend
.on_scroll(app, &hit, scroll_y, scroll_x);
}
pointer = &mut app.input_state.pointers[idx];
}
if pointer.now.click && !pointer.before.click {
pointer.interaction.clicked_id = Some(hit.overlay);
update_focus(
&mut app.hid_provider.keyboard_focus,
&hovered.config.keyboard_focus,
);
hovered.config.backend.on_pointer(app, &hit, true);
} else if !pointer.now.click && pointer.before.click {
if let Some(clicked_id) = pointer.interaction.clicked_id.take() {
if let Some(clicked) = overlays.mut_by_id(clicked_id) {
clicked.config.backend.on_pointer(app, &hit, false);
}
} else {
hovered.config.backend.on_pointer(app, &hit, false);
}
}
(hit.dist, haptics)
}
impl Pointer {
fn get_nearest_hit<O>(&mut self, overlays: &mut OverlayWindowManager<O>) -> Option<PointerHit>
where
O: Default,
{
let mut hits: SmallVec<[RayHit; 8]> = smallvec!();
fn get_nearest_hit<O>(
pointer_idx: usize,
overlays: &mut OverlayWindowManager<O>,
app: &mut AppState,
) -> (Option<PointerHit>, Option<Haptics>)
where
O: Default,
{
let pointer = &mut app.input_state.pointers[pointer_idx];
let ray_origin = pointer.pose;
let mode = pointer.interaction.mode;
for (id, overlay) in overlays.iter() {
let Some(overlay_state) = overlay.config.active_state.as_ref() else {
continue;
};
if !overlay_state.interactable {
continue;
}
let mut hits: SmallVec<[RayHit; 8]> = smallvec!();
if let Some(hit) = self.ray_test(
id,
&overlay_state.transform,
overlay_state.curvature.as_ref(),
) {
if hit.dist.is_infinite() || hit.dist.is_nan() {
continue;
}
for (id, overlay) in overlays.iter() {
let Some(overlay_state) = overlay.config.active_state.as_ref() else {
continue;
};
if !overlay_state.interactable {
continue;
}
if let Some(hit) = ray_test(
&ray_origin,
id,
&overlay_state.transform,
overlay_state.curvature.as_ref(),
) {
if hit.dist.is_finite() {
hits.push(hit);
}
}
hits.sort_by(|a, b| a.dist.total_cmp(&b.dist));
for hit in &hits {
let overlay = overlays.mut_by_id(hit.overlay).unwrap(); // safe because we just got the id from the overlay
let Some(uv) = overlay
.config
.backend
.as_mut()
.get_interaction_transform()
.map(|a| a.transform_point2(hit.local_pos))
else {
continue;
};
if uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0 {
continue;
}
return Some(PointerHit {
pointer: self.idx,
overlay: hit.overlay,
mode: self.interaction.mode,
primary: false,
uv,
dist: hit.dist,
});
}
None
}
hits.sort_by(|a, b| a.dist.total_cmp(&b.dist));
for hit in &hits {
let overlay = overlays.mut_by_id(hit.overlay).unwrap(); // safe because we just got the id from the overlay
let Some(uv) = overlay
.config
.backend
.as_mut()
.get_interaction_transform()
.map(|a| a.transform_point2(hit.local_pos))
else {
continue;
};
if uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0 {
continue;
}
let hit = PointerHit {
pointer: pointer_idx,
overlay: hit.overlay,
mode,
primary: false,
uv,
dist: hit.dist,
};
let result = overlay.config.backend.on_hover(app, &hit);
if result.consume {
return (Some(hit), result.haptics);
}
}
(None, None)
}
impl Pointer {
fn start_grab(
&mut self,
id: OverlayID,
@@ -608,37 +646,37 @@ impl Pointer {
log::debug!("Hand {}: dropped {}", idx, overlay.config.name);
}
}
}
fn ray_test(
&self,
overlay: OverlayID,
transform: &Affine3A,
curvature: Option<&f32>,
) -> Option<RayHit> {
let (dist, local_pos) = curvature.map_or_else(
|| {
Some(raycast_plane(
&self.pose,
Vec3A::NEG_Z,
transform,
Vec3A::NEG_Z,
))
},
|curvature| raycast_cylinder(&self.pose, Vec3A::NEG_Z, transform, *curvature),
)?;
fn ray_test(
ray_origin: &Affine3A,
overlay: OverlayID,
overlay_pose: &Affine3A,
curvature: Option<&f32>,
) -> Option<RayHit> {
let (dist, local_pos) = curvature.map_or_else(
|| {
Some(raycast_plane(
&ray_origin,
Vec3A::NEG_Z,
overlay_pose,
Vec3A::NEG_Z,
))
},
|curvature| raycast_cylinder(ray_origin, Vec3A::NEG_Z, overlay_pose, *curvature),
)?;
if dist < 0.0 {
// hit is behind us
return None;
}
Some(RayHit {
overlay,
global_pos: self.pose.transform_point3a(Vec3A::NEG_Z * dist),
local_pos,
dist,
})
if dist < 0.0 {
// hit is behind us
return None;
}
Some(RayHit {
overlay,
global_pos: ray_origin.transform_point3a(Vec3A::NEG_Z * dist),
local_pos,
dist,
})
}
fn raycast_plane(

6
wlx-overlay-s/src/backend/openvr/lines.rs
View File

@@ -22,7 +22,7 @@ use vulkano::{
};
use wgui::gfx::WGfx;
use crate::backend::input::{Haptics, PointerHit};
use crate::backend::input::{HoverResult, PointerHit};
use crate::graphics::CommandBuffers;
use crate::state::AppState;
use crate::windowing::backend::{FrameMeta, OverlayBackend, ShouldRender};
@@ -206,8 +206,8 @@ impl OverlayBackend for LineBackend {
})
}
fn on_hover(&mut self, _: &mut AppState, _: &PointerHit) -> Option<Haptics> {
None
fn on_hover(&mut self, _: &mut AppState, _: &PointerHit) -> HoverResult {
HoverResult::default()
}
fn on_left(&mut self, _: &mut AppState, _: usize) {}
fn on_pointer(&mut self, _: &mut AppState, _: &PointerHit, _: bool) {}

37
wlx-overlay-s/src/gui/panel/mod.rs
View File

@@ -15,11 +15,11 @@ use wgui::{
layout::{Layout, LayoutParams, WidgetID},
parser::ParserState,
renderer_vk::context::Context as WguiContext,
widget::{label::WidgetLabel, rectangle::WidgetRectangle},
widget::{label::WidgetLabel, rectangle::WidgetRectangle, EventResult},
};
use crate::{
backend::input::{Haptics, PointerHit, PointerMode},
backend::input::{Haptics, HoverResult, PointerHit, PointerMode},
graphics::{CommandBuffers, ExtentExt},
state::AppState,
windowing::backend::{ui_transform, FrameMeta, OverlayBackend, ShouldRender},
@@ -150,9 +150,13 @@ impl<S: 'static> GuiPanel<S> {
self.layout.update(MAX_SIZE_VEC2, 0.0)
}
pub fn push_event(&mut self, app: &mut AppState, event: &WguiEvent) {
if let Err(e) = self.layout.push_event(event, app, &mut self.state) {
log::error!("Failed to push event: {e:?}");
pub fn push_event(&mut self, app: &mut AppState, event: &WguiEvent) -> EventResult {
match self.layout.push_event(event, app, &mut self.state) {
Ok(r) => r,
Err(e) => {
log::error!("Failed to push event: {e:?}");
EventResult::NoHit
}
}
}
@@ -266,23 +270,28 @@ impl<S: 'static> OverlayBackend for GuiPanel<S> {
self.push_event(app, &e);
}
fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> Option<Haptics> {
fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> HoverResult {
let e = &WguiEvent::MouseMotion(MouseMotionEvent {
pos: hit.uv * self.layout.content_size,
device: hit.pointer,
});
self.push_event(app, &e);
let result = self.push_event(app, &e);
self.layout
.check_toggle_haptics_triggered()
.then_some(Haptics {
intensity: 0.1,
duration: 0.01,
frequency: 5.0,
})
HoverResult {
consume: result != EventResult::NoHit,
haptics: self
.layout
.check_toggle_haptics_triggered()
.then_some(Haptics {
intensity: 0.1,
duration: 0.01,
frequency: 5.0,
}),
}
}
fn on_left(&mut self, app: &mut AppState, pointer: usize) {
log::info!("panel: on left");
let e = WguiEvent::MouseLeave(MouseLeaveEvent { device: pointer });
self.push_event(app, &e);
}

6
wlx-overlay-s/src/overlays/keyboard/mod.rs
View File

@@ -11,11 +11,11 @@ use wgui::{
};
use crate::{
backend::input::{Haptics, PointerHit},
backend::input::{HoverResult, PointerHit},
graphics::CommandBuffers,
gui::panel::GuiPanel,
state::AppState,
subsystem::hid::{ALT, CTRL, KeyModifier, META, SHIFT, SUPER, VirtualKey},
subsystem::hid::{KeyModifier, VirtualKey, ALT, CTRL, META, SHIFT, SUPER},
windowing::backend::{FrameMeta, OverlayBackend, ShouldRender},
};
@@ -75,7 +75,7 @@ impl OverlayBackend for KeyboardBackend {
fn on_left(&mut self, app: &mut AppState, pointer: usize) {
self.panel.on_left(app, pointer);
}
fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> Option<Haptics> {
fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> HoverResult {
self.panel.on_hover(app, hit)
}
fn get_interaction_transform(&mut self) -> Option<glam::Affine2> {

9
wlx-overlay-s/src/overlays/mirror.rs
View File

@@ -10,7 +10,7 @@ use wlx_capture::pipewire::{pipewire_select_screen, PipewireCapture, PipewireSel
use crate::{
backend::{
input::{Haptics, PointerHit},
input::{HoverResult, PointerHit},
task::TaskType,
},
graphics::CommandBuffers,
@@ -130,8 +130,11 @@ impl OverlayBackend for MirrorBackend {
self.renderer.as_mut().and_then(ScreenBackend::frame_meta)
}
fn on_hover(&mut self, _: &mut AppState, _: &PointerHit) -> Option<Haptics> {
None
fn on_hover(&mut self, _: &mut AppState, _: &PointerHit) -> HoverResult {
HoverResult {
consume: true,
..HoverResult::default()
}
}
fn on_left(&mut self, _: &mut AppState, _: usize) {}
fn on_pointer(&mut self, _: &mut AppState, _: &PointerHit, _: bool) {}

17
wlx-overlay-s/src/overlays/screen/backend.rs
View File

@@ -1,21 +1,21 @@
use std::{
sync::{Arc, LazyLock, atomic::AtomicU64},
sync::{atomic::AtomicU64, Arc, LazyLock},
time::Instant,
};
use glam::{Affine2, Vec2, vec2};
use glam::{vec2, Affine2, Vec2};
use vulkano::image::view::ImageView;
use wlx_capture::{WlxCapture, frame::Transform};
use wlx_capture::{frame::Transform, WlxCapture};
use crate::{
backend::input::{Haptics, PointerHit, PointerMode},
backend::input::{HoverResult, PointerHit, PointerMode},
graphics::{CommandBuffers, ExtentExt},
state::AppState,
subsystem::hid::{MOUSE_LEFT, MOUSE_MIDDLE, MOUSE_RIGHT},
windowing::backend::{FrameMeta, OverlayBackend, ShouldRender},
};
use super::capture::{ScreenPipeline, WlxCaptureIn, WlxCaptureOut, receive_callback};
use super::capture::{receive_callback, ScreenPipeline, WlxCaptureIn, WlxCaptureOut};
const CURSOR_SIZE: f32 = 16. / 1440.;
@@ -211,7 +211,7 @@ impl OverlayBackend for ScreenBackend {
self.meta
}
fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> Option<Haptics> {
fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> HoverResult {
#[cfg(debug_assertions)]
log::trace!("Hover: {:?}", hit.uv);
if can_move()
@@ -222,7 +222,10 @@ impl OverlayBackend for ScreenBackend {
app.hid_provider.inner.mouse_move(pos);
set_next_move(u64::from(app.session.config.mouse_move_interval_ms));
}
None
HoverResult {
consume: true,
..HoverResult::default()
}
}
fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
let btn = match hit.mode {

13
wlx-overlay-s/src/overlays/wayvr.rs
View File

@@ -17,7 +17,7 @@ use wlx_capture::frame::{DmabufFrame, FourCC, FrameFormat, FramePlane};
use crate::{
backend::{
input::{self},
input::{self, HoverResult},
task::TaskType,
wayvr::{
self, display,
@@ -716,11 +716,7 @@ impl OverlayBackend for WayVRBackend {
})
}
fn on_hover(
&mut self,
_app: &mut state::AppState,
hit: &input::PointerHit,
) -> Option<input::Haptics> {
fn on_hover(&mut self, _app: &mut state::AppState, hit: &input::PointerHit) -> HoverResult {
let ctx = self.context.borrow();
let wayvr = &mut ctx.wayvr.borrow_mut();
@@ -737,7 +733,10 @@ impl OverlayBackend for WayVRBackend {
.send_mouse_move(ctx.display, x as u32, y as u32);
}
wayvr.pending_haptics.take()
HoverResult {
haptics: wayvr.pending_haptics.take(),
consume: true,
}
}
fn on_left(&mut self, _app: &mut state::AppState, _pointer: usize) {

4
wlx-overlay-s/src/windowing/backend.rs
View File

@@ -3,7 +3,7 @@ use std::sync::Arc;
use vulkano::{format::Format, image::view::ImageView};
use crate::{
backend::input::{Haptics, PointerHit},
backend::input::{HoverResult, PointerHit},
graphics::CommandBuffers,
state::AppState,
};
@@ -48,7 +48,7 @@ pub trait OverlayBackend {
/// Must be true if should_render was also true on the same frame.
fn frame_meta(&mut self) -> Option<FrameMeta>;
fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> Option<Haptics>;
fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> HoverResult;
fn on_left(&mut self, app: &mut AppState, pointer: usize);
fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool);
fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta_y: f32, delta_x: f32);

2
wlx-overlay-s/src/windowing/manager.rs
View File

@@ -91,7 +91,9 @@ where
Ok(me)
}
}
impl<T> OverlayWindowManager<T> {
pub fn mut_by_selector(
&mut self,
selector: &OverlaySelector,

5
wlx-overlay-s/src/windowing/window.rs
View File

@@ -63,10 +63,7 @@ where
}
}
impl<T> OverlayWindowData<T>
where
T: Default,
{
impl<T> OverlayWindowData<T> {
pub fn init(&mut self, app: &mut AppState) -> anyhow::Result<()> {
//TODO: load state?