new workspace

wlx-overlay-s/src/backend/common.rs

@@ -0,0 +1,370 @@
+use std::sync::{Arc, LazyLock};
+
+#[cfg(feature = "openxr")]
+use openxr as xr;
+
+use glam::{Affine3A, Vec3, Vec3A};
+use idmap::IdMap;
+use serde::Deserialize;
+use thiserror::Error;
+
+use crate::{
+    config::AStrSetExt,
+    hid::{get_keymap_wl, get_keymap_x11},
+    overlays::{
+        anchor::create_anchor,
+        keyboard::{create_keyboard, KEYBOARD_NAME},
+        screen::WlxClientAlias,
+        watch::{create_watch, WATCH_NAME},
+    },
+    state::AppState,
+};
+
+use super::overlay::{OverlayData, OverlayID};
+
+#[derive(Error, Debug)]
+pub enum BackendError {
+    #[error("backend not supported")]
+    NotSupported,
+    #[cfg(feature = "openxr")]
+    #[error("OpenXR Error: {0:?}")]
+    OpenXrError(#[from] xr::sys::Result),
+    #[error("Shutdown")]
+    Shutdown,
+    #[error("Restart")]
+    Restart,
+    #[error("Fatal: {0:?}")]
+    Fatal(#[from] anyhow::Error),
+}
+
+#[cfg(feature = "wayland")]
+fn create_wl_client() -> Option<WlxClientAlias> {
+    wlx_capture::wayland::WlxClient::new()
+}
+
+#[cfg(not(feature = "wayland"))]
+fn create_wl_client() -> Option<WlxClientAlias> {
+    None
+}
+
+pub struct OverlayContainer<T>
+where
+    T: Default,
+{
+    overlays: IdMap<usize, OverlayData<T>>,
+    wl: Option<WlxClientAlias>,
+}
+
+impl<T> OverlayContainer<T>
+where
+    T: Default,
+{
+    pub fn new(app: &mut AppState, headless: bool) -> anyhow::Result<Self> {
+        let mut overlays = IdMap::new();
+        let mut show_screens = app.session.config.show_screens.clone();
+        let mut wl = None;
+        let mut keymap = None;
+
+        app.screens.clear();
+
+        if headless {
+            log::info!("Running in headless mode; keyboard will be en-US");
+        } else {
+            wl = create_wl_client();
+
+            let data = if let Some(wl) = wl.as_mut() {
+                log::info!("Wayland detected.");
+                keymap = get_keymap_wl()
+                    .map_err(|f| log::warn!("Could not load keyboard layout: {f}"))
+                    .ok();
+                crate::overlays::screen::create_screens_wayland(wl, app)
+            } else {
+                log::info!("Wayland not detected, assuming X11.");
+                keymap = get_keymap_x11()
+                    .map_err(|f| log::warn!("Could not load keyboard layout: {f}"))
+                    .ok();
+                match crate::overlays::screen::create_screens_x11pw(app) {
+                    Ok(data) => data,
+                    Err(e) => {
+                        log::info!("Will not use X11 PipeWire capture: {e:?}");
+                        crate::overlays::screen::create_screens_xshm(app)?
+                    }
+                }
+            };
+
+            if show_screens.is_empty() {
+                if let Some((_, s, _)) = data.screens.first() {
+                    show_screens.arc_set(s.name.clone());
+                }
+            }
+
+            for (meta, mut state, backend) in data.screens {
+                if show_screens.arc_get(state.name.as_ref()) {
+                    state.show_hide = true;
+                }
+                overlays.insert(
+                    state.id.0,
+                    OverlayData::<T> {
+                        state,
+                        backend,
+                        ..Default::default()
+                    },
+                );
+                app.screens.push(meta);
+            }
+        }
+
+        let anchor = create_anchor(app)?;
+        overlays.insert(anchor.state.id.0, anchor);
+
+        let mut watch = create_watch::<T>(app)?;
+        watch.state.want_visible = true;
+        overlays.insert(watch.state.id.0, watch);
+
+        let mut keyboard = create_keyboard(app, keymap)?;
+        keyboard.state.show_hide = show_screens.arc_get(KEYBOARD_NAME);
+        keyboard.state.want_visible = false;
+        overlays.insert(keyboard.state.id.0, keyboard);
+
+        Ok(Self { overlays, wl })
+    }
+
+    #[cfg(not(feature = "wayland"))]
+    pub fn update(&mut self, _app: &mut AppState) -> anyhow::Result<Vec<OverlayData<T>>> {
+        Ok(vec![])
+    }
+    #[cfg(feature = "wayland")]
+    #[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
+    #[allow(clippy::all, clippy::pedantic, clippy::nursery, clippy::cargo)]
+    pub fn update(&mut self, app: &mut AppState) -> anyhow::Result<Vec<OverlayData<T>>> {
+        use crate::overlays::screen::{
+            create_screen_interaction, create_screen_renderer_wl, load_pw_token_config,
+        };
+        use glam::vec2;
+        use wlx_capture::wayland::OutputChangeEvent;
+
+        let mut removed_overlays = vec![];
+        let Some(wl) = self.wl.as_mut() else {
+            return Ok(removed_overlays);
+        };
+
+        wl.dispatch_pending();
+
+        let mut create_ran = false;
+        let mut extent_dirty = false;
+        let mut watch_dirty = false;
+
+        let mut maybe_token_store = None;
+
+        for ev in wl.iter_events().collect::<Vec<_>>() {
+            match ev {
+                OutputChangeEvent::Create(_) => {
+                    if create_ran {
+                        continue;
+                    }
+                    let data = crate::overlays::screen::create_screens_wayland(wl, app);
+                    create_ran = true;
+                    for (meta, state, backend) in data.screens {
+                        self.overlays.insert(
+                            state.id.0,
+                            OverlayData::<T> {
+                                state,
+                                backend,
+                                ..Default::default()
+                            },
+                        );
+                        app.screens.push(meta);
+                        watch_dirty = true;
+                    }
+                }
+                OutputChangeEvent::Destroy(id) => {
+                    let Some(idx) = app.screens.iter().position(|s| s.native_handle == id) else {
+                        continue;
+                    };
+
+                    let meta = &app.screens[idx];
+                    let removed = self.overlays.remove(meta.id.0).unwrap();
+                    removed_overlays.push(removed);
+                    log::info!("{}: Destroyed", meta.name);
+                    app.screens.remove(idx);
+                    watch_dirty = true;
+                    extent_dirty = true;
+                }
+                OutputChangeEvent::Logical(id) => {
+                    let Some(meta) = app.screens.iter().find(|s| s.native_handle == id) else {
+                        continue;
+                    };
+                    let output = wl.outputs.get(id).unwrap();
+                    let Some(overlay) = self.overlays.get_mut(meta.id.0) else {
+                        continue;
+                    };
+                    let logical_pos =
+                        vec2(output.logical_pos.0 as f32, output.logical_pos.1 as f32);
+                    let logical_size =
+                        vec2(output.logical_size.0 as f32, output.logical_size.1 as f32);
+                    let transform = output.transform.into();
+                    overlay
+                        .backend
+                        .set_interaction(Box::new(create_screen_interaction(
+                            logical_pos,
+                            logical_size,
+                            transform,
+                        )));
+                    extent_dirty = true;
+                }
+                OutputChangeEvent::Physical(id) => {
+                    let Some(meta) = app.screens.iter().find(|s| s.native_handle == id) else {
+                        continue;
+                    };
+                    let output = wl.outputs.get(id).unwrap();
+                    let Some(overlay) = self.overlays.get_mut(meta.id.0) else {
+                        continue;
+                    };
+
+                    let has_wlr_dmabuf = wl.maybe_wlr_dmabuf_mgr.is_some();
+                    let has_wlr_screencopy = wl.maybe_wlr_screencopy_mgr.is_some();
+
+                    let pw_token_store = maybe_token_store.get_or_insert_with(|| {
+                        load_pw_token_config().unwrap_or_else(|e| {
+                            log::warn!("Failed to load PipeWire token config: {:?}", e);
+                            Default::default()
+                        })
+                    });
+
+                    if let Some(renderer) = create_screen_renderer_wl(
+                        output,
+                        has_wlr_dmabuf,
+                        has_wlr_screencopy,
+                        pw_token_store,
+                        &app,
+                    ) {
+                        overlay.backend.set_renderer(Box::new(renderer));
+                    }
+                    extent_dirty = true;
+                }
+            }
+        }
+
+        if extent_dirty && !create_ran {
+            let extent = wl.get_desktop_extent();
+            let origin = wl.get_desktop_origin();
+            app.hid_provider
+                .set_desktop_extent(vec2(extent.0 as f32, extent.1 as f32));
+            app.hid_provider
+                .set_desktop_origin(vec2(origin.0 as f32, origin.1 as f32));
+        }
+
+        if watch_dirty {
+            let _watch = self.mut_by_name(WATCH_NAME).unwrap(); // want panic
+            todo!();
+        }
+
+        Ok(removed_overlays)
+    }
+
+    pub fn mut_by_selector(&mut self, selector: &OverlaySelector) -> Option<&mut OverlayData<T>> {
+        match selector {
+            OverlaySelector::Id(id) => self.mut_by_id(*id),
+            OverlaySelector::Name(name) => self.mut_by_name(name),
+        }
+    }
+
+    pub fn remove_by_selector(&mut self, selector: &OverlaySelector) -> Option<OverlayData<T>> {
+        match selector {
+            OverlaySelector::Id(id) => self.overlays.remove(id.0),
+            OverlaySelector::Name(name) => {
+                let id = self
+                    .overlays
+                    .iter()
+                    .find(|(_, o)| *o.state.name == **name)
+                    .map(|(id, _)| *id);
+                id.and_then(|id| self.overlays.remove(id))
+            }
+        }
+    }
+
+    pub fn get_by_id(&mut self, id: OverlayID) -> Option<&OverlayData<T>> {
+        self.overlays.get(id.0)
+    }
+
+    pub fn mut_by_id(&mut self, id: OverlayID) -> Option<&mut OverlayData<T>> {
+        self.overlays.get_mut(id.0)
+    }
+
+    pub fn get_by_name<'a>(&'a mut self, name: &str) -> Option<&'a OverlayData<T>> {
+        self.overlays.values().find(|o| *o.state.name == *name)
+    }
+
+    pub fn mut_by_name<'a>(&'a mut self, name: &str) -> Option<&'a mut OverlayData<T>> {
+        self.overlays.values_mut().find(|o| *o.state.name == *name)
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &'_ OverlayData<T>> {
+        self.overlays.values()
+    }
+
+    pub fn iter_mut(&mut self) -> impl Iterator<Item = &'_ mut OverlayData<T>> {
+        self.overlays.values_mut()
+    }
+
+    pub fn add(&mut self, overlay: OverlayData<T>) {
+        self.overlays.insert(overlay.state.id.0, overlay);
+    }
+
+    pub fn show_hide(&mut self, app: &mut AppState) {
+        let any_shown = self
+            .overlays
+            .values()
+            .any(|o| o.state.show_hide && o.state.want_visible);
+
+        if !any_shown {
+            static ANCHOR_LOCAL: LazyLock<Affine3A> =
+                LazyLock::new(|| Affine3A::from_translation(Vec3::NEG_Z));
+            let hmd = snap_upright(app.input_state.hmd, Vec3A::Y);
+            app.anchor = hmd * *ANCHOR_LOCAL;
+        }
+
+        self.overlays.values_mut().for_each(|o| {
+            if o.state.show_hide {
+                o.state.want_visible = !any_shown;
+                if o.state.want_visible
+                    && app.session.config.realign_on_showhide
+                    && o.state.recenter
+                {
+                    o.state.reset(app, false);
+                }
+            }
+            // toggle watch back on if it was hidden
+            if !any_shown && *o.state.name == *WATCH_NAME {
+                o.state.reset(app, true);
+            }
+        });
+    }
+}
+
+#[derive(Clone, Deserialize, Debug)]
+#[serde(untagged)]
+pub enum OverlaySelector {
+    Id(OverlayID),
+    Name(Arc<str>),
+}
+
+pub fn snap_upright(transform: Affine3A, up_dir: Vec3A) -> Affine3A {
+    if transform.x_axis.dot(up_dir).abs() < 0.2 {
+        let scale = transform.x_axis.length();
+        let col_z = transform.z_axis.normalize();
+        let col_y = up_dir;
+        let col_x = col_y.cross(col_z);
+        let col_y = col_z.cross(col_x).normalize();
+        let col_x = col_x.normalize();
+
+        Affine3A::from_cols(
+            col_x * scale,
+            col_y * scale,
+            col_z * scale,
+            transform.translation,
+        )
+    } else {
+        transform
+    }
+}

wlx-overlay-s/src/backend/input.rs

@@ -0,0 +1,744 @@
+use std::f32::consts::PI;
+use std::process::{Child, Command};
+use std::{collections::VecDeque, time::Instant};
+
+use glam::{Affine3A, Vec2, Vec3, Vec3A, Vec3Swizzles};
+
+use smallvec::{smallvec, SmallVec};
+
+use crate::backend::common::OverlaySelector;
+use crate::backend::overlay::Positioning;
+use crate::config::AStrMapExt;
+use crate::overlays::anchor::ANCHOR_NAME;
+use crate::state::{AppSession, AppState, KeyboardFocus};
+
+use super::overlay::{OverlayID, OverlayState};
+use super::task::{TaskContainer, TaskType};
+use super::{common::OverlayContainer, overlay::OverlayData};
+
+pub struct TrackedDevice {
+    pub soc: Option<f32>,
+    pub charging: bool,
+    pub role: TrackedDeviceRole,
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TrackedDeviceRole {
+    None,
+    Hmd,
+    LeftHand,
+    RightHand,
+    Tracker,
+}
+
+pub struct InputState {
+    pub hmd: Affine3A,
+    pub ipd: f32,
+    pub pointers: [Pointer; 2],
+    pub devices: Vec<TrackedDevice>,
+    processes: Vec<Child>,
+}
+
+impl InputState {
+    pub fn new() -> Self {
+        Self {
+            hmd: Affine3A::IDENTITY,
+            ipd: 0.0,
+            pointers: [Pointer::new(0), Pointer::new(1)],
+            devices: Vec::new(),
+            processes: Vec::new(),
+        }
+    }
+
+    pub const fn pre_update(&mut self) {
+        self.pointers[0].before = self.pointers[0].now;
+        self.pointers[1].before = self.pointers[1].now;
+    }
+
+    pub fn post_update(&mut self, session: &AppSession) {
+        for hand in &mut self.pointers {
+            #[cfg(debug_assertions)]
+            debug_print_hand(hand);
+
+            if hand.now.click {
+                hand.last_click = Instant::now();
+            }
+
+            if hand.now.click_modifier_right {
+                hand.interaction.mode = PointerMode::Right;
+                continue;
+            }
+
+            if hand.now.click_modifier_middle {
+                hand.interaction.mode = PointerMode::Middle;
+                continue;
+            }
+
+            let hmd_up = self.hmd.transform_vector3a(Vec3A::Y);
+            let dot = hmd_up.dot(hand.pose.transform_vector3a(Vec3A::X))
+                * 2.0f32.mul_add(-(hand.idx as f32), 1.0);
+
+            hand.interaction.mode = if dot < -0.85 {
+                PointerMode::Right
+            } else if dot > 0.7 {
+                PointerMode::Middle
+            } else {
+                PointerMode::Left
+            };
+
+            let middle_click_orientation = false;
+            let right_click_orientation = false;
+            match hand.interaction.mode {
+                PointerMode::Middle => {
+                    if !middle_click_orientation {
+                        hand.interaction.mode = PointerMode::Left;
+                    }
+                }
+                PointerMode::Right => {
+                    if !right_click_orientation {
+                        hand.interaction.mode = PointerMode::Left;
+                    }
+                }
+                _ => {}
+            }
+
+            if hand.now.alt_click != hand.before.alt_click {
+                // Reap previous processes
+                self.processes
+                    .retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_))));
+
+                let mut args = if hand.now.alt_click {
+                    session.config.alt_click_down.iter()
+                } else {
+                    session.config.alt_click_up.iter()
+                };
+
+                if let Some(program) = args.next() {
+                    if let Ok(child) = Command::new(program).args(args).spawn() {
+                        self.processes.push(child);
+                    }
+                }
+            }
+        }
+    }
+}
+
+#[cfg(debug_assertions)]
+fn debug_print_hand(hand: &Pointer) {
+    {
+        if hand.now.click != hand.before.click {
+            log::debug!("Hand {}: click {}", hand.idx, hand.now.click);
+        }
+        if hand.now.grab != hand.before.grab {
+            log::debug!("Hand {}: grab {}", hand.idx, hand.now.grab);
+        }
+        if hand.now.alt_click != hand.before.alt_click {
+            log::debug!("Hand {}: alt_click {}", hand.idx, hand.now.alt_click);
+        }
+        if hand.now.show_hide != hand.before.show_hide {
+            log::debug!("Hand {}: show_hide {}", hand.idx, hand.now.show_hide);
+        }
+        if hand.now.toggle_dashboard != hand.before.toggle_dashboard {
+            log::debug!(
+                "Hand {}: toggle_dashboard {}",
+                hand.idx,
+                hand.now.toggle_dashboard
+            );
+        }
+        if hand.now.space_drag != hand.before.space_drag {
+            log::debug!("Hand {}: space_drag {}", hand.idx, hand.now.space_drag);
+        }
+        if hand.now.space_rotate != hand.before.space_rotate {
+            log::debug!("Hand {}: space_rotate {}", hand.idx, hand.now.space_rotate);
+        }
+        if hand.now.space_reset != hand.before.space_reset {
+            log::debug!("Hand {}: space_reset {}", hand.idx, hand.now.space_reset);
+        }
+        if hand.now.click_modifier_right != hand.before.click_modifier_right {
+            log::debug!(
+                "Hand {}: click_modifier_right {}",
+                hand.idx,
+                hand.now.click_modifier_right
+            );
+        }
+        if hand.now.click_modifier_middle != hand.before.click_modifier_middle {
+            log::debug!(
+                "Hand {}: click_modifier_middle {}",
+                hand.idx,
+                hand.now.click_modifier_middle
+            );
+        }
+    }
+}
+
+pub struct InteractionState {
+    pub mode: PointerMode,
+    pub grabbed: Option<GrabData>,
+    pub clicked_id: Option<OverlayID>,
+    pub hovered_id: Option<OverlayID>,
+    pub release_actions: VecDeque<Box<dyn Fn()>>,
+    pub next_push: Instant,
+    pub haptics: Option<f32>,
+}
+
+impl Default for InteractionState {
+    fn default() -> Self {
+        Self {
+            mode: PointerMode::Left,
+            grabbed: None,
+            clicked_id: None,
+            hovered_id: None,
+            release_actions: VecDeque::new(),
+            next_push: Instant::now(),
+            haptics: None,
+        }
+    }
+}
+
+pub struct Pointer {
+    pub idx: usize,
+    pub pose: Affine3A,
+    pub raw_pose: Affine3A,
+    pub now: PointerState,
+    pub before: PointerState,
+    pub last_click: Instant,
+    pub(super) interaction: InteractionState,
+}
+
+impl Pointer {
+    pub fn new(idx: usize) -> Self {
+        debug_assert!(idx == 0 || idx == 1);
+        Self {
+            idx,
+            pose: Affine3A::IDENTITY,
+            raw_pose: Affine3A::IDENTITY,
+            now: PointerState::default(),
+            before: PointerState::default(),
+            last_click: Instant::now(),
+            interaction: InteractionState::default(),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Default)]
+pub struct PointerState {
+    pub scroll_x: f32,
+    pub scroll_y: f32,
+    pub click: bool,
+    pub grab: bool,
+    pub alt_click: bool,
+    pub show_hide: bool,
+    pub toggle_dashboard: bool,
+    pub space_drag: bool,
+    pub space_rotate: bool,
+    pub space_reset: bool,
+    pub click_modifier_right: bool,
+    pub click_modifier_middle: bool,
+    pub move_mouse: bool,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct PointerHit {
+    pub pointer: usize,
+    pub overlay: OverlayID,
+    pub mode: PointerMode,
+    pub primary: bool,
+    pub uv: Vec2,
+    pub dist: f32,
+}
+
+#[derive(Clone)]
+pub struct Haptics {
+    pub intensity: f32,
+    pub duration: f32,
+    pub frequency: f32,
+}
+
+pub trait InteractionHandler {
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> Option<Haptics>;
+    fn on_left(&mut self, app: &mut AppState, pointer: usize);
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool);
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta_y: f32, delta_x: f32);
+}
+
+pub struct DummyInteractionHandler;
+
+impl InteractionHandler for DummyInteractionHandler {
+    fn on_left(&mut self, _app: &mut AppState, _pointer: usize) {}
+    fn on_hover(&mut self, _app: &mut AppState, _hit: &PointerHit) -> Option<Haptics> {
+        None
+    }
+    fn on_pointer(&mut self, _app: &mut AppState, _hit: &PointerHit, _pressed: bool) {}
+    fn on_scroll(&mut self, _app: &mut AppState, _hit: &PointerHit, _delta_y: f32, _delta_x: f32) {}
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+struct RayHit {
+    overlay: OverlayID,
+    global_pos: Vec3A,
+    local_pos: Vec2,
+    dist: f32,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct GrabData {
+    pub offset: Vec3A,
+    pub grabbed_id: OverlayID,
+    pub old_curvature: Option<f32>,
+    pub grab_all: bool,
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, Default)]
+pub enum PointerMode {
+    #[default]
+    Left,
+    Right,
+    Middle,
+    Special,
+}
+
+fn update_focus(focus: &mut KeyboardFocus, state: &OverlayState) {
+    if let Some(f) = &state.keyboard_focus {
+        if *focus != *f {
+            log::info!("Setting keyboard focus to {:?}", *f);
+            *focus = *f;
+        }
+    }
+}
+
+pub fn interact<O>(
+    overlays: &mut OverlayContainer<O>,
+    app: &mut AppState,
+) -> [(f32, Option<Haptics>); 2]
+where
+    O: Default,
+{
+    if app.input_state.pointers[1].last_click > app.input_state.pointers[0].last_click {
+        let right = interact_hand(1, overlays, app);
+        let left = interact_hand(0, overlays, app);
+        [left, right]
+    } else {
+        let left = interact_hand(0, overlays, app);
+        let right = interact_hand(1, overlays, app);
+        [left, right]
+    }
+}
+
+#[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
+fn interact_hand<O>(
+    idx: usize,
+    overlays: &mut OverlayContainer<O>,
+    app: &mut AppState,
+) -> (f32, Option<Haptics>)
+where
+    O: Default,
+{
+    let mut pointer = &mut app.input_state.pointers[idx];
+    if let Some(grab_data) = pointer.interaction.grabbed {
+        if let Some(grabbed) = overlays.mut_by_id(grab_data.grabbed_id) {
+            Pointer::handle_grabbed(idx, grabbed, app);
+        } else {
+            log::warn!("Grabbed overlay {} does not exist", grab_data.grabbed_id.0);
+            pointer.interaction.grabbed = None;
+        }
+        return (0.1, None);
+    }
+
+    let Some(mut hit) = pointer.get_nearest_hit(overlays) else {
+        if let Some(hovered_id) = pointer.interaction.hovered_id.take() {
+            if let Some(hovered) = overlays.mut_by_id(hovered_id) {
+                hovered.backend.on_left(app, idx);
+            }
+            pointer = &mut app.input_state.pointers[idx];
+            pointer.interaction.hovered_id = None;
+        }
+        if !pointer.now.click && pointer.before.click {
+            if let Some(clicked_id) = pointer.interaction.clicked_id.take() {
+                if let Some(clicked) = overlays.mut_by_id(clicked_id) {
+                    let hit = PointerHit {
+                        pointer: pointer.idx,
+                        overlay: clicked_id,
+                        mode: pointer.interaction.mode,
+                        ..Default::default()
+                    };
+                    clicked.backend.on_pointer(app, &hit, false);
+                }
+            }
+        }
+        return (0.0, None); // no hit
+    };
+
+    if let Some(hovered_id) = pointer.interaction.hovered_id {
+        if hovered_id != hit.overlay {
+            if let Some(old_hovered) = overlays.mut_by_id(hovered_id) {
+                if Some(pointer.idx) == old_hovered.primary_pointer {
+                    old_hovered.primary_pointer = None;
+                }
+                old_hovered.backend.on_left(app, idx);
+                pointer = &mut app.input_state.pointers[idx];
+            }
+        }
+    }
+    let Some(hovered) = overlays.mut_by_id(hit.overlay) else {
+        log::warn!("Hit overlay {} does not exist", hit.overlay.0);
+        return (0.0, None); // no hit
+    };
+
+    pointer.interaction.hovered_id = Some(hit.overlay);
+
+    if let Some(primary_pointer) = hovered.primary_pointer {
+        if hit.pointer <= primary_pointer {
+            hovered.primary_pointer = Some(hit.pointer);
+            hit.primary = true;
+        }
+    } else {
+        hovered.primary_pointer = Some(hit.pointer);
+        hit.primary = true;
+    }
+
+    #[cfg(debug_assertions)]
+    log::trace!("Hit: {} {:?}", hovered.state.name, hit);
+
+    if pointer.now.grab && !pointer.before.grab && hovered.state.grabbable {
+        update_focus(&mut app.keyboard_focus, &hovered.state);
+        pointer.start_grab(hovered, &mut app.tasks);
+        return (
+            hit.dist,
+            Some(Haptics {
+                intensity: 0.25,
+                duration: 0.1,
+                frequency: 0.1,
+            }),
+        );
+    }
+
+    // Pass mouse motion events only if not scrolling
+    // (allows scrolling on all Chromium-based applications)
+    let haptics = hovered.backend.on_hover(app, &hit);
+
+    pointer = &mut app.input_state.pointers[idx];
+
+    if pointer.now.scroll_x.abs() > 0.1 || pointer.now.scroll_y.abs() > 0.1 {
+        let scroll_x = pointer.now.scroll_x;
+        let scroll_y = pointer.now.scroll_y;
+        if app.input_state.pointers[1 - idx]
+            .interaction
+            .grabbed
+            .is_some_and(|x| x.grabbed_id == hit.overlay)
+        {
+            let can_curve = hovered
+                .frame_meta()
+                .is_some_and(|e| e.extent[0] >= e.extent[1]);
+
+            if can_curve {
+                let cur = hovered.state.curvature.unwrap_or(0.0);
+                let new = scroll_y.mul_add(-0.01, cur).min(0.5);
+                if new <= f32::EPSILON {
+                    hovered.state.curvature = None;
+                } else {
+                    hovered.state.curvature = Some(new);
+                }
+            } else {
+                hovered.state.curvature = None;
+            }
+        } else {
+            hovered.backend.on_scroll(app, &hit, scroll_y, scroll_x);
+        }
+        pointer = &mut app.input_state.pointers[idx];
+    }
+
+    if pointer.now.click && !pointer.before.click {
+        pointer.interaction.clicked_id = Some(hit.overlay);
+        update_focus(&mut app.keyboard_focus, &hovered.state);
+        hovered.backend.on_pointer(app, &hit, true);
+    } else if !pointer.now.click && pointer.before.click {
+        if let Some(clicked_id) = pointer.interaction.clicked_id.take() {
+            if let Some(clicked) = overlays.mut_by_id(clicked_id) {
+                clicked.backend.on_pointer(app, &hit, false);
+            }
+        } else {
+            hovered.backend.on_pointer(app, &hit, false);
+        }
+    }
+    (hit.dist, haptics)
+}
+
+impl Pointer {
+    fn get_nearest_hit<O>(&mut self, overlays: &mut OverlayContainer<O>) -> Option<PointerHit>
+    where
+        O: Default,
+    {
+        let mut hits: SmallVec<[RayHit; 8]> = smallvec!();
+
+        for overlay in overlays.iter() {
+            if !overlay.state.want_visible || !overlay.state.interactable {
+                continue;
+            }
+
+            if let Some(hit) = self.ray_test(
+                overlay.state.id,
+                &overlay.state.transform,
+                overlay.state.curvature.as_ref(),
+            ) {
+                if hit.dist.is_infinite() || hit.dist.is_nan() {
+                    continue;
+                }
+                hits.push(hit);
+            }
+        }
+
+        hits.sort_by(|a, b| a.dist.total_cmp(&b.dist));
+
+        for hit in &hits {
+            let overlay = overlays.get_by_id(hit.overlay).unwrap(); // safe because we just got the id from the overlay
+
+            let uv = overlay
+                .state
+                .interaction_transform
+                .transform_point2(hit.local_pos);
+
+            if uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0 {
+                continue;
+            }
+
+            return Some(PointerHit {
+                pointer: self.idx,
+                overlay: hit.overlay,
+                mode: self.interaction.mode,
+                primary: false,
+                uv,
+                dist: hit.dist,
+            });
+        }
+
+        None
+    }
+
+    fn start_grab<O>(&mut self, overlay: &mut OverlayData<O>, tasks: &mut TaskContainer)
+    where
+        O: Default,
+    {
+        let offset = self
+            .pose
+            .inverse()
+            .transform_point3a(overlay.state.transform.translation);
+
+        self.interaction.grabbed = Some(GrabData {
+            offset,
+            grabbed_id: overlay.state.id,
+            old_curvature: overlay.state.curvature,
+            grab_all: matches!(self.interaction.mode, PointerMode::Right),
+        });
+        overlay.state.positioning = match overlay.state.positioning {
+            Positioning::FollowHand { hand, lerp } => Positioning::FollowHandPaused { hand, lerp },
+            Positioning::FollowHead { lerp } => Positioning::FollowHeadPaused { lerp },
+            x => x,
+        };
+
+        // Show anchor
+        tasks.enqueue(TaskType::Overlay(
+            OverlaySelector::Name(ANCHOR_NAME.clone()),
+            Box::new(|app, o| {
+                o.transform = app.anchor
+                    * Affine3A::from_scale_rotation_translation(
+                        Vec3::ONE * o.spawn_scale,
+                        o.spawn_rotation,
+                        o.spawn_point.into(),
+                    );
+                o.dirty = true;
+                o.want_visible = true;
+            }),
+        ));
+        log::info!("Hand {}: grabbed {}", self.idx, overlay.state.name);
+    }
+
+    fn handle_grabbed<O>(idx: usize, overlay: &mut OverlayData<O>, app: &mut AppState)
+    where
+        O: Default,
+    {
+        let mut pointer = &mut app.input_state.pointers[idx];
+        if pointer.now.grab {
+            if let Some(grab_data) = pointer.interaction.grabbed.as_mut() {
+                if pointer.now.click {
+                    pointer.interaction.mode = PointerMode::Special;
+                    let cur_scale = overlay.state.transform.x_axis.length();
+                    if cur_scale < 0.1 && pointer.now.scroll_y > 0.0 {
+                        return;
+                    }
+                    if cur_scale > 20. && pointer.now.scroll_y < 0.0 {
+                        return;
+                    }
+
+                    overlay.state.transform.matrix3 = overlay
+                        .state
+                        .transform
+                        .matrix3
+                        .mul_scalar(0.025f32.mul_add(-pointer.now.scroll_y, 1.0));
+                } else if app.session.config.allow_sliding && pointer.now.scroll_y.is_finite() {
+                    grab_data.offset.z -= pointer.now.scroll_y * 0.05;
+                }
+                overlay.state.transform.translation =
+                    pointer.pose.transform_point3a(grab_data.offset);
+                overlay.state.realign(&app.input_state.hmd);
+                overlay.state.dirty = true;
+            } else {
+                log::error!("Grabbed overlay {} does not exist", overlay.state.id.0);
+                pointer.interaction.grabbed = None;
+            }
+        } else {
+            overlay.state.positioning = match overlay.state.positioning {
+                Positioning::FollowHandPaused { hand, lerp } => {
+                    Positioning::FollowHand { hand, lerp }
+                }
+                Positioning::FollowHeadPaused { lerp } => Positioning::FollowHead { lerp },
+                x => x,
+            };
+
+            let save_success = overlay.state.save_transform(app);
+
+            // re-borrow
+            pointer = &mut app.input_state.pointers[idx];
+
+            if save_success {
+                if let Some(grab_data) = pointer.interaction.grabbed.as_ref() {
+                    if overlay.state.curvature != grab_data.old_curvature {
+                        if let Some(val) = overlay.state.curvature {
+                            app.session
+                                .config
+                                .curve_values
+                                .arc_set(overlay.state.name.clone(), val);
+                        } else {
+                            let ref_name = overlay.state.name.as_ref();
+                            app.session.config.curve_values.arc_rm(ref_name);
+                        }
+                    }
+                }
+                app.session.config.transform_values.arc_set(
+                    overlay.state.name.clone(),
+                    overlay.state.saved_transform.unwrap(), // safe
+                );
+            }
+
+            pointer.interaction.grabbed = None;
+
+            // Hide anchor
+            app.tasks.enqueue(TaskType::Overlay(
+                OverlaySelector::Name(ANCHOR_NAME.clone()),
+                Box::new(|_app, o| {
+                    o.want_visible = false;
+                }),
+            ));
+            log::info!("Hand {}: dropped {}", idx, overlay.state.name);
+        }
+    }
+
+    fn ray_test(
+        &self,
+        overlay: OverlayID,
+        transform: &Affine3A,
+        curvature: Option<&f32>,
+    ) -> Option<RayHit> {
+        let (dist, local_pos) = curvature.map_or_else(
+            || {
+                Some(raycast_plane(
+                    &self.pose,
+                    Vec3A::NEG_Z,
+                    transform,
+                    Vec3A::NEG_Z,
+                ))
+            },
+            |curvature| raycast_cylinder(&self.pose, Vec3A::NEG_Z, transform, *curvature),
+        )?;
+
+        if dist < 0.0 {
+            // hit is behind us
+            return None;
+        }
+
+        Some(RayHit {
+            overlay,
+            global_pos: self.pose.transform_point3a(Vec3A::NEG_Z * dist),
+            local_pos,
+            dist,
+        })
+    }
+}
+
+fn raycast_plane(
+    source: &Affine3A,
+    source_fwd: Vec3A,
+    plane: &Affine3A,
+    plane_norm: Vec3A,
+) -> (f32, Vec2) {
+    let plane_normal = plane.transform_vector3a(plane_norm);
+    let ray_dir = source.transform_vector3a(source_fwd);
+
+    let d = plane.translation.dot(-plane_normal);
+    let dist = -(d + source.translation.dot(plane_normal)) / ray_dir.dot(plane_normal);
+
+    let hit_local = plane
+        .inverse()
+        .transform_point3a(source.translation + ray_dir * dist)
+        .xy();
+
+    (dist, hit_local)
+}
+
+fn raycast_cylinder(
+    source: &Affine3A,
+    source_fwd: Vec3A,
+    plane: &Affine3A,
+    curvature: f32,
+) -> Option<(f32, Vec2)> {
+    // this is solved locally; (0,0) is the center of the cylinder, and the cylinder is aligned with the Y axis
+    let size = plane.x_axis.length();
+    let to_local = Affine3A {
+        matrix3: plane.matrix3.mul_scalar(1.0 / size),
+        translation: plane.translation,
+    }
+    .inverse();
+
+    let radius = size / (2.0 * PI * curvature);
+
+    let ray_dir = to_local.transform_vector3a(source.transform_vector3a(source_fwd));
+    let ray_origin = to_local.transform_point3a(source.translation) + Vec3A::NEG_Z * radius;
+
+    let v_dir = ray_dir.xz();
+    let v_pos = ray_origin.xz();
+
+    let l_dir = v_dir.dot(v_dir);
+    let l_pos = v_dir.dot(v_pos);
+    let c = radius.mul_add(-radius, v_pos.dot(v_pos));
+
+    let d = l_pos.mul_add(l_pos, -(l_dir * c));
+    if d < f32::EPSILON {
+        return None;
+    }
+
+    let sqrt_d = d.sqrt();
+
+    let t1 = (-l_pos - sqrt_d) / l_dir;
+    let t2 = (-l_pos + sqrt_d) / l_dir;
+
+    let t = t1.max(t2);
+
+    if t < f32::EPSILON {
+        return None;
+    }
+
+    let mut hit_local = ray_origin + ray_dir * t;
+    if hit_local.z > 0.0 {
+        // hitting the opposite half of the cylinder
+        return None;
+    }
+
+    let max_angle = 2.0 * (size / (2.0 * radius));
+    let x_angle = (hit_local.x / radius).asin();
+
+    hit_local.x = x_angle / max_angle;
+    hit_local.y /= size;
+
+    Some((t, hit_local.xy()))
+}

wlx-overlay-s/src/backend/mod.rs

@@ -0,0 +1,22 @@
+pub mod common;
+pub mod input;
+pub mod notifications;
+
+#[allow(clippy::all)]
+mod notifications_dbus;
+
+#[cfg(feature = "openvr")]
+pub mod openvr;
+
+#[cfg(feature = "openxr")]
+pub mod openxr;
+
+#[cfg(feature = "osc")]
+pub mod osc;
+
+#[cfg(feature = "wayvr")]
+pub mod wayvr;
+
+pub mod overlay;
+
+pub mod task;

wlx-overlay-s/src/backend/notifications.rs

@@ -0,0 +1,292 @@
+use dbus::{
+    arg::{PropMap, Variant},
+    blocking::Connection,
+    channel::MatchingReceiver,
+    message::MatchRule,
+};
+use serde::Deserialize;
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        mpsc, Arc,
+    },
+    time::Duration,
+};
+
+use crate::{
+    backend::notifications_dbus::OrgFreedesktopNotifications,
+    overlays::toast::{Toast, ToastTopic},
+    state::AppState,
+};
+
+pub struct NotificationManager {
+    rx_toast: mpsc::Receiver<Toast>,
+    tx_toast: mpsc::SyncSender<Toast>,
+    dbus_data: Option<Connection>,
+    running: Arc<AtomicBool>,
+}
+
+impl NotificationManager {
+    pub fn new() -> Self {
+        let (tx_toast, rx_toast) = mpsc::sync_channel(10);
+        Self {
+            rx_toast,
+            tx_toast,
+            dbus_data: None,
+            running: Arc::new(AtomicBool::new(true)),
+        }
+    }
+
+    pub fn submit_pending(&self, app: &mut AppState) {
+        if let Some(c) = &self.dbus_data {
+            let _ = c.process(Duration::ZERO);
+        }
+
+        if app.session.config.notifications_enabled {
+            self.rx_toast.try_iter().for_each(|toast| {
+                toast.submit(app);
+            });
+        } else {
+            // consume without submitting
+            self.rx_toast.try_iter().last();
+        }
+    }
+
+    pub fn run_dbus(&mut self) {
+        let c = match Connection::new_session() {
+            Ok(c) => c,
+            Err(e) => {
+                log::error!(
+                    "Failed to connect to dbus. Desktop notifications will not work. Cause: {e:?}"
+                );
+                return;
+            }
+        };
+
+        let mut rule = MatchRule::new_method_call();
+        rule.member = Some("Notify".into());
+        rule.interface = Some("org.freedesktop.Notifications".into());
+        rule.path = Some("/org/freedesktop/Notifications".into());
+        rule.eavesdrop = true;
+
+        let proxy = c.with_proxy(
+            "org.freedesktop.DBus",
+            "/org/freedesktop/DBus",
+            Duration::from_millis(5000),
+        );
+        let result: Result<(), dbus::Error> = proxy.method_call(
+            "org.freedesktop.DBus.Monitoring",
+            "BecomeMonitor",
+            (vec![rule.match_str()], 0u32),
+        );
+
+        if matches!(result, Ok(())) {
+            let sender = self.tx_toast.clone();
+            c.start_receive(
+                rule,
+                Box::new(move |msg, _| {
+                    if let Ok(toast) = parse_dbus(&msg) {
+                        match sender.try_send(toast) {
+                            Ok(()) => {}
+                            Err(e) => {
+                                log::error!("Failed to send notification: {e:?}");
+                            }
+                        }
+                    }
+                    true
+                }),
+            );
+            log::info!("Listening to DBus notifications via BecomeMonitor.");
+        } else {
+            let rule_with_eavesdrop = {
+                let mut rule = rule.clone();
+                rule.eavesdrop = true;
+                rule
+            };
+
+            let sender2 = self.tx_toast.clone();
+            let result = c.add_match(rule_with_eavesdrop, move |(): (), _, msg| {
+                if let Ok(toast) = parse_dbus(msg) {
+                    match sender2.try_send(toast) {
+                        Ok(()) => {}
+                        Err(e) => {
+                            log::error!("Failed to send notification: {e:?}");
+                        }
+                    }
+                }
+                true
+            });
+
+            match result {
+                Ok(_) => {
+                    log::info!("Listening to DBus notifications via eavesdrop.");
+                }
+                Err(_) => {
+                    log::error!("Failed to add DBus match. Desktop notifications will not work.",);
+                }
+            }
+        }
+
+        self.dbus_data = Some(c);
+    }
+
+    pub fn run_udp(&mut self) {
+        let sender = self.tx_toast.clone();
+        let running = self.running.clone();
+        let _ = std::thread::spawn(move || {
+            let addr = "127.0.0.1:42069";
+            let socket = match std::net::UdpSocket::bind(addr) {
+                Ok(s) => s,
+                Err(e) => {
+                    log::error!("Failed to bind notification socket @ {addr}: {e:?}");
+                    return;
+                }
+            };
+            if let Err(err) = socket.set_read_timeout(Some(Duration::from_millis(200))) {
+                log::error!("Failed to set read timeout: {err:?}");
+            }
+
+            let mut buf = [0u8; 1024 * 16]; // vrcx embeds icons as b64
+
+            while running.load(Ordering::Relaxed) {
+                if let Ok((num_bytes, _)) = socket.recv_from(&mut buf) {
+                    let json_str = match std::str::from_utf8(&buf[..num_bytes]) {
+                        Ok(s) => s,
+                        Err(e) => {
+                            log::error!("Failed to receive notification message: {e:?}");
+                            continue;
+                        }
+                    };
+                    let msg = match serde_json::from_str::<XsoMessage>(json_str) {
+                        Ok(m) => m,
+                        Err(e) => {
+                            log::error!("Failed to parse notification message: {e:?}");
+                            continue;
+                        }
+                    };
+
+                    if msg.messageType != 1 {
+                        continue;
+                    }
+
+                    let toast = Toast::new(
+                        ToastTopic::XSNotification,
+                        msg.title,
+                        msg.content.unwrap_or_else(|| "".into()),
+                    )
+                    .with_timeout(msg.timeout.unwrap_or(5.))
+                    .with_sound(msg.volume.unwrap_or(-1.) >= 0.); // XSOverlay still plays at 0,
+
+                    match sender.try_send(toast) {
+                        Ok(()) => {}
+                        Err(e) => {
+                            log::error!("Failed to send notification: {e:?}");
+                        }
+                    }
+                }
+            }
+            log::info!("Notification listener stopped.");
+        });
+    }
+}
+
+impl Drop for NotificationManager {
+    fn drop(&mut self) {
+        self.running.store(false, Ordering::Relaxed);
+    }
+}
+
+pub struct DbusNotificationSender {
+    connection: Connection,
+}
+
+impl DbusNotificationSender {
+    pub fn new() -> anyhow::Result<Self> {
+        Ok(Self {
+            connection: Connection::new_session()?,
+        })
+    }
+
+    pub fn notify_send(
+        &self,
+        summary: &str,
+        body: &str,
+        urgency: u8,
+        timeout: i32,
+        replaces_id: u32,
+        transient: bool,
+    ) -> anyhow::Result<u32> {
+        let proxy = self.connection.with_proxy(
+            "org.freedesktop.Notifications",
+            "/org/freedesktop/Notifications",
+            Duration::from_millis(1000),
+        );
+
+        let mut hints = PropMap::new();
+        hints.insert("urgency".to_string(), Variant(Box::new(urgency)));
+        hints.insert("transient".to_string(), Variant(Box::new(transient)));
+
+        Ok(proxy.notify(
+            "WlxOverlay-S",
+            replaces_id,
+            "",
+            summary,
+            body,
+            vec![],
+            hints,
+            timeout,
+        )?)
+    }
+
+    pub fn notify_close(&self, id: u32) -> anyhow::Result<()> {
+        let proxy = self.connection.with_proxy(
+            "org.freedesktop.Notifications",
+            "/org/freedesktop/Notifications",
+            Duration::from_millis(1000),
+        );
+
+        proxy.close_notification(id)?;
+        Ok(())
+    }
+}
+
+fn parse_dbus(msg: &dbus::Message) -> anyhow::Result<Toast> {
+    let mut args = msg.iter_init();
+    let app_name: String = args.read()?;
+    let _replaces_id: u32 = args.read()?;
+    let _app_icon: String = args.read()?;
+    let summary: String = args.read()?;
+    let body: String = args.read()?;
+
+    let title = if summary.is_empty() {
+        app_name
+    } else {
+        summary
+    };
+
+    Ok(
+        Toast::new(ToastTopic::DesktopNotification, title.into(), body.into())
+            .with_timeout(5.0)
+            .with_opacity(1.0),
+    )
+    // leave the audio part to the desktop env
+}
+
+#[allow(dead_code)]
+#[allow(non_snake_case)]
+#[derive(Debug, Deserialize)]
+struct XsoMessage {
+    messageType: i32,
+    index: Option<i32>,
+    volume: Option<f32>,
+    audioPath: Option<String>,
+    timeout: Option<f32>,
+    title: String,
+    content: Option<String>,
+    icon: Option<String>,
+    height: Option<f32>,
+    opacity: Option<f32>,
+    useBase64Icon: Option<bool>,
+    sourceApp: Option<String>,
+    alwaysShow: Option<bool>,
+}

wlx-overlay-s/src/backend/notifications_dbus.rs

@@ -0,0 +1,353 @@
+// This code was autogenerated with `dbus-codegen-rust -g -m None -d org.freedesktop.Notifications -p /org/freedesktop/Notifications`, see https://github.com/diwic/dbus-rs
+use dbus;
+#[allow(unused_imports)]
+use dbus::arg;
+use dbus::blocking;
+
+pub trait OrgFreedesktopDBusProperties {
+    fn get<R0: for<'b> arg::Get<'b> + 'static>(
+        &self,
+        interface_name: &str,
+        property_name: &str,
+    ) -> Result<R0, dbus::Error>;
+    fn get_all(&self, interface_name: &str) -> Result<arg::PropMap, dbus::Error>;
+    fn set<I2: arg::Arg + arg::Append>(
+        &self,
+        interface_name: &str,
+        property_name: &str,
+        value: I2,
+    ) -> Result<(), dbus::Error>;
+}
+
+#[derive(Debug)]
+pub struct OrgFreedesktopDBusPropertiesPropertiesChanged {
+    pub interface_name: String,
+    pub changed_properties: arg::PropMap,
+    pub invalidated_properties: Vec<String>,
+}
+
+impl arg::AppendAll for OrgFreedesktopDBusPropertiesPropertiesChanged {
+    fn append(&self, i: &mut arg::IterAppend) {
+        arg::RefArg::append(&self.interface_name, i);
+        arg::RefArg::append(&self.changed_properties, i);
+        arg::RefArg::append(&self.invalidated_properties, i);
+    }
+}
+
+impl arg::ReadAll for OrgFreedesktopDBusPropertiesPropertiesChanged {
+    fn read(i: &mut arg::Iter) -> Result<Self, arg::TypeMismatchError> {
+        Ok(Self {
+            interface_name: i.read()?,
+            changed_properties: i.read()?,
+            invalidated_properties: i.read()?,
+        })
+    }
+}
+
+impl dbus::message::SignalArgs for OrgFreedesktopDBusPropertiesPropertiesChanged {
+    const NAME: &'static str = "PropertiesChanged";
+    const INTERFACE: &'static str = "org.freedesktop.DBus.Properties";
+}
+
+impl<T: blocking::BlockingSender, C: ::std::ops::Deref<Target = T>> OrgFreedesktopDBusProperties
+    for blocking::Proxy<'_, C>
+{
+    fn get<R0: for<'b> arg::Get<'b> + 'static>(
+        &self,
+        interface_name: &str,
+        property_name: &str,
+    ) -> Result<R0, dbus::Error> {
+        self.method_call(
+            "org.freedesktop.DBus.Properties",
+            "Get",
+            (interface_name, property_name),
+        )
+        .and_then(|r: (arg::Variant<R0>,)| Ok((r.0).0))
+    }
+
+    fn get_all(&self, interface_name: &str) -> Result<arg::PropMap, dbus::Error> {
+        self.method_call(
+            "org.freedesktop.DBus.Properties",
+            "GetAll",
+            (interface_name,),
+        )
+        .and_then(|r: (arg::PropMap,)| Ok(r.0))
+    }
+
+    fn set<I2: arg::Arg + arg::Append>(
+        &self,
+        interface_name: &str,
+        property_name: &str,
+        value: I2,
+    ) -> Result<(), dbus::Error> {
+        self.method_call(
+            "org.freedesktop.DBus.Properties",
+            "Set",
+            (interface_name, property_name, arg::Variant(value)),
+        )
+    }
+}
+
+pub trait OrgFreedesktopDBusIntrospectable {
+    fn introspect(&self) -> Result<String, dbus::Error>;
+}
+
+impl<T: blocking::BlockingSender, C: ::std::ops::Deref<Target = T>> OrgFreedesktopDBusIntrospectable
+    for blocking::Proxy<'_, C>
+{
+    fn introspect(&self) -> Result<String, dbus::Error> {
+        self.method_call("org.freedesktop.DBus.Introspectable", "Introspect", ())
+            .and_then(|r: (String,)| Ok(r.0))
+    }
+}
+
+pub trait OrgFreedesktopDBusPeer {
+    fn ping(&self) -> Result<(), dbus::Error>;
+    fn get_machine_id(&self) -> Result<String, dbus::Error>;
+}
+
+impl<T: blocking::BlockingSender, C: ::std::ops::Deref<Target = T>> OrgFreedesktopDBusPeer
+    for blocking::Proxy<'_, C>
+{
+    fn ping(&self) -> Result<(), dbus::Error> {
+        self.method_call("org.freedesktop.DBus.Peer", "Ping", ())
+    }
+
+    fn get_machine_id(&self) -> Result<String, dbus::Error> {
+        self.method_call("org.freedesktop.DBus.Peer", "GetMachineId", ())
+            .and_then(|r: (String,)| Ok(r.0))
+    }
+}
+
+pub trait OrgFreedesktopNotifications {
+    fn set_noti_window_visibility(&self, value: bool) -> Result<(), dbus::Error>;
+    fn toggle_dnd(&self) -> Result<bool, dbus::Error>;
+    fn set_dnd(&self, state: bool) -> Result<(), dbus::Error>;
+    fn get_dnd(&self) -> Result<bool, dbus::Error>;
+    fn manually_close_notification(&self, id: u32, timeout: bool) -> Result<(), dbus::Error>;
+    fn close_all_notifications(&self) -> Result<(), dbus::Error>;
+    fn hide_latest_notification(&self, close: bool) -> Result<(), dbus::Error>;
+    fn get_capabilities(&self) -> Result<Vec<String>, dbus::Error>;
+    fn notify(
+        &self,
+        app_name: &str,
+        replaces_id: u32,
+        app_icon: &str,
+        summary: &str,
+        body: &str,
+        actions: Vec<&str>,
+        hints: arg::PropMap,
+        expire_timeout: i32,
+    ) -> Result<u32, dbus::Error>;
+    fn close_notification(&self, id: u32) -> Result<(), dbus::Error>;
+    fn get_server_information(&self) -> Result<(String, String, String, String), dbus::Error>;
+    fn dnd(&self) -> Result<bool, dbus::Error>;
+    fn set_dnd_(&self, value: bool) -> Result<(), dbus::Error>;
+}
+
+#[derive(Debug)]
+pub struct OrgFreedesktopNotificationsOnDndToggle {
+    pub dnd: bool,
+}
+
+impl arg::AppendAll for OrgFreedesktopNotificationsOnDndToggle {
+    fn append(&self, i: &mut arg::IterAppend) {
+        arg::RefArg::append(&self.dnd, i);
+    }
+}
+
+impl arg::ReadAll for OrgFreedesktopNotificationsOnDndToggle {
+    fn read(i: &mut arg::Iter) -> Result<Self, arg::TypeMismatchError> {
+        Ok(Self { dnd: i.read()? })
+    }
+}
+
+impl dbus::message::SignalArgs for OrgFreedesktopNotificationsOnDndToggle {
+    const NAME: &'static str = "OnDndToggle";
+    const INTERFACE: &'static str = "org.freedesktop.Notifications";
+}
+
+#[derive(Debug)]
+pub struct OrgFreedesktopNotificationsNotificationClosed {
+    pub id: u32,
+    pub reason: u32,
+}
+
+impl arg::AppendAll for OrgFreedesktopNotificationsNotificationClosed {
+    fn append(&self, i: &mut arg::IterAppend) {
+        arg::RefArg::append(&self.id, i);
+        arg::RefArg::append(&self.reason, i);
+    }
+}
+
+impl arg::ReadAll for OrgFreedesktopNotificationsNotificationClosed {
+    fn read(i: &mut arg::Iter) -> Result<Self, arg::TypeMismatchError> {
+        Ok(Self {
+            id: i.read()?,
+            reason: i.read()?,
+        })
+    }
+}
+
+impl dbus::message::SignalArgs for OrgFreedesktopNotificationsNotificationClosed {
+    const NAME: &'static str = "NotificationClosed";
+    const INTERFACE: &'static str = "org.freedesktop.Notifications";
+}
+
+#[derive(Debug)]
+pub struct OrgFreedesktopNotificationsActionInvoked {
+    pub id: u32,
+    pub action_key: String,
+}
+
+impl arg::AppendAll for OrgFreedesktopNotificationsActionInvoked {
+    fn append(&self, i: &mut arg::IterAppend) {
+        arg::RefArg::append(&self.id, i);
+        arg::RefArg::append(&self.action_key, i);
+    }
+}
+
+impl arg::ReadAll for OrgFreedesktopNotificationsActionInvoked {
+    fn read(i: &mut arg::Iter) -> Result<Self, arg::TypeMismatchError> {
+        Ok(Self {
+            id: i.read()?,
+            action_key: i.read()?,
+        })
+    }
+}
+
+impl dbus::message::SignalArgs for OrgFreedesktopNotificationsActionInvoked {
+    const NAME: &'static str = "ActionInvoked";
+    const INTERFACE: &'static str = "org.freedesktop.Notifications";
+}
+
+#[derive(Debug)]
+pub struct OrgFreedesktopNotificationsNotificationReplied {
+    pub id: u32,
+    pub text: String,
+}
+
+impl arg::AppendAll for OrgFreedesktopNotificationsNotificationReplied {
+    fn append(&self, i: &mut arg::IterAppend) {
+        arg::RefArg::append(&self.id, i);
+        arg::RefArg::append(&self.text, i);
+    }
+}
+
+impl arg::ReadAll for OrgFreedesktopNotificationsNotificationReplied {
+    fn read(i: &mut arg::Iter) -> Result<Self, arg::TypeMismatchError> {
+        Ok(Self {
+            id: i.read()?,
+            text: i.read()?,
+        })
+    }
+}
+
+impl dbus::message::SignalArgs for OrgFreedesktopNotificationsNotificationReplied {
+    const NAME: &'static str = "NotificationReplied";
+    const INTERFACE: &'static str = "org.freedesktop.Notifications";
+}
+
+impl<T: blocking::BlockingSender, C: ::std::ops::Deref<Target = T>> OrgFreedesktopNotifications
+    for blocking::Proxy<'_, C>
+{
+    fn set_noti_window_visibility(&self, value: bool) -> Result<(), dbus::Error> {
+        self.method_call(
+            "org.freedesktop.Notifications",
+            "SetNotiWindowVisibility",
+            (value,),
+        )
+    }
+
+    fn toggle_dnd(&self) -> Result<bool, dbus::Error> {
+        self.method_call("org.freedesktop.Notifications", "ToggleDnd", ())
+            .and_then(|r: (bool,)| Ok(r.0))
+    }
+
+    fn set_dnd(&self, state: bool) -> Result<(), dbus::Error> {
+        self.method_call("org.freedesktop.Notifications", "SetDnd", (state,))
+    }
+
+    fn get_dnd(&self) -> Result<bool, dbus::Error> {
+        self.method_call("org.freedesktop.Notifications", "GetDnd", ())
+            .and_then(|r: (bool,)| Ok(r.0))
+    }
+
+    fn manually_close_notification(&self, id: u32, timeout: bool) -> Result<(), dbus::Error> {
+        self.method_call(
+            "org.freedesktop.Notifications",
+            "ManuallyCloseNotification",
+            (id, timeout),
+        )
+    }
+
+    fn close_all_notifications(&self) -> Result<(), dbus::Error> {
+        self.method_call("org.freedesktop.Notifications", "CloseAllNotifications", ())
+    }
+
+    fn hide_latest_notification(&self, close: bool) -> Result<(), dbus::Error> {
+        self.method_call(
+            "org.freedesktop.Notifications",
+            "HideLatestNotification",
+            (close,),
+        )
+    }
+
+    fn get_capabilities(&self) -> Result<Vec<String>, dbus::Error> {
+        self.method_call("org.freedesktop.Notifications", "GetCapabilities", ())
+            .and_then(|r: (Vec<String>,)| Ok(r.0))
+    }
+
+    fn notify(
+        &self,
+        app_name: &str,
+        replaces_id: u32,
+        app_icon: &str,
+        summary: &str,
+        body: &str,
+        actions: Vec<&str>,
+        hints: arg::PropMap,
+        expire_timeout: i32,
+    ) -> Result<u32, dbus::Error> {
+        self.method_call(
+            "org.freedesktop.Notifications",
+            "Notify",
+            (
+                app_name,
+                replaces_id,
+                app_icon,
+                summary,
+                body,
+                actions,
+                hints,
+                expire_timeout,
+            ),
+        )
+        .and_then(|r: (u32,)| Ok(r.0))
+    }
+
+    fn close_notification(&self, id: u32) -> Result<(), dbus::Error> {
+        self.method_call("org.freedesktop.Notifications", "CloseNotification", (id,))
+    }
+
+    fn get_server_information(&self) -> Result<(String, String, String, String), dbus::Error> {
+        self.method_call("org.freedesktop.Notifications", "GetServerInformation", ())
+    }
+
+    fn dnd(&self) -> Result<bool, dbus::Error> {
+        <Self as blocking::stdintf::org_freedesktop_dbus::Properties>::get(
+            self,
+            "org.freedesktop.Notifications",
+            "Dnd",
+        )
+    }
+
+    fn set_dnd_(&self, value: bool) -> Result<(), dbus::Error> {
+        <Self as blocking::stdintf::org_freedesktop_dbus::Properties>::set(
+            self,
+            "org.freedesktop.Notifications",
+            "Dnd",
+            value,
+        )
+    }
+}

wlx-overlay-s/src/backend/openvr/helpers.rs

@@ -0,0 +1,188 @@
+use std::ffi::CStr;
+
+use glam::Affine3A;
+use ovr_overlay::{pose::Matrix3x4, settings::SettingsManager, sys::HmdMatrix34_t};
+use thiserror::Error;
+
+use crate::backend::{common::BackendError, task::ColorChannel};
+
+pub trait Affine3AConvert {
+    fn from_affine(affine: &Affine3A) -> Self;
+    fn to_affine(&self) -> Affine3A;
+}
+
+impl Affine3AConvert for Matrix3x4 {
+    fn from_affine(affine: &Affine3A) -> Self {
+        Self([
+            [
+                affine.matrix3.x_axis.x,
+                affine.matrix3.y_axis.x,
+                affine.matrix3.z_axis.x,
+                affine.translation.x,
+            ],
+            [
+                affine.matrix3.x_axis.y,
+                affine.matrix3.y_axis.y,
+                affine.matrix3.z_axis.y,
+                affine.translation.y,
+            ],
+            [
+                affine.matrix3.x_axis.z,
+                affine.matrix3.y_axis.z,
+                affine.matrix3.z_axis.z,
+                affine.translation.z,
+            ],
+        ])
+    }
+
+    fn to_affine(&self) -> Affine3A {
+        Affine3A::from_cols_array_2d(&[
+            [self.0[0][0], self.0[1][0], self.0[2][0]],
+            [self.0[0][1], self.0[1][1], self.0[2][1]],
+            [self.0[0][2], self.0[1][2], self.0[2][2]],
+            [self.0[0][3], self.0[1][3], self.0[2][3]],
+        ])
+    }
+}
+
+impl Affine3AConvert for HmdMatrix34_t {
+    fn from_affine(affine: &Affine3A) -> Self {
+        Self {
+            m: [
+                [
+                    affine.matrix3.x_axis.x,
+                    affine.matrix3.y_axis.x,
+                    affine.matrix3.z_axis.x,
+                    affine.translation.x,
+                ],
+                [
+                    affine.matrix3.x_axis.y,
+                    affine.matrix3.y_axis.y,
+                    affine.matrix3.z_axis.y,
+                    affine.translation.y,
+                ],
+                [
+                    affine.matrix3.x_axis.z,
+                    affine.matrix3.y_axis.z,
+                    affine.matrix3.z_axis.z,
+                    affine.translation.z,
+                ],
+            ],
+        }
+    }
+
+    fn to_affine(&self) -> Affine3A {
+        Affine3A::from_cols_array_2d(&[
+            [self.m[0][0], self.m[1][0], self.m[2][0]],
+            [self.m[0][1], self.m[1][1], self.m[2][1]],
+            [self.m[0][2], self.m[1][2], self.m[2][2]],
+            [self.m[0][3], self.m[1][3], self.m[2][3]],
+        ])
+    }
+}
+
+#[derive(Error, Debug)]
+pub(super) enum OVRError {
+    #[error("ovr input error: {0}")]
+    InputError(&'static str),
+}
+
+impl From<ovr_overlay::errors::EVRInputError> for OVRError {
+    fn from(e: ovr_overlay::errors::EVRInputError) -> Self {
+        Self::InputError(e.description())
+    }
+}
+
+impl From<OVRError> for BackendError {
+    fn from(e: OVRError) -> Self {
+        Self::Fatal(anyhow::Error::new(e))
+    }
+}
+
+const STEAMVR_SECTION: &CStr = c"steamvr";
+const COLOR_GAIN_CSTR: [&CStr; 3] = [
+    c"hmdDisplayColorGainR",
+    c"hmdDisplayColorGainG",
+    c"hmdDisplayColorGainB",
+];
+
+pub(super) fn adjust_gain(
+    settings: &mut SettingsManager,
+    ch: ColorChannel,
+    delta: f32,
+) -> Option<()> {
+    let current = [
+        settings
+            .get_float(STEAMVR_SECTION, COLOR_GAIN_CSTR[0])
+            .ok()?,
+        settings
+            .get_float(STEAMVR_SECTION, COLOR_GAIN_CSTR[1])
+            .ok()?,
+        settings
+            .get_float(STEAMVR_SECTION, COLOR_GAIN_CSTR[2])
+            .ok()?,
+    ];
+
+    // prevent user from turning everything black
+    let mut min = if current[0] + current[1] + current[2] < 0.11 {
+        0.1
+    } else {
+        0.0
+    };
+
+    match ch {
+        ColorChannel::R => {
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[0],
+                    (current[0] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+        }
+        ColorChannel::G => {
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[1],
+                    (current[1] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+        }
+        ColorChannel::B => {
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[2],
+                    (current[2] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+        }
+        ColorChannel::All => {
+            min *= 0.3333;
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[0],
+                    (current[0] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[1],
+                    (current[1] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[2],
+                    (current[2] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+        }
+    }
+
+    Some(())
+}

wlx-overlay-s/src/backend/openvr/input.rs

@@ -0,0 +1,360 @@
+use std::{array, fs::File, io::Write, time::Duration};
+
+use anyhow::bail;
+use ovr_overlay::{
+    input::{ActionHandle, ActionSetHandle, ActiveActionSet, InputManager, InputValueHandle},
+    sys::{
+        ETrackedControllerRole, ETrackedDeviceClass, ETrackedDeviceProperty,
+        ETrackingUniverseOrigin,
+    },
+    system::SystemManager,
+    TrackedDeviceIndex,
+};
+
+use crate::{
+    backend::input::{Haptics, TrackedDevice, TrackedDeviceRole},
+    config_io,
+    state::AppState,
+};
+
+use super::helpers::{Affine3AConvert, OVRError};
+
+const SET_DEFAULT: &str = "/actions/default";
+const INPUT_SOURCES: [&str; 2] = ["/user/hand/left", "/user/hand/right"];
+const PATH_POSES: [&str; 2] = [
+    "/actions/default/in/LeftHand",
+    "/actions/default/in/RightHand",
+];
+const PATH_HAPTICS: [&str; 2] = [
+    "/actions/default/out/HapticsLeft",
+    "/actions/default/out/HapticsRight",
+];
+
+const PATH_ALT_CLICK: &str = "/actions/default/in/AltClick";
+const PATH_CLICK_MODIFIER_MIDDLE: &str = "/actions/default/in/ClickModifierMiddle";
+const PATH_CLICK_MODIFIER_RIGHT: &str = "/actions/default/in/ClickModifierRight";
+const PATH_CLICK: &str = "/actions/default/in/Click";
+const PATH_GRAB: &str = "/actions/default/in/Grab";
+const PATH_MOVE_MOUSE: &str = "/actions/default/in/MoveMouse";
+const PATH_SCROLL: &str = "/actions/default/in/Scroll";
+const PATH_SHOW_HIDE: &str = "/actions/default/in/ShowHide";
+const PATH_SPACE_DRAG: &str = "/actions/default/in/SpaceDrag";
+const PATH_SPACE_ROTATE: &str = "/actions/default/in/SpaceRotate";
+const PATH_TOGGLE_DASHBOARD: &str = "/actions/default/in/ToggleDashboard";
+
+const INPUT_ANY: InputValueHandle = InputValueHandle(ovr_overlay::sys::k_ulInvalidInputValueHandle);
+
+pub(super) struct OpenVrInputSource {
+    hands: [OpenVrHandSource; 2],
+    set_hnd: ActionSetHandle,
+    click_hnd: ActionHandle,
+    grab_hnd: ActionHandle,
+    scroll_hnd: ActionHandle,
+    alt_click_hnd: ActionHandle,
+    show_hide_hnd: ActionHandle,
+    toggle_dashboard_hnd: ActionHandle,
+    space_drag_hnd: ActionHandle,
+    space_rotate_hnd: ActionHandle,
+    click_modifier_right_hnd: ActionHandle,
+    click_modifier_middle_hnd: ActionHandle,
+    move_mouse_hnd: ActionHandle,
+}
+
+pub(super) struct OpenVrHandSource {
+    has_pose: bool,
+    device: Option<TrackedDeviceIndex>,
+    input_hnd: InputValueHandle,
+    pose_hnd: ActionHandle,
+    haptics_hnd: ActionHandle,
+}
+
+impl OpenVrInputSource {
+    pub fn new(input: &mut InputManager) -> Result<Self, OVRError> {
+        let set_hnd = input.get_action_set_handle(SET_DEFAULT)?;
+
+        let click_hnd = input.get_action_handle(PATH_CLICK)?;
+        let grab_hnd = input.get_action_handle(PATH_GRAB)?;
+        let scroll_hnd = input.get_action_handle(PATH_SCROLL)?;
+        let alt_click_hnd = input.get_action_handle(PATH_ALT_CLICK)?;
+        let show_hide_hnd = input.get_action_handle(PATH_SHOW_HIDE)?;
+        let toggle_dashboard_hnd = input.get_action_handle(PATH_TOGGLE_DASHBOARD)?;
+        let space_drag_hnd = input.get_action_handle(PATH_SPACE_DRAG)?;
+        let space_rotate_hnd = input.get_action_handle(PATH_SPACE_ROTATE)?;
+        let click_modifier_right_hnd = input.get_action_handle(PATH_CLICK_MODIFIER_RIGHT)?;
+        let click_modifier_middle_hnd = input.get_action_handle(PATH_CLICK_MODIFIER_MIDDLE)?;
+        let move_mouse_hnd = input.get_action_handle(PATH_MOVE_MOUSE)?;
+
+        let input_hnd: Vec<InputValueHandle> = INPUT_SOURCES
+            .iter()
+            .map(|path| Ok((input.get_input_source_handle(path))?))
+            .collect::<Result<_, OVRError>>()?;
+
+        let pose_hnd: Vec<ActionHandle> = PATH_POSES
+            .iter()
+            .map(|path| Ok((input.get_action_handle(path))?))
+            .collect::<Result<_, OVRError>>()?;
+
+        let haptics_hnd: Vec<ActionHandle> = PATH_HAPTICS
+            .iter()
+            .map(|path| Ok((input.get_action_handle(path))?))
+            .collect::<Result<_, OVRError>>()?;
+
+        let hands: [OpenVrHandSource; 2] = array::from_fn(|i| OpenVrHandSource {
+            has_pose: false,
+            device: None,
+            input_hnd: input_hnd[i],
+            pose_hnd: pose_hnd[i],
+            haptics_hnd: haptics_hnd[i],
+        });
+
+        Ok(Self {
+            hands,
+            set_hnd,
+            click_hnd,
+            grab_hnd,
+            scroll_hnd,
+            alt_click_hnd,
+            show_hide_hnd,
+            toggle_dashboard_hnd,
+            space_drag_hnd,
+            space_rotate_hnd,
+            click_modifier_right_hnd,
+            click_modifier_middle_hnd,
+            move_mouse_hnd,
+        })
+    }
+
+    pub fn haptics(&mut self, input: &mut InputManager, hand: usize, haptics: &Haptics) {
+        let action_handle = self.hands[hand].haptics_hnd;
+        let _ = input.trigger_haptic_vibration_action(
+            action_handle,
+            0.0,
+            Duration::from_secs_f32(haptics.duration),
+            haptics.frequency,
+            haptics.intensity,
+            INPUT_ANY,
+        );
+    }
+
+    pub fn update(
+        &mut self,
+        universe: ETrackingUniverseOrigin,
+        input: &mut InputManager,
+        system: &mut SystemManager,
+        app: &mut AppState,
+    ) {
+        let aas = ActiveActionSet(ovr_overlay::sys::VRActiveActionSet_t {
+            ulActionSet: self.set_hnd.0,
+            ulRestrictedToDevice: 0,
+            ulSecondaryActionSet: 0,
+            unPadding: 0,
+            nPriority: 0,
+        });
+
+        let _ = input.update_actions(&mut [aas]);
+
+        let devices = system.get_device_to_absolute_tracking_pose(universe.clone(), 0.005);
+        app.input_state.hmd = devices[0].mDeviceToAbsoluteTracking.to_affine();
+
+        for i in 0..2 {
+            let hand = &mut self.hands[i];
+            let app_hand = &mut app.input_state.pointers[i];
+
+            if let Some(device) = hand.device {
+                app_hand.raw_pose = devices[device.0 as usize]
+                    .mDeviceToAbsoluteTracking
+                    .to_affine();
+            }
+
+            hand.has_pose = false;
+
+            let _ = input
+                .get_pose_action_data_relative_to_now(
+                    hand.pose_hnd,
+                    universe.clone(),
+                    0.005,
+                    INPUT_ANY,
+                )
+                .map(|pose| {
+                    app_hand.pose = pose.0.pose.mDeviceToAbsoluteTracking.to_affine();
+                    hand.has_pose = true;
+                });
+
+            app_hand.now.click = input
+                .get_digital_action_data(self.click_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.grab = input
+                .get_digital_action_data(self.grab_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.alt_click = input
+                .get_digital_action_data(self.alt_click_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.show_hide = input
+                .get_digital_action_data(self.show_hide_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.toggle_dashboard = input
+                .get_digital_action_data(self.toggle_dashboard_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.space_drag = input
+                .get_digital_action_data(self.space_drag_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.space_rotate = input
+                .get_digital_action_data(self.space_rotate_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.click_modifier_right = input
+                .get_digital_action_data(self.click_modifier_right_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.click_modifier_middle = input
+                .get_digital_action_data(self.click_modifier_middle_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.move_mouse = input
+                .get_digital_action_data(self.move_mouse_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            let scroll = input
+                .get_analog_action_data(self.scroll_hnd, hand.input_hnd)
+                .map(|x| (x.0.x, x.0.y))
+                .unwrap_or((0.0, 0.0));
+            app_hand.now.scroll_x = scroll.0;
+            app_hand.now.scroll_y = scroll.1;
+        }
+    }
+
+    pub fn update_devices(&mut self, system: &mut SystemManager, app: &mut AppState) {
+        app.input_state.devices.clear();
+        for idx in 0..TrackedDeviceIndex::MAX {
+            let device = TrackedDeviceIndex::new(idx as _).unwrap(); // safe
+            if !system.is_tracked_device_connected(device) {
+                continue;
+            }
+
+            let class = system.get_tracked_device_class(device);
+
+            let role = match class {
+                ETrackedDeviceClass::TrackedDeviceClass_HMD => TrackedDeviceRole::Hmd,
+                ETrackedDeviceClass::TrackedDeviceClass_Controller => {
+                    let role = system.get_controller_role_for_tracked_device_index(device);
+                    match role {
+                        ETrackedControllerRole::TrackedControllerRole_LeftHand => {
+                            self.hands[0].device = Some(device);
+                            TrackedDeviceRole::LeftHand
+                        }
+                        ETrackedControllerRole::TrackedControllerRole_RightHand => {
+                            self.hands[1].device = Some(device);
+                            TrackedDeviceRole::RightHand
+                        }
+                        _ => continue,
+                    }
+                }
+                ETrackedDeviceClass::TrackedDeviceClass_GenericTracker => {
+                    TrackedDeviceRole::Tracker
+                }
+                _ => continue,
+            };
+
+            if let Some(device) = get_tracked_device(system, device, role) {
+                app.input_state.devices.push(device);
+            }
+        }
+
+        app.input_state.devices.sort_by(|a, b| {
+            u8::from(a.soc.is_none())
+                .cmp(&u8::from(b.soc.is_none()))
+                .then((a.role as u8).cmp(&(b.role as u8)))
+                .then(a.soc.unwrap_or(999.).total_cmp(&b.soc.unwrap_or(999.)))
+        });
+    }
+}
+
+fn get_tracked_device(
+    system: &mut SystemManager,
+    index: TrackedDeviceIndex,
+    role: TrackedDeviceRole,
+) -> Option<TrackedDevice> {
+    let soc = system
+        .get_tracked_device_property(
+            index,
+            ETrackedDeviceProperty::Prop_DeviceBatteryPercentage_Float,
+        )
+        .ok();
+
+    let charging = if soc.is_some() {
+        system
+            .get_tracked_device_property(index, ETrackedDeviceProperty::Prop_DeviceIsCharging_Bool)
+            .unwrap_or(false)
+    } else {
+        false
+    };
+
+    // TODO: cache this
+    let is_alvr = system
+        .get_tracked_device_property(
+            index,
+            ETrackedDeviceProperty::Prop_TrackingSystemName_String,
+        )
+        .map(|x: String| x.contains("ALVR"))
+        .unwrap_or(false);
+
+    if is_alvr {
+        // don't show ALVR's fake trackers on battery panel
+        return None;
+    }
+
+    Some(TrackedDevice {
+        soc,
+        charging,
+        role,
+    })
+}
+
+pub fn set_action_manifest(input: &mut InputManager) -> anyhow::Result<()> {
+    let action_path = config_io::get_config_root().join("actions.json");
+
+    if let Err(e) = File::create(&action_path)
+        .and_then(|mut f| f.write_all(include_bytes!("../../res/actions.json")))
+    {
+        log::warn!("Could not write action manifest: {e}");
+    }
+
+    let binding_path = config_io::get_config_root().join("actions_binding_knuckles.json");
+    if !binding_path.is_file() {
+        File::create(&binding_path)?
+            .write_all(include_bytes!("../../res/actions_binding_knuckles.json"))?;
+    }
+
+    let binding_path = config_io::get_config_root().join("actions_binding_vive.json");
+    if !binding_path.is_file() {
+        File::create(&binding_path)?
+            .write_all(include_bytes!("../../res/actions_binding_vive.json"))?;
+    }
+
+    let binding_path = config_io::get_config_root().join("actions_binding_oculus.json");
+    if !binding_path.is_file() {
+        File::create(&binding_path)?
+            .write_all(include_bytes!("../../res/actions_binding_oculus.json"))?;
+    }
+
+    if let Err(e) = input.set_action_manifest(action_path.as_path()) {
+        bail!("Failed to set action manifest: {}", e);
+    }
+    Ok(())
+}

wlx-overlay-s/src/backend/openvr/lines.rs

@@ -0,0 +1,264 @@
+use std::f32::consts::PI;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+
+use ash::vk::SubmitInfo;
+use glam::{Affine3A, Vec3, Vec3A, Vec4};
+use idmap::IdMap;
+use ovr_overlay::overlay::OverlayManager;
+use ovr_overlay::sys::ETrackingUniverseOrigin;
+use vulkano::{
+    command_buffer::{
+        CommandBufferBeginInfo, CommandBufferLevel, CommandBufferUsage, RecordingCommandBuffer,
+    },
+    format::Format,
+    image::view::ImageView,
+    image::{Image, ImageLayout},
+    sync::{
+        fence::{Fence, FenceCreateInfo},
+        AccessFlags, DependencyInfo, ImageMemoryBarrier, PipelineStages,
+    },
+    VulkanObject,
+};
+use wgui::gfx::WGfx;
+
+use crate::backend::overlay::{
+    FrameMeta, OverlayData, OverlayRenderer, OverlayState, ShouldRender, SplitOverlayBackend,
+    Z_ORDER_LINES,
+};
+use crate::graphics::CommandBuffers;
+use crate::state::AppState;
+
+use super::overlay::OpenVrOverlayData;
+
+static LINE_AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(1);
+
+pub(super) struct LinePool {
+    lines: IdMap<usize, OverlayData<OpenVrOverlayData>>,
+    view: Arc<ImageView>,
+    colors: [Vec4; 5],
+}
+
+impl LinePool {
+    pub fn new(graphics: Arc<WGfx>) -> anyhow::Result<Self> {
+        let mut command_buffer =
+            graphics.create_xfer_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+
+        let buf = vec![255; 16];
+
+        let texture = command_buffer.upload_image(2, 2, Format::R8G8B8A8_UNORM, &buf)?;
+        command_buffer.build_and_execute_now()?;
+
+        transition_layout(
+            &graphics,
+            texture.clone(),
+            ImageLayout::ShaderReadOnlyOptimal,
+            ImageLayout::TransferSrcOptimal,
+        )?
+        .wait(None)?;
+
+        let view = ImageView::new_default(texture)?;
+
+        Ok(Self {
+            lines: IdMap::new(),
+            view,
+            colors: [
+                Vec4::new(1., 1., 1., 1.),
+                Vec4::new(0., 0.375, 0.5, 1.),
+                Vec4::new(0.69, 0.188, 0., 1.),
+                Vec4::new(0.375, 0., 0.5, 1.),
+                Vec4::new(1., 0., 0., 1.),
+            ],
+        })
+    }
+
+    pub fn allocate(&mut self) -> usize {
+        let id = LINE_AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed);
+
+        let mut data = OverlayData::<OpenVrOverlayData> {
+            state: OverlayState {
+                name: Arc::from(format!("wlx-line{id}")),
+                show_hide: true,
+                ..Default::default()
+            },
+            backend: Box::new(SplitOverlayBackend {
+                renderer: Box::new(StaticRenderer {
+                    view: self.view.clone(),
+                }),
+                ..Default::default()
+            }),
+            data: OpenVrOverlayData {
+                width: 0.002,
+                override_width: true,
+                image_view: Some(self.view.clone()),
+                image_dirty: true,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        data.state.z_order = Z_ORDER_LINES;
+        data.state.dirty = true;
+
+        self.lines.insert(id, data);
+        id
+    }
+
+    pub fn draw_from(
+        &mut self,
+        id: usize,
+        mut from: Affine3A,
+        len: f32,
+        color: usize,
+        hmd: &Affine3A,
+    ) {
+        let rotation = Affine3A::from_axis_angle(Vec3::X, -PI * 0.5);
+
+        from.translation += from.transform_vector3a(Vec3A::NEG_Z) * (len * 0.5);
+        let mut transform = from * rotation * Affine3A::from_scale(Vec3::new(1., len / 0.002, 1.));
+
+        let to_hmd = hmd.translation - from.translation;
+        let sides = [Vec3A::Z, Vec3A::X, Vec3A::NEG_Z, Vec3A::NEG_X];
+        let rotations = [
+            Affine3A::IDENTITY,
+            Affine3A::from_axis_angle(Vec3::Y, PI * 0.5),
+            Affine3A::from_axis_angle(Vec3::Y, PI * 1.0),
+            Affine3A::from_axis_angle(Vec3::Y, PI * 1.5),
+        ];
+        let mut closest = (0, 0.0);
+        for (i, &side) in sides.iter().enumerate() {
+            let dot = to_hmd.dot(transform.transform_vector3a(side));
+            if i == 0 || dot > closest.1 {
+                closest = (i, dot);
+            }
+        }
+
+        transform *= rotations[closest.0];
+
+        debug_assert!(color < self.colors.len());
+
+        self.draw_transform(id, transform, self.colors[color]);
+    }
+
+    fn draw_transform(&mut self, id: usize, transform: Affine3A, color: Vec4) {
+        if let Some(data) = self.lines.get_mut(id) {
+            data.state.want_visible = true;
+            data.state.transform = transform;
+            data.data.color = color;
+        } else {
+            log::warn!("Line {id} does not exist");
+        }
+    }
+
+    pub fn update(
+        &mut self,
+        universe: ETrackingUniverseOrigin,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> anyhow::Result<()> {
+        for data in self.lines.values_mut() {
+            data.after_input(overlay, app)?;
+            if data.state.want_visible {
+                if data.state.dirty {
+                    data.upload_texture(overlay, &app.gfx);
+                    data.state.dirty = false;
+                }
+
+                data.upload_transform(universe.clone(), overlay);
+                data.upload_color(overlay);
+            }
+        }
+        Ok(())
+    }
+
+    pub fn mark_dirty(&mut self) {
+        for data in self.lines.values_mut() {
+            data.state.dirty = true;
+        }
+    }
+}
+
+struct StaticRenderer {
+    view: Arc<ImageView>,
+}
+
+impl OverlayRenderer for StaticRenderer {
+    fn init(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn pause(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn resume(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn should_render(&mut self, _app: &mut AppState) -> anyhow::Result<ShouldRender> {
+        Ok(ShouldRender::Unable)
+    }
+    fn render(
+        &mut self,
+        _app: &mut AppState,
+        _tgt: Arc<ImageView>,
+        _buf: &mut CommandBuffers,
+        _alpha: f32,
+    ) -> anyhow::Result<bool> {
+        Ok(false)
+    }
+    fn frame_meta(&mut self) -> Option<FrameMeta> {
+        Some(FrameMeta {
+            extent: self.view.image().extent(),
+            ..Default::default()
+        })
+    }
+}
+
+pub fn transition_layout(
+    gfx: &WGfx,
+    image: Arc<Image>,
+    old_layout: ImageLayout,
+    new_layout: ImageLayout,
+) -> anyhow::Result<Fence> {
+    let barrier = ImageMemoryBarrier {
+        src_stages: PipelineStages::ALL_TRANSFER,
+        src_access: AccessFlags::TRANSFER_WRITE,
+        dst_stages: PipelineStages::ALL_TRANSFER,
+        dst_access: AccessFlags::TRANSFER_READ,
+        old_layout,
+        new_layout,
+        subresource_range: image.subresource_range(),
+        ..ImageMemoryBarrier::image(image)
+    };
+
+    let command_buffer = unsafe {
+        let mut builder = RecordingCommandBuffer::new(
+            gfx.command_buffer_allocator.clone(),
+            gfx.queue_gfx.queue_family_index(),
+            CommandBufferLevel::Primary,
+            CommandBufferBeginInfo {
+                usage: CommandBufferUsage::OneTimeSubmit,
+                inheritance_info: None,
+                ..Default::default()
+            },
+        )?;
+
+        builder.pipeline_barrier(&DependencyInfo {
+            image_memory_barriers: smallvec::smallvec![barrier],
+            ..Default::default()
+        })?;
+        builder.end()?
+    };
+
+    let fence = Fence::new(gfx.device.clone(), FenceCreateInfo::default())?;
+
+    let fns = gfx.device.fns();
+    unsafe {
+        (fns.v1_0.queue_submit)(
+            gfx.queue_gfx.handle(),
+            1,
+            [SubmitInfo::default().command_buffers(&[command_buffer.handle()])].as_ptr(),
+            fence.handle(),
+        )
+    }
+    .result()?;
+
+    Ok(fence)
+}

wlx-overlay-s/src/backend/openvr/manifest.rs

@@ -0,0 +1,88 @@
+use std::{fs::File, io::Read};
+
+use anyhow::bail;
+use json::{array, object};
+use ovr_overlay::applications::ApplicationsManager;
+
+use crate::config_io;
+
+const APP_KEY: &str = "galister.wlxoverlay-s";
+
+pub(super) fn install_manifest(app_mgr: &mut ApplicationsManager) -> anyhow::Result<()> {
+    let manifest_path = config_io::get_config_root().join("wlx-overlay-s.vrmanifest");
+
+    let appimage_path = std::env::var("APPIMAGE");
+    let executable_pathbuf = std::env::current_exe()?;
+
+    let executable_path = match appimage_path {
+        Ok(ref path) => path,
+        Err(_) => executable_pathbuf
+            .to_str()
+            .ok_or_else(|| anyhow::anyhow!("Invalid executable path"))?,
+    };
+
+    if app_mgr.is_application_installed(APP_KEY) == Ok(true) {
+        if let Ok(mut file) = File::open(&manifest_path) {
+            let mut buf = String::new();
+            if file.read_to_string(&mut buf).is_ok() {
+                let manifest: json::JsonValue = json::parse(&buf)?;
+                if manifest["applications"][0]["binary_path_linux"] == executable_path {
+                    log::info!("Manifest already up to date");
+                    return Ok(());
+                }
+            }
+        }
+    }
+
+    let manifest = object! {
+        source: "builtin",
+        applications: array![
+            object! {
+                app_key: APP_KEY,
+                launch_type: "binary",
+                binary_path_linux: executable_path,
+                is_dashboard_overlay: true,
+                strings: object!{
+                    "en_us": object!{
+                        name: "WlxOverlay-S",
+                        description: "A lightweight Wayland desktop overlay for OpenVR/OpenXR",
+                    },
+                },
+            },
+        ],
+    };
+
+    let Ok(mut file) = File::create(&manifest_path) else {
+        bail!("Failed to create manifest file at {:?}", manifest_path);
+    };
+
+    if let Err(e) = manifest.write(&mut file) {
+        bail!(
+            "Failed to write manifest file at {:?}: {:?}",
+            manifest_path,
+            e
+        );
+    }
+
+    if let Err(e) = app_mgr.add_application_manifest(&manifest_path, false) {
+        bail!("Failed to add manifest to OpenVR: {}", e.description());
+    }
+
+    if let Err(e) = app_mgr.set_application_auto_launch(APP_KEY, true) {
+        bail!("Failed to set auto launch: {}", e.description());
+    }
+
+    Ok(())
+}
+
+pub(super) fn uninstall_manifest(app_mgr: &mut ApplicationsManager) -> anyhow::Result<()> {
+    let manifest_path = config_io::get_config_root().join("wlx-overlay-s.vrmanifest");
+
+    if app_mgr.is_application_installed(APP_KEY) == Ok(true) {
+        if let Err(e) = app_mgr.remove_application_manifest(&manifest_path) {
+            bail!("Failed to remove manifest from OpenVR: {}", e.description());
+        }
+        log::info!("Uninstalled manifest");
+    }
+    Ok(())
+}

wlx-overlay-s/src/backend/openvr/mod.rs

@@ -0,0 +1,389 @@
+use std::{
+    collections::VecDeque,
+    ops::Add,
+    sync::{
+        atomic::{AtomicBool, AtomicUsize, Ordering},
+        Arc,
+    },
+    time::{Duration, Instant},
+};
+
+use anyhow::{anyhow, Result};
+use ovr_overlay::{
+    sys::{ETrackedDeviceProperty, EVRApplicationType, EVREventType},
+    TrackedDeviceIndex,
+};
+use vulkano::{device::physical::PhysicalDevice, Handle, VulkanObject};
+
+use crate::{
+    backend::{
+        common::{BackendError, OverlayContainer},
+        input::interact,
+        notifications::NotificationManager,
+        openvr::{
+            helpers::adjust_gain,
+            input::{set_action_manifest, OpenVrInputSource},
+            lines::LinePool,
+            manifest::{install_manifest, uninstall_manifest},
+            overlay::OpenVrOverlayData,
+        },
+        overlay::{OverlayData, ShouldRender},
+        task::{SystemTask, TaskType},
+    },
+    graphics::{init_openvr_graphics, CommandBuffers},
+    overlays::{
+        toast::{Toast, ToastTopic},
+        watch::{watch_fade, WATCH_NAME},
+    },
+    state::AppState,
+};
+
+#[cfg(feature = "wayvr")]
+use crate::{backend::wayvr::WayVRAction, overlays::wayvr::wayvr_action};
+
+pub mod helpers;
+pub mod input;
+pub mod lines;
+pub mod manifest;
+pub mod overlay;
+pub mod playspace;
+
+static FRAME_COUNTER: AtomicUsize = AtomicUsize::new(0);
+
+pub fn openvr_uninstall() {
+    let app_type = EVRApplicationType::VRApplication_Overlay;
+    let Ok(context) = ovr_overlay::Context::init(app_type) else {
+        log::error!("Uninstall failed: could not reach OpenVR");
+        return;
+    };
+
+    let mut app_mgr = context.applications_mngr();
+    let _ = uninstall_manifest(&mut app_mgr);
+}
+
+#[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
+pub fn openvr_run(
+    running: Arc<AtomicBool>,
+    show_by_default: bool,
+    headless: bool,
+) -> Result<(), BackendError> {
+    let app_type = EVRApplicationType::VRApplication_Overlay;
+    let Ok(context) = ovr_overlay::Context::init(app_type) else {
+        log::warn!("Will not use OpenVR: Context init failed");
+        return Err(BackendError::NotSupported);
+    };
+
+    log::info!("Using OpenVR runtime");
+
+    let mut app_mgr = context.applications_mngr();
+    let mut input_mgr = context.input_mngr();
+    let mut system_mgr = context.system_mngr();
+    let mut overlay_mgr = context.overlay_mngr();
+    let mut settings_mgr = context.settings_mngr();
+    let mut chaperone_mgr = context.chaperone_setup_mngr();
+    let mut compositor_mgr = context.compositor_mngr();
+
+    let device_extensions_fn = |device: &PhysicalDevice| {
+        let names = compositor_mgr.get_vulkan_device_extensions_required(device.handle().as_raw());
+        names.iter().map(std::string::String::as_str).collect()
+    };
+
+    let mut compositor_mgr = context.compositor_mngr();
+    let instance_extensions = {
+        let names = compositor_mgr.get_vulkan_instance_extensions_required();
+        names.iter().map(std::string::String::as_str).collect()
+    };
+
+    let mut state = {
+        let (gfx, gfx_extras) = init_openvr_graphics(instance_extensions, device_extensions_fn)?;
+        AppState::from_graphics(gfx, gfx_extras)?
+    };
+
+    if show_by_default {
+        state.tasks.enqueue_at(
+            TaskType::System(SystemTask::ShowHide),
+            Instant::now().add(Duration::from_secs(1)),
+        );
+    }
+
+    if let Ok(ipd) = system_mgr.get_tracked_device_property::<f32>(
+        TrackedDeviceIndex::HMD,
+        ETrackedDeviceProperty::Prop_UserIpdMeters_Float,
+    ) {
+        state.input_state.ipd = (ipd * 10000.0).round() * 0.1;
+        log::info!("IPD: {:.1} mm", state.input_state.ipd);
+    }
+
+    let _ = install_manifest(&mut app_mgr);
+
+    let mut overlays = OverlayContainer::<OpenVrOverlayData>::new(&mut state, headless)?;
+    let mut notifications = NotificationManager::new();
+    notifications.run_dbus();
+    notifications.run_udp();
+
+    let mut playspace = playspace::PlayspaceMover::new();
+    playspace.playspace_changed(&mut compositor_mgr, &mut chaperone_mgr);
+
+    set_action_manifest(&mut input_mgr)?;
+
+    let mut input_source = OpenVrInputSource::new(&mut input_mgr)?;
+
+    let Ok(refresh_rate) = system_mgr.get_tracked_device_property::<f32>(
+        TrackedDeviceIndex::HMD,
+        ETrackedDeviceProperty::Prop_DisplayFrequency_Float,
+    ) else {
+        return Err(BackendError::Fatal(anyhow!(
+            "Failed to get HMD refresh rate"
+        )));
+    };
+
+    log::info!("HMD running @ {refresh_rate} Hz");
+
+    let watch_id = overlays.get_by_name(WATCH_NAME).unwrap().state.id; // want panic
+
+    // want at least half refresh rate
+    let frame_timeout = 2 * (1000.0 / refresh_rate).floor() as u32;
+
+    let mut next_device_update = Instant::now();
+    let mut due_tasks = VecDeque::with_capacity(4);
+
+    let mut lines = LinePool::new(state.gfx.clone())?;
+    let pointer_lines = [lines.allocate(), lines.allocate()];
+
+    'main_loop: loop {
+        let _ = overlay_mgr.wait_frame_sync(frame_timeout);
+
+        if !running.load(Ordering::Relaxed) {
+            log::warn!("Received shutdown signal.");
+            break 'main_loop;
+        }
+
+        let cur_frame = FRAME_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+        while let Some(event) = system_mgr.poll_next_event() {
+            match event.event_type {
+                EVREventType::VREvent_Quit => {
+                    log::warn!("Received quit event, shutting down.");
+                    break 'main_loop;
+                }
+                EVREventType::VREvent_TrackedDeviceActivated
+                | EVREventType::VREvent_TrackedDeviceDeactivated
+                | EVREventType::VREvent_TrackedDeviceUpdated => {
+                    next_device_update = Instant::now();
+                }
+                EVREventType::VREvent_SeatedZeroPoseReset
+                | EVREventType::VREvent_StandingZeroPoseReset
+                | EVREventType::VREvent_ChaperoneUniverseHasChanged
+                | EVREventType::VREvent_SceneApplicationChanged => {
+                    playspace.playspace_changed(&mut compositor_mgr, &mut chaperone_mgr);
+                }
+                EVREventType::VREvent_IpdChanged => {
+                    if let Ok(ipd) = system_mgr.get_tracked_device_property::<f32>(
+                        TrackedDeviceIndex::HMD,
+                        ETrackedDeviceProperty::Prop_UserIpdMeters_Float,
+                    ) {
+                        let ipd = (ipd * 10000.0).round() * 0.1;
+                        if (ipd - state.input_state.ipd).abs() > 0.05 {
+                            log::info!("IPD: {:.1} mm -> {:.1} mm", state.input_state.ipd, ipd);
+                            Toast::new(
+                                ToastTopic::IpdChange,
+                                "IPD".into(),
+                                format!("{ipd:.1} mm").into(),
+                            )
+                            .submit(&mut state);
+                        }
+                        state.input_state.ipd = ipd;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        if next_device_update <= Instant::now() {
+            input_source.update_devices(&mut system_mgr, &mut state);
+            next_device_update = Instant::now() + Duration::from_secs(30);
+        }
+
+        notifications.submit_pending(&mut state);
+
+        state.tasks.retrieve_due(&mut due_tasks);
+
+        let mut removed_overlays = overlays.update(&mut state)?;
+        for o in &mut removed_overlays {
+            o.destroy(&mut overlay_mgr);
+        }
+
+        while let Some(task) = due_tasks.pop_front() {
+            match task {
+                TaskType::Overlay(sel, f) => {
+                    if let Some(o) = overlays.mut_by_selector(&sel) {
+                        f(&mut state, &mut o.state);
+                    } else {
+                        log::warn!("Overlay not found for task: {sel:?}");
+                    }
+                }
+                TaskType::CreateOverlay(sel, f) => {
+                    let None = overlays.mut_by_selector(&sel) else {
+                        continue;
+                    };
+
+                    let Some((mut state, backend)) = f(&mut state) else {
+                        continue;
+                    };
+                    state.birthframe = cur_frame;
+
+                    overlays.add(OverlayData {
+                        state,
+                        backend,
+                        ..Default::default()
+                    });
+                }
+                TaskType::DropOverlay(sel) => {
+                    if let Some(o) = overlays.mut_by_selector(&sel) {
+                        if o.state.birthframe < cur_frame {
+                            o.destroy(&mut overlay_mgr);
+                            overlays.remove_by_selector(&sel);
+                        }
+                    }
+                }
+                TaskType::System(task) => match task {
+                    SystemTask::ColorGain(channel, value) => {
+                        let _ = adjust_gain(&mut settings_mgr, channel, value);
+                    }
+                    SystemTask::FixFloor => {
+                        playspace.fix_floor(&mut chaperone_mgr, &state.input_state);
+                    }
+                    SystemTask::ResetPlayspace => {
+                        playspace.reset_offset(&mut chaperone_mgr, &state.input_state);
+                    }
+                    SystemTask::ShowHide => {
+                        overlays.show_hide(&mut state);
+                    }
+                },
+                #[cfg(feature = "wayvr")]
+                TaskType::WayVR(action) => {
+                    wayvr_action(&mut state, &mut overlays, &action);
+                }
+            }
+        }
+
+        let universe = playspace.get_universe();
+
+        state.input_state.pre_update();
+        input_source.update(
+            universe.clone(),
+            &mut input_mgr,
+            &mut system_mgr,
+            &mut state,
+        );
+        state.input_state.post_update(&state.session);
+
+        if state
+            .input_state
+            .pointers
+            .iter()
+            .any(|p| p.now.show_hide && !p.before.show_hide)
+        {
+            lines.mark_dirty(); // workaround to prevent lines from not showing
+            overlays.show_hide(&mut state);
+        }
+
+        #[cfg(feature = "wayvr")]
+        if state
+            .input_state
+            .pointers
+            .iter()
+            .any(|p| p.now.toggle_dashboard && !p.before.toggle_dashboard)
+        {
+            wayvr_action(&mut state, &mut overlays, &WayVRAction::ToggleDashboard);
+        }
+
+        overlays
+            .iter_mut()
+            .for_each(|o| o.state.auto_movement(&mut state));
+
+        watch_fade(&mut state, overlays.mut_by_id(watch_id).unwrap()); // want panic
+        playspace.update(&mut chaperone_mgr, &mut overlays, &state);
+
+        let lengths_haptics = interact(&mut overlays, &mut state);
+        for (idx, (len, haptics)) in lengths_haptics.iter().enumerate() {
+            lines.draw_from(
+                pointer_lines[idx],
+                state.input_state.pointers[idx].pose,
+                *len,
+                state.input_state.pointers[idx].interaction.mode as usize + 1,
+                &state.input_state.hmd,
+            );
+            if let Some(haptics) = haptics {
+                input_source.haptics(&mut input_mgr, idx, haptics);
+            }
+        }
+
+        state.hid_provider.commit();
+        let mut buffers = CommandBuffers::default();
+
+        lines.update(universe.clone(), &mut overlay_mgr, &mut state)?;
+
+        for o in overlays.iter_mut() {
+            o.after_input(&mut overlay_mgr, &mut state)?;
+        }
+
+        #[cfg(feature = "osc")]
+        if let Some(ref mut sender) = state.osc_sender {
+            let _ = sender.send_params(&overlays, &state.input_state.devices);
+        }
+
+        #[cfg(feature = "wayvr")]
+        if let Err(e) =
+            crate::overlays::wayvr::tick_events::<OpenVrOverlayData>(&mut state, &mut overlays)
+        {
+            log::error!("WayVR tick_events failed: {e:?}");
+        }
+
+        log::trace!("Rendering frame");
+
+        for o in overlays.iter_mut() {
+            if o.state.want_visible {
+                let ShouldRender::Should = o.should_render(&mut state)? else {
+                    continue;
+                };
+                if !o.ensure_image_allocated(&mut state)? {
+                    continue;
+                }
+                o.data.image_dirty = o.render(
+                    &mut state,
+                    o.data.image_view.as_ref().unwrap().clone(),
+                    &mut buffers,
+                    1.0, // alpha is instead set using OVR API
+                )?;
+            }
+        }
+
+        log::trace!("Rendering overlays");
+
+        if let Some(mut future) = buffers.execute_now(state.gfx.queue_gfx.clone())? {
+            if let Err(e) = future.flush() {
+                return Err(BackendError::Fatal(e.into()));
+            }
+            future.cleanup_finished();
+        }
+
+        overlays
+            .iter_mut()
+            .for_each(|o| o.after_render(universe.clone(), &mut overlay_mgr, &state.gfx));
+
+        #[cfg(feature = "wayvr")]
+        if let Some(wayvr) = &state.wayvr {
+            wayvr.borrow_mut().data.tick_finish()?;
+        }
+
+        // chaperone
+
+        // close font handles?
+    }
+
+    log::warn!("OpenVR shutdown");
+    // context.shutdown() called by Drop
+
+    Ok(())
+}

wlx-overlay-s/src/backend/openvr/overlay.rs

@@ -0,0 +1,299 @@
+use core::f32;
+use std::sync::Arc;
+
+use glam::Vec4;
+use ovr_overlay::{
+    overlay::{OverlayHandle, OverlayManager},
+    pose::Matrix3x4,
+    sys::{ETrackingUniverseOrigin, VRVulkanTextureData_t},
+};
+use vulkano::{
+    image::{view::ImageView, ImageUsage},
+    Handle, VulkanObject,
+};
+use wgui::gfx::WGfx;
+
+use crate::{backend::overlay::OverlayData, state::AppState};
+
+use super::helpers::Affine3AConvert;
+
+#[derive(Default)]
+pub(super) struct OpenVrOverlayData {
+    pub(super) handle: Option<OverlayHandle>,
+    pub(super) visible: bool,
+    pub(super) color: Vec4,
+    pub(crate) width: f32,
+    pub(super) override_width: bool,
+    pub(super) image_view: Option<Arc<ImageView>>,
+    pub(super) image_dirty: bool,
+}
+
+impl OverlayData<OpenVrOverlayData> {
+    pub(super) fn initialize(
+        &mut self,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> anyhow::Result<OverlayHandle> {
+        let key = format!("wlx-{}", self.state.name);
+        log::debug!("Create overlay with key: {}", &key);
+        let handle = match overlay.create_overlay(&key, &key) {
+            Ok(handle) => handle,
+            Err(e) => {
+                panic!("Failed to create overlay: {e}");
+            }
+        };
+        log::debug!("{}: initialize", self.state.name);
+
+        self.data.handle = Some(handle);
+        self.data.color = Vec4::ONE;
+
+        self.init(app)?;
+
+        if self.data.width < f32::EPSILON {
+            self.data.width = 1.0;
+        }
+
+        self.upload_width(overlay);
+        self.upload_color(overlay);
+        self.upload_alpha(overlay);
+        self.upload_curvature(overlay);
+        self.upload_sort_order(overlay);
+
+        Ok(handle)
+    }
+
+    pub(super) fn ensure_image_allocated(&mut self, app: &mut AppState) -> anyhow::Result<bool> {
+        if self.data.image_view.is_some() {
+            return Ok(true);
+        }
+        let Some(meta) = self.backend.frame_meta() else {
+            return Ok(false);
+        };
+        let image = app.gfx.new_image(
+            meta.extent[0],
+            meta.extent[1],
+            app.gfx.surface_format,
+            ImageUsage::TRANSFER_SRC | ImageUsage::COLOR_ATTACHMENT | ImageUsage::SAMPLED,
+        )?;
+        self.data.image_view = Some(ImageView::new_default(image)?);
+        Ok(true)
+    }
+
+    pub(super) fn after_input(
+        &mut self,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> anyhow::Result<()> {
+        if self.state.want_visible && !self.data.visible {
+            self.show_internal(overlay, app)?;
+        } else if !self.state.want_visible && self.data.visible {
+            self.hide_internal(overlay, app)?;
+        }
+        Ok(())
+    }
+
+    pub(super) fn after_render(
+        &mut self,
+        universe: ETrackingUniverseOrigin,
+        overlay: &mut OverlayManager,
+        graphics: &WGfx,
+    ) {
+        if self.data.visible {
+            if self.state.dirty {
+                self.upload_curvature(overlay);
+
+                self.upload_transform(universe, overlay);
+                self.upload_alpha(overlay);
+                self.state.dirty = false;
+            }
+            self.upload_texture(overlay, graphics);
+        }
+    }
+
+    fn show_internal(
+        &mut self,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> anyhow::Result<()> {
+        let handle = match self.data.handle {
+            Some(handle) => handle,
+            None => self.initialize(overlay, app)?,
+        };
+        log::debug!("{}: show", self.state.name);
+        if let Err(e) = overlay.set_visibility(handle, true) {
+            log::error!("{}: Failed to show overlay: {}", self.state.name, e);
+        }
+        self.data.visible = true;
+        self.backend.resume(app)
+    }
+
+    fn hide_internal(
+        &mut self,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> anyhow::Result<()> {
+        let Some(handle) = self.data.handle else {
+            return Ok(());
+        };
+        log::debug!("{}: hide", self.state.name);
+        if let Err(e) = overlay.set_visibility(handle, false) {
+            log::error!("{}: Failed to hide overlay: {}", self.state.name, e);
+        }
+        self.data.visible = false;
+        self.backend.pause(app)
+    }
+
+    pub(super) fn upload_alpha(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_opacity(handle, self.state.alpha) {
+            log::error!("{}: Failed to set overlay alpha: {}", self.state.name, e);
+        }
+    }
+
+    pub(super) fn upload_color(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_tint(
+            handle,
+            ovr_overlay::ColorTint {
+                r: self.data.color.x,
+                g: self.data.color.y,
+                b: self.data.color.z,
+                a: self.data.color.w,
+            },
+        ) {
+            log::error!("{}: Failed to set overlay tint: {}", self.state.name, e);
+        }
+    }
+
+    fn upload_width(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_width(handle, self.data.width) {
+            log::error!("{}: Failed to set overlay width: {}", self.state.name, e);
+        }
+    }
+
+    fn upload_curvature(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_curvature(handle, self.state.curvature.unwrap_or(0.0)) {
+            log::error!(
+                "{}: Failed to set overlay curvature: {}",
+                self.state.name,
+                e
+            );
+        }
+    }
+
+    fn upload_sort_order(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_sort_order(handle, self.state.z_order) {
+            log::error!("{}: Failed to set overlay z order: {}", self.state.name, e);
+        }
+    }
+
+    pub(super) fn upload_transform(
+        &mut self,
+        universe: ETrackingUniverseOrigin,
+        overlay: &mut OverlayManager,
+    ) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+
+        let effective = self.state.transform
+            * self
+                .backend
+                .frame_meta()
+                .map(|f| f.transform)
+                .unwrap_or_default();
+
+        let transform = Matrix3x4::from_affine(&effective);
+
+        if let Err(e) = overlay.set_transform_absolute(handle, universe, &transform) {
+            log::error!(
+                "{}: Failed to set overlay transform: {}",
+                self.state.name,
+                e
+            );
+        }
+    }
+
+    pub(super) fn upload_texture(&mut self, overlay: &mut OverlayManager, graphics: &WGfx) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+
+        let Some(view) = self.data.image_view.as_ref() else {
+            log::debug!("{}: Not rendered", self.state.name);
+            return;
+        };
+
+        if !self.data.image_dirty {
+            return;
+        }
+        self.data.image_dirty = false;
+
+        let image = view.image().clone();
+        let dimensions = image.extent();
+        if !self.data.override_width {
+            let new_width = ((dimensions[0] as f32) / (dimensions[1] as f32)).min(1.0);
+            if (new_width - self.data.width).abs() > f32::EPSILON {
+                log::info!("{}: New width {}", self.state.name, new_width);
+                self.data.width = new_width;
+                self.upload_width(overlay);
+            }
+        }
+
+        let raw_image = image.handle().as_raw();
+        let format = image.format();
+
+        let mut texture = VRVulkanTextureData_t {
+            m_nImage: raw_image,
+            m_nFormat: format as _,
+            m_nWidth: dimensions[0],
+            m_nHeight: dimensions[1],
+            m_nSampleCount: image.samples() as u32,
+            m_pDevice: graphics.device.handle().as_raw() as *mut _,
+            m_pPhysicalDevice: graphics.device.physical_device().handle().as_raw() as *mut _,
+            m_pInstance: graphics.instance.handle().as_raw() as *mut _,
+            m_pQueue: graphics.queue_gfx.handle().as_raw() as *mut _,
+            m_nQueueFamilyIndex: graphics.queue_gfx.queue_family_index(),
+        };
+        log::trace!(
+            "{}: UploadTex {:?}, {}x{}, {:?}",
+            self.state.name,
+            format,
+            texture.m_nWidth,
+            texture.m_nHeight,
+            image.usage()
+        );
+        if let Err(e) = overlay.set_image_vulkan(handle, &mut texture) {
+            log::error!("{}: Failed to set overlay texture: {}", self.state.name, e);
+        }
+    }
+
+    pub(super) fn destroy(&mut self, overlay: &mut OverlayManager) {
+        if let Some(handle) = self.data.handle {
+            log::debug!("{}: destroy", self.state.name);
+            if let Err(e) = overlay.destroy_overlay(handle) {
+                log::error!("{}: Failed to destroy overlay: {}", self.state.name, e);
+            }
+        }
+    }
+}

wlx-overlay-s/src/backend/openvr/playspace.rs

@@ -0,0 +1,302 @@
+use glam::{Affine3A, Quat, Vec3, Vec3A};
+use ovr_overlay::{
+    chaperone_setup::ChaperoneSetupManager,
+    compositor::CompositorManager,
+    sys::{EChaperoneConfigFile, ETrackingUniverseOrigin, HmdMatrix34_t},
+};
+
+use crate::{
+    backend::{common::OverlayContainer, input::InputState},
+    state::AppState,
+};
+
+use super::{helpers::Affine3AConvert, overlay::OpenVrOverlayData};
+
+struct MoverData<T> {
+    pose: Affine3A,
+    hand: usize,
+    hand_pose: T,
+}
+
+pub(super) struct PlayspaceMover {
+    universe: ETrackingUniverseOrigin,
+    drag: Option<MoverData<Vec3A>>,
+    rotate: Option<MoverData<Quat>>,
+}
+
+impl PlayspaceMover {
+    pub const fn new() -> Self {
+        Self {
+            universe: ETrackingUniverseOrigin::TrackingUniverseRawAndUncalibrated,
+            drag: None,
+            rotate: None,
+        }
+    }
+
+    #[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
+    pub fn update(
+        &mut self,
+        chaperone_mgr: &mut ChaperoneSetupManager,
+        overlays: &mut OverlayContainer<OpenVrOverlayData>,
+        state: &AppState,
+    ) {
+        let universe = self.universe.clone();
+
+        if let Some(data) = self.rotate.as_mut() {
+            let pointer = &state.input_state.pointers[data.hand];
+            if !pointer.now.space_rotate {
+                self.rotate = None;
+                log::info!("End space rotate");
+                return;
+            }
+
+            let new_hand =
+                Quat::from_affine3(&(data.pose * state.input_state.pointers[data.hand].raw_pose));
+
+            let dq = new_hand * data.hand_pose.conjugate();
+            let rel_y = f32::atan2(
+                2.0 * dq.y.mul_add(dq.w, dq.x * dq.z),
+                2.0f32.mul_add(dq.w.mul_add(dq.w, dq.x * dq.x), -1.0),
+            );
+
+            let mut space_transform = Affine3A::from_rotation_y(rel_y);
+            let offset = (space_transform.transform_vector3a(state.input_state.hmd.translation)
+                - state.input_state.hmd.translation)
+                * -1.0;
+            let mut overlay_transform = Affine3A::from_rotation_y(-rel_y);
+
+            overlay_transform.translation = offset;
+            space_transform.translation = offset;
+
+            overlays.iter_mut().for_each(|overlay| {
+                if overlay.state.grabbable {
+                    overlay.state.dirty = true;
+                    overlay.state.transform.translation =
+                        overlay_transform.transform_point3a(overlay.state.transform.translation);
+                }
+            });
+
+            data.pose *= space_transform;
+            data.hand_pose = new_hand;
+
+            if self.universe == ETrackingUniverseOrigin::TrackingUniverseStanding {
+                apply_chaperone_transform(space_transform.inverse(), chaperone_mgr);
+            }
+            set_working_copy(&universe, chaperone_mgr, &data.pose);
+            chaperone_mgr.commit_working_copy(EChaperoneConfigFile::EChaperoneConfigFile_Live);
+        } else {
+            for (i, pointer) in state.input_state.pointers.iter().enumerate() {
+                if pointer.now.space_rotate {
+                    let Some(mat) = get_working_copy(&universe, chaperone_mgr) else {
+                        log::warn!("Can't space rotate - failed to get zero pose");
+                        return;
+                    };
+                    let hand_pose = Quat::from_affine3(&(mat * pointer.raw_pose));
+                    self.rotate = Some(MoverData {
+                        pose: mat,
+                        hand: i,
+                        hand_pose,
+                    });
+                    self.drag = None;
+                    log::info!("Start space rotate");
+                    return;
+                }
+            }
+        }
+
+        if let Some(data) = self.drag.as_mut() {
+            let pointer = &state.input_state.pointers[data.hand];
+            if !pointer.now.space_drag {
+                self.drag = None;
+                log::info!("End space drag");
+                return;
+            }
+
+            let new_hand = data
+                .pose
+                .transform_point3a(state.input_state.pointers[data.hand].raw_pose.translation);
+            let relative_pos =
+                (new_hand - data.hand_pose) * state.session.config.space_drag_multiplier;
+
+            if relative_pos.length_squared() > 1000.0 {
+                log::warn!("Space drag too fast, ignoring");
+                return;
+            }
+
+            let overlay_offset = data.pose.inverse().transform_vector3a(relative_pos) * -1.0;
+
+            overlays.iter_mut().for_each(|overlay| {
+                if overlay.state.grabbable {
+                    overlay.state.dirty = true;
+                    overlay.state.transform.translation += overlay_offset;
+                }
+            });
+
+            data.pose.translation += relative_pos;
+            data.hand_pose = new_hand;
+
+            if self.universe == ETrackingUniverseOrigin::TrackingUniverseStanding {
+                apply_chaperone_offset(overlay_offset, chaperone_mgr);
+            }
+            set_working_copy(&universe, chaperone_mgr, &data.pose);
+            chaperone_mgr.commit_working_copy(EChaperoneConfigFile::EChaperoneConfigFile_Live);
+        } else {
+            for (i, pointer) in state.input_state.pointers.iter().enumerate() {
+                if pointer.now.space_drag {
+                    let Some(mat) = get_working_copy(&universe, chaperone_mgr) else {
+                        log::warn!("Can't space drag - failed to get zero pose");
+                        return;
+                    };
+                    let hand_pos = mat.transform_point3a(pointer.raw_pose.translation);
+                    self.drag = Some(MoverData {
+                        pose: mat,
+                        hand: i,
+                        hand_pose: hand_pos,
+                    });
+                    self.rotate = None;
+                    log::info!("Start space drag");
+                    return;
+                }
+            }
+        }
+    }
+
+    pub fn reset_offset(&mut self, chaperone_mgr: &mut ChaperoneSetupManager, input: &InputState) {
+        let mut height = 1.6;
+        if let Some(mat) = get_working_copy(&self.universe, chaperone_mgr) {
+            height = input.hmd.translation.y - mat.translation.y;
+            if self.universe == ETrackingUniverseOrigin::TrackingUniverseStanding {
+                apply_chaperone_transform(mat, chaperone_mgr);
+            }
+        }
+
+        let xform = if self.universe == ETrackingUniverseOrigin::TrackingUniverseSeated {
+            Affine3A::from_translation(Vec3::NEG_Y * height)
+        } else {
+            Affine3A::IDENTITY
+        };
+
+        set_working_copy(&self.universe, chaperone_mgr, &xform);
+        chaperone_mgr.commit_working_copy(EChaperoneConfigFile::EChaperoneConfigFile_Live);
+
+        if self.drag.is_some() {
+            log::info!("Space drag interrupted by manual reset");
+            self.drag = None;
+        }
+        if self.rotate.is_some() {
+            log::info!("Space rotate interrupted by manual reset");
+            self.rotate = None;
+        }
+    }
+
+    pub fn fix_floor(&mut self, chaperone_mgr: &mut ChaperoneSetupManager, input: &InputState) {
+        let y1 = input.pointers[0].pose.translation.y;
+        let y2 = input.pointers[1].pose.translation.y;
+        let Some(mut mat) = get_working_copy(&self.universe, chaperone_mgr) else {
+            log::warn!("Can't fix floor - failed to get zero pose");
+            return;
+        };
+        let offset = y1.min(y2) - 0.03;
+        mat.translation.y += offset;
+
+        set_working_copy(&self.universe, chaperone_mgr, &mat);
+        chaperone_mgr.commit_working_copy(EChaperoneConfigFile::EChaperoneConfigFile_Live);
+
+        if self.drag.is_some() {
+            log::info!("Space drag interrupted by fix floor");
+            self.drag = None;
+        }
+        if self.rotate.is_some() {
+            log::info!("Space rotate interrupted by fix floor");
+            self.rotate = None;
+        }
+    }
+
+    pub fn playspace_changed(
+        &mut self,
+        compositor_mgr: &mut CompositorManager,
+        _chaperone_mgr: &mut ChaperoneSetupManager,
+    ) {
+        let new_universe = compositor_mgr.get_tracking_space();
+        if new_universe != self.universe {
+            log::info!(
+                "Playspace changed: {} -> {}",
+                universe_str(&self.universe),
+                universe_str(&new_universe)
+            );
+            self.universe = new_universe;
+        }
+
+        if self.drag.is_some() {
+            log::info!("Space drag interrupted by external change");
+            self.drag = None;
+        }
+        if self.rotate.is_some() {
+            log::info!("Space rotate interrupted by external change");
+            self.rotate = None;
+        }
+    }
+
+    pub fn get_universe(&self) -> ETrackingUniverseOrigin {
+        self.universe.clone()
+    }
+}
+
+const fn universe_str(universe: &ETrackingUniverseOrigin) -> &'static str {
+    match universe {
+        ETrackingUniverseOrigin::TrackingUniverseSeated => "Seated",
+        ETrackingUniverseOrigin::TrackingUniverseStanding => "Standing",
+        ETrackingUniverseOrigin::TrackingUniverseRawAndUncalibrated => "Raw",
+    }
+}
+
+fn get_working_copy(
+    universe: &ETrackingUniverseOrigin,
+    chaperone_mgr: &mut ChaperoneSetupManager,
+) -> Option<Affine3A> {
+    chaperone_mgr.revert_working_copy();
+    let mat = match universe {
+        ETrackingUniverseOrigin::TrackingUniverseStanding => {
+            chaperone_mgr.get_working_standing_zero_pose_to_raw_tracking_pose()
+        }
+        _ => chaperone_mgr.get_working_seated_zero_pose_to_raw_tracking_pose(),
+    };
+    mat.map(|m| m.to_affine())
+}
+
+fn set_working_copy(
+    universe: &ETrackingUniverseOrigin,
+    chaperone_mgr: &mut ChaperoneSetupManager,
+    mat: &Affine3A,
+) {
+    let mat = HmdMatrix34_t::from_affine(mat);
+    match universe {
+        ETrackingUniverseOrigin::TrackingUniverseStanding => {
+            chaperone_mgr.set_working_standing_zero_pose_to_raw_tracking_pose(&mat);
+        }
+        _ => chaperone_mgr.set_working_seated_zero_pose_to_raw_tracking_pose(&mat),
+    }
+}
+
+fn apply_chaperone_offset(offset: Vec3A, chaperone_mgr: &mut ChaperoneSetupManager) {
+    let mut quads = chaperone_mgr.get_live_collision_bounds_info();
+    for quad in &mut quads {
+        quad.vCorners.iter_mut().for_each(|corner| {
+            corner.v[0] += offset.x;
+            corner.v[2] += offset.z;
+        });
+    }
+    chaperone_mgr.set_working_collision_bounds_info(quads.as_mut_slice());
+}
+
+fn apply_chaperone_transform(transform: Affine3A, chaperone_mgr: &mut ChaperoneSetupManager) {
+    let mut quads = chaperone_mgr.get_live_collision_bounds_info();
+    for quad in &mut quads {
+        quad.vCorners.iter_mut().for_each(|corner| {
+            let coord = transform.transform_point3a(Vec3A::from_slice(&corner.v));
+            corner.v[0] = coord.x;
+            corner.v[2] = coord.z;
+        });
+    }
+    chaperone_mgr.set_working_collision_bounds_info(quads.as_mut_slice());
+}

wlx-overlay-s/src/backend/openxr/blocker.rs

@@ -0,0 +1,73 @@
+use libmonado::{ClientState, Monado};
+use log::{info, warn};
+
+use crate::{backend::overlay::OverlayID, state::AppState};
+
+pub(super) struct InputBlocker {
+    hovered_last_frame: bool,
+}
+
+impl InputBlocker {
+    pub const fn new() -> Self {
+        Self {
+            hovered_last_frame: false,
+        }
+    }
+
+    pub fn update(&mut self, state: &AppState, watch_id: OverlayID, monado: &mut Monado) {
+        if !state.session.config.block_game_input {
+            return;
+        }
+
+        let any_hovered = state.input_state.pointers.iter().any(|p| {
+            p.interaction.hovered_id.is_some_and(|id| {
+                id != watch_id || !state.session.config.block_game_input_ignore_watch
+            })
+        });
+
+        match (any_hovered, self.hovered_last_frame) {
+            (true, false) => {
+                info!("Blocking input");
+                set_clients_io_active(monado, false);
+            }
+            (false, true) => {
+                info!("Unblocking input");
+                set_clients_io_active(monado, true);
+            }
+            _ => {}
+        }
+
+        self.hovered_last_frame = any_hovered;
+    }
+}
+
+fn set_clients_io_active(monado: &mut Monado, active: bool) {
+    match monado.clients() {
+        Ok(clients) => {
+            for mut client in clients {
+                let name = match client.name() {
+                    Ok(n) => n,
+                    Err(e) => {
+                        warn!("Failed to get client name: {e}");
+                        continue;
+                    }
+                };
+
+                let state = match client.state() {
+                    Ok(s) => s,
+                    Err(e) => {
+                        warn!("Failed to get client state: {e}");
+                        continue;
+                    }
+                };
+
+                if name != "wlx-overlay-s" && state.contains(ClientState::ClientSessionVisible) {
+                    if let Err(e) = client.set_io_active(active) {
+                        warn!("Failed to set io active for client: {e}");
+                    }
+                }
+            }
+        }
+        Err(e) => warn!("Failed to get clients from Monado: {e}"),
+    }
+}

wlx-overlay-s/src/backend/openxr/helpers.rs

@@ -0,0 +1,190 @@
+use anyhow::{bail, ensure};
+use glam::{Affine3A, Quat, Vec3, Vec3A};
+use openxr::{self as xr, SessionCreateFlags, Version};
+use xr::OverlaySessionCreateFlagsEXTX;
+
+pub(super) fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
+    let entry = xr::Entry::linked();
+
+    let Ok(available_extensions) = entry.enumerate_extensions() else {
+        bail!("Failed to enumerate OpenXR extensions.");
+    };
+    ensure!(
+        available_extensions.khr_vulkan_enable2,
+        "Missing KHR_vulkan_enable2 extension."
+    );
+    ensure!(
+        available_extensions.extx_overlay,
+        "Missing EXTX_overlay extension."
+    );
+
+    let mut enabled_extensions = xr::ExtensionSet::default();
+    enabled_extensions.khr_vulkan_enable2 = true;
+    enabled_extensions.extx_overlay = true;
+    if available_extensions.khr_binding_modification && available_extensions.ext_dpad_binding {
+        enabled_extensions.khr_binding_modification = true;
+        enabled_extensions.ext_dpad_binding = true;
+    } else {
+        log::warn!("Missing EXT_dpad_binding extension.");
+    }
+    if available_extensions.ext_hp_mixed_reality_controller {
+        enabled_extensions.ext_hp_mixed_reality_controller = true;
+    } else {
+        log::warn!("Missing EXT_hp_mixed_reality_controller extension.");
+    }
+    if available_extensions.khr_composition_layer_cylinder {
+        enabled_extensions.khr_composition_layer_cylinder = true;
+    } else {
+        log::warn!("Missing EXT_composition_layer_cylinder extension.");
+    }
+    if available_extensions.khr_composition_layer_equirect2 {
+        enabled_extensions.khr_composition_layer_equirect2 = true;
+    } else {
+        log::warn!("Missing EXT_composition_layer_equirect2 extension.");
+    }
+    if available_extensions
+        .other
+        .contains(&"XR_MNDX_system_buttons".to_owned())
+    {
+        enabled_extensions
+            .other
+            .push("XR_MNDX_system_buttons".to_owned());
+    }
+
+    //#[cfg(not(debug_assertions))]
+    let layers = [];
+    //#[cfg(debug_assertions)]
+    //let layers = [
+    //    "XR_APILAYER_LUNARG_api_dump",
+    //    "XR_APILAYER_LUNARG_standard_validation",
+    //];
+
+    let Ok(xr_instance) = entry.create_instance(
+        &xr::ApplicationInfo {
+            api_version: Version::new(1, 1, 37),
+            application_name: "wlx-overlay-s",
+            application_version: 0,
+            engine_name: "wlx-overlay-s",
+            engine_version: 0,
+        },
+        &enabled_extensions,
+        &layers,
+    ) else {
+        bail!("Failed to create OpenXR instance.");
+    };
+
+    let Ok(instance_props) = xr_instance.properties() else {
+        bail!("Failed to query OpenXR instance properties.");
+    };
+    log::info!(
+        "Using OpenXR runtime: {} {}",
+        instance_props.runtime_name,
+        instance_props.runtime_version
+    );
+
+    let Ok(system) = xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY) else {
+        bail!("Failed to access OpenXR HMD system.");
+    };
+
+    let vk_target_version_xr = xr::Version::new(1, 1, 0);
+
+    let Ok(reqs) = xr_instance.graphics_requirements::<xr::Vulkan>(system) else {
+        bail!("Failed to query OpenXR Vulkan requirements.");
+    };
+
+    if vk_target_version_xr < reqs.min_api_version_supported
+        || vk_target_version_xr.major() > reqs.max_api_version_supported.major()
+    {
+        bail!(
+            "OpenXR runtime requires Vulkan version > {}, < {}.0.0",
+            reqs.min_api_version_supported,
+            reqs.max_api_version_supported.major() + 1
+        );
+    }
+
+    Ok((xr_instance, system))
+}
+
+pub(super) unsafe fn create_overlay_session(
+    instance: &xr::Instance,
+    system: xr::SystemId,
+    info: &xr::vulkan::SessionCreateInfo,
+) -> Result<xr::sys::Session, xr::sys::Result> {
+    let overlay = xr::sys::SessionCreateInfoOverlayEXTX {
+        ty: xr::sys::SessionCreateInfoOverlayEXTX::TYPE,
+        next: std::ptr::null(),
+        create_flags: OverlaySessionCreateFlagsEXTX::EMPTY,
+        session_layers_placement: 5,
+    };
+    let binding = xr::sys::GraphicsBindingVulkanKHR {
+        ty: xr::sys::GraphicsBindingVulkanKHR::TYPE,
+        next: (&raw const overlay).cast(),
+        instance: info.instance,
+        physical_device: info.physical_device,
+        device: info.device,
+        queue_family_index: info.queue_family_index,
+        queue_index: info.queue_index,
+    };
+    let info = xr::sys::SessionCreateInfo {
+        ty: xr::sys::SessionCreateInfo::TYPE,
+        next: (&raw const binding).cast(),
+        create_flags: SessionCreateFlags::default(),
+        system_id: system,
+    };
+    let mut out = xr::sys::Session::NULL;
+    let x = (instance.fp().create_session)(instance.as_raw(), &info, &mut out);
+    if x.into_raw() >= 0 {
+        Ok(out)
+    } else {
+        Err(x)
+    }
+}
+
+type Vec3M = mint::Vector3<f32>;
+type QuatM = mint::Quaternion<f32>;
+
+pub(super) fn ipd_from_views(views: &[xr::View]) -> f32 {
+    let p0: Vec3 = Vec3M::from(views[0].pose.position).into();
+    let p1: Vec3 = Vec3M::from(views[1].pose.position).into();
+
+    (p0.distance(p1) * 10000.0).round() * 0.1
+}
+
+pub(super) fn transform_to_norm_quat(transform: &Affine3A) -> Quat {
+    let norm_mat3 = transform
+        .matrix3
+        .mul_scalar(1.0 / transform.matrix3.x_axis.length());
+    Quat::from_mat3a(&norm_mat3).normalize()
+}
+
+pub(super) fn translation_rotation_to_posef(translation: Vec3A, mut rotation: Quat) -> xr::Posef {
+    if !rotation.is_finite() {
+        rotation = Quat::IDENTITY;
+    }
+
+    xr::Posef {
+        orientation: xr::Quaternionf {
+            x: rotation.x,
+            y: rotation.y,
+            z: rotation.z,
+            w: rotation.w,
+        },
+        position: xr::Vector3f {
+            x: translation.x,
+            y: translation.y,
+            z: translation.z,
+        },
+    }
+}
+
+pub(super) fn transform_to_posef(transform: &Affine3A) -> xr::Posef {
+    let translation = transform.translation;
+    let rotation = transform_to_norm_quat(transform);
+    translation_rotation_to_posef(translation, rotation)
+}
+
+pub(super) fn posef_to_transform(pose: &xr::Posef) -> Affine3A {
+    let rotation = QuatM::from(pose.orientation).into();
+    let translation = Vec3M::from(pose.position).into();
+    Affine3A::from_rotation_translation(rotation, translation)
+}

wlx-overlay-s/src/backend/openxr/input.rs

@@ -0,0 +1,692 @@
+use std::{
+    array::from_fn,
+    mem::transmute,
+    time::{Duration, Instant},
+};
+
+use glam::{bool, Affine3A, Quat, Vec3};
+use libmonado as mnd;
+use openxr::{self as xr, Quaternionf, Vector2f, Vector3f};
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    backend::input::{Haptics, Pointer, TrackedDevice, TrackedDeviceRole},
+    config_io,
+    state::{AppSession, AppState},
+};
+
+use super::{helpers::posef_to_transform, XrState};
+
+static CLICK_TIMES: [Duration; 3] = [
+    Duration::ZERO,
+    Duration::from_millis(500),
+    Duration::from_millis(750),
+];
+
+pub(super) struct OpenXrInputSource {
+    action_set: xr::ActionSet,
+    hands: [OpenXrHand; 2],
+}
+
+pub(super) struct OpenXrHand {
+    source: OpenXrHandSource,
+    space: xr::Space,
+}
+
+pub struct MultiClickHandler<const COUNT: usize> {
+    name: String,
+    action_f32: xr::Action<f32>,
+    action_bool: xr::Action<bool>,
+    previous: [Instant; COUNT],
+    held_active: bool,
+    held_inactive: bool,
+}
+
+impl<const COUNT: usize> MultiClickHandler<COUNT> {
+    fn new(action_set: &xr::ActionSet, action_name: &str, side: &str) -> anyhow::Result<Self> {
+        let name = format!("{side}_{COUNT}-{action_name}");
+        let name_f32 = format!("{}_value", &name);
+
+        let action_bool = action_set.create_action::<bool>(&name, &name, &[])?;
+        let action_f32 = action_set.create_action::<f32>(&name_f32, &name_f32, &[])?;
+
+        Ok(Self {
+            name,
+            action_f32,
+            action_bool,
+            previous: from_fn(|_| Instant::now()),
+            held_active: false,
+            held_inactive: false,
+        })
+    }
+    fn check<G>(&mut self, session: &xr::Session<G>, threshold: f32) -> anyhow::Result<bool> {
+        let res = self.action_bool.state(session, xr::Path::NULL)?;
+        let mut state = res.is_active && res.current_state;
+
+        if !state {
+            let res = self.action_f32.state(session, xr::Path::NULL)?;
+            state = res.is_active && res.current_state > threshold;
+        }
+
+        if !state {
+            self.held_active = false;
+            self.held_inactive = false;
+            return Ok(false);
+        }
+
+        if self.held_active {
+            return Ok(true);
+        }
+
+        if self.held_inactive {
+            return Ok(false);
+        }
+
+        let passed = self
+            .previous
+            .iter()
+            .all(|instant| instant.elapsed() < CLICK_TIMES[COUNT]);
+
+        if passed {
+            log::trace!("{}: passed", self.name);
+            self.held_active = true;
+            self.held_inactive = false;
+
+            // reset to no prior clicks
+            let long_ago = Instant::now().checked_sub(Duration::from_secs(10)).unwrap();
+            self.previous
+                .iter_mut()
+                .for_each(|instant| *instant = long_ago);
+        } else if COUNT > 0 {
+            log::trace!("{}: rotate", self.name);
+            self.previous.rotate_right(1);
+            self.previous[0] = Instant::now();
+            self.held_inactive = true;
+        }
+
+        Ok(passed)
+    }
+}
+
+pub struct CustomClickAction {
+    single: MultiClickHandler<0>,
+    double: MultiClickHandler<1>,
+    triple: MultiClickHandler<2>,
+}
+
+impl CustomClickAction {
+    pub fn new(action_set: &xr::ActionSet, name: &str, side: &str) -> anyhow::Result<Self> {
+        let single = MultiClickHandler::new(action_set, name, side)?;
+        let double = MultiClickHandler::new(action_set, name, side)?;
+        let triple = MultiClickHandler::new(action_set, name, side)?;
+
+        Ok(Self {
+            single,
+            double,
+            triple,
+        })
+    }
+    pub fn state(
+        &mut self,
+        before: bool,
+        state: &XrState,
+        session: &AppSession,
+    ) -> anyhow::Result<bool> {
+        let threshold = if before {
+            session.config.xr_click_sensitivity_release
+        } else {
+            session.config.xr_click_sensitivity
+        };
+
+        Ok(self.single.check(&state.session, threshold)?
+            || self.double.check(&state.session, threshold)?
+            || self.triple.check(&state.session, threshold)?)
+    }
+}
+
+pub(super) struct OpenXrHandSource {
+    pose: xr::Action<xr::Posef>,
+    click: CustomClickAction,
+    grab: CustomClickAction,
+    alt_click: CustomClickAction,
+    show_hide: CustomClickAction,
+    toggle_dashboard: CustomClickAction,
+    space_drag: CustomClickAction,
+    space_rotate: CustomClickAction,
+    space_reset: CustomClickAction,
+    modifier_right: CustomClickAction,
+    modifier_middle: CustomClickAction,
+    move_mouse: CustomClickAction,
+    scroll: xr::Action<Vector2f>,
+    haptics: xr::Action<xr::Haptic>,
+}
+
+impl OpenXrInputSource {
+    pub fn new(xr: &XrState) -> anyhow::Result<Self> {
+        let mut action_set =
+            xr.session
+                .instance()
+                .create_action_set("wlx-overlay-s", "WlxOverlay-S Actions", 0)?;
+
+        let left_source = OpenXrHandSource::new(&mut action_set, "left")?;
+        let right_source = OpenXrHandSource::new(&mut action_set, "right")?;
+
+        suggest_bindings(&xr.instance, &[&left_source, &right_source]);
+
+        xr.session.attach_action_sets(&[&action_set])?;
+
+        Ok(Self {
+            action_set,
+            hands: [
+                OpenXrHand::new(xr, left_source)?,
+                OpenXrHand::new(xr, right_source)?,
+            ],
+        })
+    }
+
+    pub fn haptics(&self, xr: &XrState, hand: usize, haptics: &Haptics) {
+        let action = &self.hands[hand].source.haptics;
+
+        let duration_nanos = f64::from(haptics.duration) * 1_000_000_000.0;
+
+        let _ = action.apply_feedback(
+            &xr.session,
+            xr::Path::NULL,
+            &xr::HapticVibration::new()
+                .amplitude(haptics.intensity)
+                .frequency(haptics.frequency)
+                .duration(xr::Duration::from_nanos(duration_nanos as _)),
+        );
+    }
+
+    pub fn update(&mut self, xr: &XrState, state: &mut AppState) -> anyhow::Result<()> {
+        xr.session.sync_actions(&[(&self.action_set).into()])?;
+
+        let loc = xr.view.locate(&xr.stage, xr.predicted_display_time)?;
+        let hmd = posef_to_transform(&loc.pose);
+        if loc
+            .location_flags
+            .contains(xr::SpaceLocationFlags::ORIENTATION_VALID)
+        {
+            state.input_state.hmd.matrix3 = hmd.matrix3;
+        }
+
+        if loc
+            .location_flags
+            .contains(xr::SpaceLocationFlags::POSITION_VALID)
+        {
+            state.input_state.hmd.translation = hmd.translation;
+        }
+
+        for i in 0..2 {
+            self.hands[i].update(&mut state.input_state.pointers[i], xr, &state.session)?;
+        }
+        Ok(())
+    }
+
+    fn update_device_battery_status(
+        device: &mut mnd::Device,
+        role: TrackedDeviceRole,
+        app: &mut AppState,
+    ) {
+        if let Ok(status) = device.battery_status() {
+            if status.present {
+                app.input_state.devices.push(TrackedDevice {
+                    soc: Some(status.charge),
+                    charging: status.charging,
+                    role,
+                });
+                log::debug!(
+                    "Device {} role {:#?}: {:.0}% (charging {})",
+                    device.index,
+                    role,
+                    status.charge * 100.0f32,
+                    status.charging
+                );
+            }
+        }
+    }
+
+    pub fn update_devices(app: &mut AppState, monado: &mut mnd::Monado) {
+        app.input_state.devices.clear();
+
+        let roles = [
+            (mnd::DeviceRole::Head, TrackedDeviceRole::Hmd),
+            (mnd::DeviceRole::Eyes, TrackedDeviceRole::None),
+            (mnd::DeviceRole::Left, TrackedDeviceRole::LeftHand),
+            (mnd::DeviceRole::Right, TrackedDeviceRole::RightHand),
+            (mnd::DeviceRole::Gamepad, TrackedDeviceRole::None),
+            (
+                mnd::DeviceRole::HandTrackingLeft,
+                TrackedDeviceRole::LeftHand,
+            ),
+            (
+                mnd::DeviceRole::HandTrackingRight,
+                TrackedDeviceRole::RightHand,
+            ),
+        ];
+        let mut seen = Vec::<u32>::with_capacity(32);
+        for (mnd_role, wlx_role) in roles {
+            let device = monado.device_from_role(mnd_role);
+            if let Ok(mut device) = device {
+                if !seen.contains(&device.index) {
+                    seen.push(device.index);
+                    Self::update_device_battery_status(&mut device, wlx_role, app);
+                }
+            }
+        }
+        if let Ok(devices) = monado.devices() {
+            for mut device in devices {
+                if !seen.contains(&device.index) {
+                    let role = if device.name_id >= 4 && device.name_id <= 8 {
+                        TrackedDeviceRole::Tracker
+                    } else {
+                        TrackedDeviceRole::None
+                    };
+                    Self::update_device_battery_status(&mut device, role, app);
+                }
+            }
+        }
+
+        app.input_state.devices.sort_by(|a, b| {
+            u8::from(a.soc.is_none())
+                .cmp(&u8::from(b.soc.is_none()))
+                .then((a.role as u8).cmp(&(b.role as u8)))
+                .then(a.soc.unwrap_or(999.).total_cmp(&b.soc.unwrap_or(999.)))
+        });
+    }
+}
+
+impl OpenXrHand {
+    pub(super) fn new(xr: &XrState, source: OpenXrHandSource) -> Result<Self, xr::sys::Result> {
+        let space = source
+            .pose
+            .create_space(&xr.session, xr::Path::NULL, xr::Posef::IDENTITY)?;
+
+        Ok(Self { source, space })
+    }
+
+    pub(super) fn update(
+        &mut self,
+        pointer: &mut Pointer,
+        xr: &XrState,
+        session: &AppSession,
+    ) -> anyhow::Result<()> {
+        let location = self.space.locate(&xr.stage, xr.predicted_display_time)?;
+        if location
+            .location_flags
+            .contains(xr::SpaceLocationFlags::ORIENTATION_VALID)
+        {
+            let (cur_quat, cur_pos) = (Quat::from_affine3(&pointer.pose), pointer.pose.translation);
+
+            let (new_quat, new_pos) = unsafe {
+                (
+                    transmute::<Quaternionf, Quat>(location.pose.orientation),
+                    transmute::<Vector3f, Vec3>(location.pose.position),
+                )
+            };
+            let lerp_factor =
+                (1.0 / (xr.fps / 100.0) * session.config.pointer_lerp_factor).clamp(0.1, 1.0);
+            pointer.raw_pose = Affine3A::from_rotation_translation(new_quat, new_pos);
+            pointer.pose = Affine3A::from_rotation_translation(
+                cur_quat.lerp(new_quat, lerp_factor),
+                cur_pos.lerp(new_pos.into(), lerp_factor).into(),
+            );
+        }
+
+        pointer.now.click = self.source.click.state(pointer.before.click, xr, session)?;
+
+        pointer.now.grab = self.source.grab.state(pointer.before.grab, xr, session)?;
+
+        let scroll = self
+            .source
+            .scroll
+            .state(&xr.session, xr::Path::NULL)?
+            .current_state;
+
+        pointer.now.scroll_x = scroll.x;
+        pointer.now.scroll_y = scroll.y;
+
+        pointer.now.alt_click =
+            self.source
+                .alt_click
+                .state(pointer.before.alt_click, xr, session)?;
+
+        pointer.now.show_hide =
+            self.source
+                .show_hide
+                .state(pointer.before.show_hide, xr, session)?;
+
+        pointer.now.click_modifier_right =
+            self.source
+                .modifier_right
+                .state(pointer.before.click_modifier_right, xr, session)?;
+
+        pointer.now.toggle_dashboard =
+            self.source
+                .toggle_dashboard
+                .state(pointer.before.toggle_dashboard, xr, session)?;
+
+        pointer.now.click_modifier_middle =
+            self.source
+                .modifier_middle
+                .state(pointer.before.click_modifier_middle, xr, session)?;
+
+        pointer.now.move_mouse =
+            self.source
+                .move_mouse
+                .state(pointer.before.move_mouse, xr, session)?;
+
+        pointer.now.space_drag =
+            self.source
+                .space_drag
+                .state(pointer.before.space_drag, xr, session)?;
+
+        pointer.now.space_rotate =
+            self.source
+                .space_rotate
+                .state(pointer.before.space_rotate, xr, session)?;
+
+        pointer.now.space_reset =
+            self.source
+                .space_reset
+                .state(pointer.before.space_reset, xr, session)?;
+
+        Ok(())
+    }
+}
+
+// supported action types: Haptic, Posef, Vector2f, f32, bool
+impl OpenXrHandSource {
+    pub(super) fn new(action_set: &mut xr::ActionSet, side: &str) -> anyhow::Result<Self> {
+        let action_pose = action_set.create_action::<xr::Posef>(
+            &format!("{side}_hand"),
+            &format!("{side} hand pose"),
+            &[],
+        )?;
+
+        let action_scroll = action_set.create_action::<Vector2f>(
+            &format!("{side}_scroll"),
+            &format!("{side} hand scroll"),
+            &[],
+        )?;
+        let action_haptics = action_set.create_action::<xr::Haptic>(
+            &format!("{side}_haptics"),
+            &format!("{side} hand haptics"),
+            &[],
+        )?;
+
+        Ok(Self {
+            pose: action_pose,
+            click: CustomClickAction::new(action_set, "click", side)?,
+            grab: CustomClickAction::new(action_set, "grab", side)?,
+            scroll: action_scroll,
+            alt_click: CustomClickAction::new(action_set, "alt_click", side)?,
+            show_hide: CustomClickAction::new(action_set, "show_hide", side)?,
+            toggle_dashboard: CustomClickAction::new(action_set, "toggle_dashboard", side)?,
+            space_drag: CustomClickAction::new(action_set, "space_drag", side)?,
+            space_rotate: CustomClickAction::new(action_set, "space_rotate", side)?,
+            space_reset: CustomClickAction::new(action_set, "space_reset", side)?,
+            modifier_right: CustomClickAction::new(action_set, "click_modifier_right", side)?,
+            modifier_middle: CustomClickAction::new(action_set, "click_modifier_middle", side)?,
+            move_mouse: CustomClickAction::new(action_set, "move_mouse", side)?,
+            haptics: action_haptics,
+        })
+    }
+}
+
+fn to_path(maybe_path_str: Option<&String>, instance: &xr::Instance) -> Option<xr::Path> {
+    maybe_path_str.as_ref().and_then(|s| {
+        instance
+            .string_to_path(s)
+            .inspect_err(|_| {
+                log::warn!("Invalid binding path: {s}");
+            })
+            .ok()
+    })
+}
+
+fn is_bool(maybe_type_str: Option<&String>) -> bool {
+    maybe_type_str
+        .as_ref()
+        .unwrap() // want panic
+        .split('/')
+        .next_back()
+        .is_some_and(|last| matches!(last, "click" | "touch") || last.starts_with("dpad_"))
+}
+
+macro_rules! add_custom {
+    ($action:expr, $left:expr, $right:expr, $bindings:expr, $instance:expr) => {
+        if let Some(action) = $action.as_ref() {
+            if let Some(p) = to_path(action.left.as_ref(), $instance) {
+                if is_bool(action.left.as_ref()) {
+                    if action.triple_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$left.triple.action_bool, p));
+                    } else if action.double_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$left.double.action_bool, p));
+                    } else {
+                        $bindings.push(xr::Binding::new(&$left.single.action_bool, p));
+                    }
+                } else {
+                    if action.triple_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$left.triple.action_f32, p));
+                    } else if action.double_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$left.double.action_f32, p));
+                    } else {
+                        $bindings.push(xr::Binding::new(&$left.single.action_f32, p));
+                    }
+                }
+            }
+            if let Some(p) = to_path(action.right.as_ref(), $instance) {
+                if is_bool(action.right.as_ref()) {
+                    if action.triple_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$right.triple.action_bool, p));
+                    } else if action.double_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$right.double.action_bool, p));
+                    } else {
+                        $bindings.push(xr::Binding::new(&$right.single.action_bool, p));
+                    }
+                } else {
+                    if action.triple_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$right.triple.action_f32, p));
+                    } else if action.double_click.unwrap_or(false) {
+                        $bindings.push(xr::Binding::new(&$right.double.action_f32, p));
+                    } else {
+                        $bindings.push(xr::Binding::new(&$right.single.action_f32, p));
+                    }
+                }
+            }
+        }
+    };
+}
+
+#[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
+fn suggest_bindings(instance: &xr::Instance, hands: &[&OpenXrHandSource; 2]) {
+    let profiles = load_action_profiles();
+
+    for profile in profiles {
+        let Ok(profile_path) = instance.string_to_path(&profile.profile) else {
+            log::debug!("Profile not supported: {}", profile.profile);
+            continue;
+        };
+
+        let mut bindings: Vec<xr::Binding> = vec![];
+
+        if let Some(action) = profile.pose {
+            if let Some(p) = to_path(action.left.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[0].pose, p));
+            }
+            if let Some(p) = to_path(action.right.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[1].pose, p));
+            }
+        }
+
+        if let Some(action) = profile.haptic {
+            if let Some(p) = to_path(action.left.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[0].haptics, p));
+            }
+            if let Some(p) = to_path(action.right.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[1].haptics, p));
+            }
+        }
+
+        if let Some(action) = profile.scroll {
+            if let Some(p) = to_path(action.left.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[0].scroll, p));
+            }
+            if let Some(p) = to_path(action.right.as_ref(), instance) {
+                bindings.push(xr::Binding::new(&hands[1].scroll, p));
+            }
+        }
+
+        add_custom!(
+            profile.click,
+            hands[0].click,
+            hands[1].click,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.alt_click,
+            &hands[0].alt_click,
+            &hands[1].alt_click,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.grab,
+            &hands[0].grab,
+            &hands[1].grab,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.show_hide,
+            &hands[0].show_hide,
+            &hands[1].show_hide,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.toggle_dashboard,
+            &hands[0].toggle_dashboard,
+            &hands[1].toggle_dashboard,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.space_drag,
+            &hands[0].space_drag,
+            &hands[1].space_drag,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.space_rotate,
+            &hands[0].space_rotate,
+            &hands[1].space_rotate,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.space_reset,
+            &hands[0].space_reset,
+            &hands[1].space_reset,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.click_modifier_right,
+            &hands[0].modifier_right,
+            &hands[1].modifier_right,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.click_modifier_middle,
+            &hands[0].modifier_middle,
+            &hands[1].modifier_middle,
+            bindings,
+            instance
+        );
+
+        add_custom!(
+            profile.move_mouse,
+            &hands[0].move_mouse,
+            &hands[1].move_mouse,
+            bindings,
+            instance
+        );
+
+        if instance
+            .suggest_interaction_profile_bindings(profile_path, &bindings)
+            .is_err()
+        {
+            log::error!("Bad bindings for {}", &profile.profile[22..]);
+            log::error!("Verify config: ~/.config/wlxoverlay/openxr_actions.json5");
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct OpenXrActionConfAction {
+    left: Option<String>,
+    right: Option<String>,
+    threshold: Option<[f32; 2]>,
+    double_click: Option<bool>,
+    triple_click: Option<bool>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct OpenXrActionConfProfile {
+    profile: String,
+    pose: Option<OpenXrActionConfAction>,
+    click: Option<OpenXrActionConfAction>,
+    grab: Option<OpenXrActionConfAction>,
+    alt_click: Option<OpenXrActionConfAction>,
+    show_hide: Option<OpenXrActionConfAction>,
+    toggle_dashboard: Option<OpenXrActionConfAction>,
+    space_drag: Option<OpenXrActionConfAction>,
+    space_rotate: Option<OpenXrActionConfAction>,
+    space_reset: Option<OpenXrActionConfAction>,
+    click_modifier_right: Option<OpenXrActionConfAction>,
+    click_modifier_middle: Option<OpenXrActionConfAction>,
+    move_mouse: Option<OpenXrActionConfAction>,
+    scroll: Option<OpenXrActionConfAction>,
+    haptic: Option<OpenXrActionConfAction>,
+}
+
+const DEFAULT_PROFILES: &str = include_str!("openxr_actions.json5");
+
+fn load_action_profiles() -> Vec<OpenXrActionConfProfile> {
+    let mut profiles: Vec<OpenXrActionConfProfile> =
+        serde_json5::from_str(DEFAULT_PROFILES).unwrap(); // want panic
+
+    let Some(conf) = config_io::load("openxr_actions.json5") else {
+        return profiles;
+    };
+
+    match serde_json5::from_str::<Vec<OpenXrActionConfProfile>>(&conf) {
+        Ok(override_profiles) => {
+            for new in override_profiles {
+                if let Some(i) = profiles.iter().position(|old| old.profile == new.profile) {
+                    profiles[i] = new;
+                }
+            }
+        }
+        Err(e) => {
+            log::error!("Failed to load openxr_actions.json5: {e}");
+        }
+    }
+
+    profiles
+}

wlx-overlay-s/src/backend/openxr/lines.rs

@@ -0,0 +1,201 @@
+use glam::{Affine3A, Vec3, Vec3A};
+use idmap::IdMap;
+use openxr as xr;
+use std::{
+    f32::consts::PI,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+};
+
+use wgui::gfx::{pipeline::WGfxPipeline, WGfx};
+
+use crate::{
+    backend::openxr::helpers,
+    graphics::{CommandBuffers, ExtentExt, Vert2Uv},
+    state::AppState,
+};
+use vulkano::{
+    command_buffer::CommandBufferUsage, pipeline::graphics::input_assembly::PrimitiveTopology,
+};
+
+use super::{
+    swapchain::{create_swapchain, SwapchainOpts, WlxSwapchain},
+    CompositionLayer, XrState,
+};
+
+static LINE_AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(1);
+pub(super) const LINE_WIDTH: f32 = 0.002;
+
+// TODO customizable colors
+static COLORS: [[f32; 6]; 5] = {
+    [
+        [1., 1., 1., 1., 0., 0.],
+        [0., 0.375, 0.5, 1., 0., 0.],
+        [0.69, 0.188, 0., 1., 0., 0.],
+        [0.375, 0., 0.5, 1., 0., 0.],
+        [1., 0., 0., 1., 0., 0.],
+    ]
+};
+
+pub(super) struct LinePool {
+    lines: IdMap<usize, LineContainer>,
+    pipeline: Arc<WGfxPipeline<Vert2Uv>>,
+}
+
+impl LinePool {
+    pub(super) fn new(app: &AppState) -> anyhow::Result<Self> {
+        let pipeline = app.gfx.create_pipeline(
+            app.gfx_extras.shaders.get("vert_quad").unwrap().clone(), // want panic
+            app.gfx_extras.shaders.get("frag_color").unwrap().clone(), // want panic
+            app.gfx.surface_format,
+            None,
+            PrimitiveTopology::TriangleStrip,
+            false,
+        )?;
+
+        Ok(Self {
+            lines: IdMap::new(),
+            pipeline,
+        })
+    }
+
+    pub(super) fn allocate(&mut self, xr: &XrState, gfx: Arc<WGfx>) -> anyhow::Result<usize> {
+        let id = LINE_AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed);
+
+        let srd = create_swapchain(xr, gfx, [1, 1, 1], SwapchainOpts::new())?;
+        self.lines.insert(
+            id,
+            LineContainer {
+                swapchain: srd,
+                maybe_line: None,
+            },
+        );
+        Ok(id)
+    }
+
+    pub(super) fn draw_from(
+        &mut self,
+        id: usize,
+        mut from: Affine3A,
+        len: f32,
+        color: usize,
+        hmd: &Affine3A,
+    ) {
+        if len < 0.01 {
+            return;
+        }
+
+        debug_assert!(color < COLORS.len());
+
+        let Some(line) = self.lines.get_mut(id) else {
+            log::warn!("Line {id} not found");
+            return;
+        };
+
+        let rotation = Affine3A::from_axis_angle(Vec3::X, PI * 1.5);
+
+        from.translation += from.transform_vector3a(Vec3A::NEG_Z) * (len * 0.5);
+        let mut transform = from * rotation;
+
+        let to_hmd = hmd.translation - from.translation;
+        let sides = [Vec3A::Z, Vec3A::X, Vec3A::NEG_Z, Vec3A::NEG_X];
+        let rotations = [
+            Affine3A::IDENTITY,
+            Affine3A::from_axis_angle(Vec3::Y, PI * 0.5),
+            Affine3A::from_axis_angle(Vec3::Y, PI * -1.0),
+            Affine3A::from_axis_angle(Vec3::Y, PI * 1.5),
+        ];
+        let mut closest = (0, 0.0);
+        for (i, &side) in sides.iter().enumerate() {
+            let dot = to_hmd.dot(transform.transform_vector3a(side));
+            if i == 0 || dot > closest.1 {
+                closest = (i, dot);
+            }
+        }
+
+        transform *= rotations[closest.0];
+
+        let posef = helpers::transform_to_posef(&transform);
+
+        line.maybe_line = Some(Line {
+            color,
+            pose: posef,
+            length: len,
+        });
+    }
+
+    pub(super) fn render(
+        &mut self,
+        app: &AppState,
+        buf: &mut CommandBuffers,
+    ) -> anyhow::Result<()> {
+        for line in self.lines.values_mut() {
+            if let Some(inner) = line.maybe_line.as_mut() {
+                let tgt = line.swapchain.acquire_wait_image()?;
+
+                let set0 = self
+                    .pipeline
+                    .uniform_buffer_upload(0, COLORS[inner.color].to_vec())?;
+
+                let pass = self.pipeline.create_pass(
+                    tgt.extent_f32(),
+                    app.gfx_extras.quad_verts.clone(),
+                    0..4,
+                    0..1,
+                    vec![set0],
+                )?;
+
+                let mut cmd_buffer = app
+                    .gfx
+                    .create_gfx_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+                cmd_buffer.begin_rendering(tgt)?;
+                cmd_buffer.run_ref(&pass)?;
+                cmd_buffer.end_rendering()?;
+
+                buf.push(cmd_buffer.build()?);
+            }
+        }
+
+        Ok(())
+    }
+
+    pub(super) fn present<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+    ) -> anyhow::Result<Vec<CompositionLayer<'a>>> {
+        let mut quads = Vec::new();
+
+        for line in self.lines.values_mut() {
+            line.swapchain.ensure_image_released()?;
+
+            if let Some(inner) = line.maybe_line.take() {
+                let quad = xr::CompositionLayerQuad::new()
+                    .pose(inner.pose)
+                    .sub_image(line.swapchain.get_subimage())
+                    .eye_visibility(xr::EyeVisibility::BOTH)
+                    .space(&xr.stage)
+                    .size(xr::Extent2Df {
+                        width: LINE_WIDTH,
+                        height: inner.length,
+                    });
+
+                quads.push(CompositionLayer::Quad(quad));
+            }
+        }
+
+        Ok(quads)
+    }
+}
+
+pub(super) struct Line {
+    pub(super) color: usize,
+    pub(super) pose: xr::Posef,
+    pub(super) length: f32,
+}
+
+struct LineContainer {
+    swapchain: WlxSwapchain,
+    maybe_line: Option<Line>,
+}

wlx-overlay-s/src/backend/openxr/mod.rs

@@ -0,0 +1,577 @@
+use std::{
+    collections::VecDeque,
+    ops::Add,
+    sync::{
+        atomic::{AtomicBool, AtomicUsize, Ordering},
+        Arc,
+    },
+    time::{Duration, Instant},
+};
+
+use glam::{Affine3A, Vec3};
+use input::OpenXrInputSource;
+use libmonado::Monado;
+use openxr as xr;
+use skybox::create_skybox;
+use vulkano::{Handle, VulkanObject};
+
+use crate::{
+    backend::{
+        common::{BackendError, OverlayContainer},
+        input::interact,
+        notifications::NotificationManager,
+        openxr::{lines::LinePool, overlay::OpenXrOverlayData},
+        overlay::{OverlayData, ShouldRender},
+        task::{SystemTask, TaskType},
+    },
+    graphics::{init_openxr_graphics, CommandBuffers},
+    overlays::{
+        toast::{Toast, ToastTopic},
+        watch::{watch_fade, WATCH_NAME},
+    },
+    state::AppState,
+};
+
+#[cfg(feature = "wayvr")]
+use crate::{backend::wayvr::WayVRAction, overlays::wayvr::wayvr_action};
+
+mod blocker;
+mod helpers;
+mod input;
+mod lines;
+mod overlay;
+mod playspace;
+mod skybox;
+mod swapchain;
+
+const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
+static FRAME_COUNTER: AtomicUsize = AtomicUsize::new(0);
+
+struct XrState {
+    instance: xr::Instance,
+    session: xr::Session<xr::Vulkan>,
+    predicted_display_time: xr::Time,
+    fps: f32,
+    stage: Arc<xr::Space>,
+    view: Arc<xr::Space>,
+}
+
+#[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
+pub fn openxr_run(
+    running: Arc<AtomicBool>,
+    show_by_default: bool,
+    headless: bool,
+) -> Result<(), BackendError> {
+    let (xr_instance, system) = match helpers::init_xr() {
+        Ok((xr_instance, system)) => (xr_instance, system),
+        Err(e) => {
+            log::warn!("Will not use OpenXR: {e}");
+            return Err(BackendError::NotSupported);
+        }
+    };
+
+    let mut app = {
+        let (gfx, gfx_extras) = init_openxr_graphics(xr_instance.clone(), system)?;
+        AppState::from_graphics(gfx, gfx_extras)?
+    };
+
+    let environment_blend_mode = {
+        let modes = xr_instance.enumerate_environment_blend_modes(system, VIEW_TYPE)?;
+        if modes.contains(&xr::EnvironmentBlendMode::ALPHA_BLEND)
+            && app.session.config.use_passthrough
+        {
+            xr::EnvironmentBlendMode::ALPHA_BLEND
+        } else {
+            modes[0]
+        }
+    };
+    log::info!("Using environment blend mode: {environment_blend_mode:?}");
+
+    if show_by_default {
+        app.tasks.enqueue_at(
+            TaskType::System(SystemTask::ShowHide),
+            Instant::now().add(Duration::from_secs(1)),
+        );
+    }
+
+    let mut overlays = OverlayContainer::<OpenXrOverlayData>::new(&mut app, headless)?;
+    let mut lines = LinePool::new(&app)?;
+
+    let mut notifications = NotificationManager::new();
+    notifications.run_dbus();
+    notifications.run_udp();
+
+    let mut delete_queue = vec![];
+
+    let mut monado = Monado::auto_connect()
+        .map_err(|e| log::warn!("Will not use libmonado: {e}"))
+        .ok();
+
+    let mut playspace = monado.as_mut().and_then(|m| {
+        playspace::PlayspaceMover::new(m)
+            .map_err(|e| log::warn!("Will not use Monado playspace mover: {e}"))
+            .ok()
+    });
+
+    let mut blocker = monado.is_some().then(blocker::InputBlocker::new);
+
+    let (session, mut frame_wait, mut frame_stream) = unsafe {
+        let raw_session = helpers::create_overlay_session(
+            &xr_instance,
+            system,
+            &xr::vulkan::SessionCreateInfo {
+                instance: app.gfx.instance.handle().as_raw() as _,
+                physical_device: app.gfx.device.physical_device().handle().as_raw() as _,
+                device: app.gfx.device.handle().as_raw() as _,
+                queue_family_index: app.gfx.queue_gfx.queue_family_index(),
+                queue_index: 0,
+            },
+        )?;
+        xr::Session::from_raw(xr_instance.clone(), raw_session, Box::new(()))
+    };
+
+    let stage =
+        session.create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)?;
+
+    let view = session.create_reference_space(xr::ReferenceSpaceType::VIEW, xr::Posef::IDENTITY)?;
+
+    let mut xr_state = XrState {
+        instance: xr_instance,
+        session,
+        predicted_display_time: xr::Time::from_nanos(0),
+        fps: 30.0,
+        stage: Arc::new(stage),
+        view: Arc::new(view),
+    };
+
+    let mut skybox = if environment_blend_mode == xr::EnvironmentBlendMode::OPAQUE {
+        create_skybox(&xr_state, &app)
+    } else {
+        None
+    };
+
+    let pointer_lines = [
+        lines.allocate(&xr_state, app.gfx.clone())?,
+        lines.allocate(&xr_state, app.gfx.clone())?,
+    ];
+
+    let watch_id = overlays.get_by_name(WATCH_NAME).unwrap().state.id; // want panic
+
+    let mut input_source = input::OpenXrInputSource::new(&xr_state)?;
+
+    let mut session_running = false;
+    let mut event_storage = xr::EventDataBuffer::new();
+
+    let mut next_device_update = Instant::now();
+    let mut due_tasks = VecDeque::with_capacity(4);
+
+    let mut fps_counter: VecDeque<Instant> = VecDeque::new();
+
+    let mut main_session_visible = false;
+
+    'main_loop: loop {
+        let cur_frame = FRAME_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+        if !running.load(Ordering::Relaxed) {
+            log::warn!("Received shutdown signal.");
+            match xr_state.session.request_exit() {
+                Ok(()) => log::info!("OpenXR session exit requested."),
+                Err(xr::sys::Result::ERROR_SESSION_NOT_RUNNING) => break 'main_loop,
+                Err(e) => {
+                    log::error!("Failed to request OpenXR session exit: {e}");
+                    break 'main_loop;
+                }
+            }
+        }
+
+        while let Some(event) = xr_state.instance.poll_event(&mut event_storage)? {
+            match event {
+                xr::Event::SessionStateChanged(e) => {
+                    // Session state change is where we can begin and end sessions, as well as
+                    // find quit messages!
+                    log::info!("entered state {:?}", e.state());
+                    match e.state() {
+                        xr::SessionState::READY => {
+                            xr_state.session.begin(VIEW_TYPE)?;
+                            session_running = true;
+                        }
+                        xr::SessionState::STOPPING => {
+                            xr_state.session.end()?;
+                            session_running = false;
+                        }
+                        xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => {
+                            break 'main_loop;
+                        }
+                        _ => {}
+                    }
+                }
+                xr::Event::InstanceLossPending(_) => {
+                    break 'main_loop;
+                }
+                xr::Event::EventsLost(e) => {
+                    log::warn!("lost {} events", e.lost_event_count());
+                }
+                xr::Event::MainSessionVisibilityChangedEXTX(e) => {
+                    if main_session_visible != e.visible() {
+                        main_session_visible = e.visible();
+                        log::info!("Main session visible: {main_session_visible}");
+                        if main_session_visible {
+                            log::debug!("Destroying skybox.");
+                            skybox = None;
+                        } else if environment_blend_mode == xr::EnvironmentBlendMode::OPAQUE {
+                            log::debug!("Allocating skybox.");
+                            skybox = create_skybox(&xr_state, &app);
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        if next_device_update <= Instant::now() {
+            if let Some(monado) = &mut monado {
+                OpenXrInputSource::update_devices(&mut app, monado);
+                next_device_update = Instant::now() + Duration::from_secs(30);
+            }
+        }
+
+        if !session_running {
+            std::thread::sleep(Duration::from_millis(100));
+            continue 'main_loop;
+        }
+
+        let xr_frame_state = frame_wait.wait()?;
+        frame_stream.begin()?;
+
+        xr_state.predicted_display_time = xr_frame_state.predicted_display_time;
+        xr_state.fps = {
+            fps_counter.push_back(Instant::now());
+
+            while let Some(time) = fps_counter.front() {
+                if time.elapsed().as_secs_f32() > 1. {
+                    fps_counter.pop_front();
+                } else {
+                    break;
+                }
+            }
+
+            let total_elapsed = fps_counter
+                .front()
+                .map_or(0f32, |time| time.elapsed().as_secs_f32());
+
+            fps_counter.len() as f32 / total_elapsed
+        };
+
+        if !xr_frame_state.should_render {
+            frame_stream.end(
+                xr_frame_state.predicted_display_time,
+                environment_blend_mode,
+                &[],
+            )?;
+            continue 'main_loop;
+        }
+
+        app.input_state.pre_update();
+        input_source.update(&xr_state, &mut app)?;
+        app.input_state.post_update(&app.session);
+
+        if let Some(ref mut blocker) = blocker {
+            blocker.update(
+                &app,
+                watch_id,
+                monado.as_mut().unwrap(), // safe
+            );
+        }
+
+        if app
+            .input_state
+            .pointers
+            .iter()
+            .any(|p| p.now.show_hide && !p.before.show_hide)
+        {
+            overlays.show_hide(&mut app);
+        }
+
+        #[cfg(feature = "wayvr")]
+        if app
+            .input_state
+            .pointers
+            .iter()
+            .any(|p| p.now.toggle_dashboard && !p.before.toggle_dashboard)
+        {
+            wayvr_action(&mut app, &mut overlays, &WayVRAction::ToggleDashboard);
+        }
+
+        watch_fade(&mut app, overlays.mut_by_id(watch_id).unwrap()); // want panic
+        if let Some(ref mut space_mover) = playspace {
+            space_mover.update(
+                &mut overlays,
+                &app,
+                monado.as_mut().unwrap(), // safe
+            );
+        }
+
+        for o in overlays.iter_mut() {
+            o.after_input(&mut app)?;
+        }
+
+        #[cfg(feature = "osc")]
+        if let Some(ref mut sender) = app.osc_sender {
+            let _ = sender.send_params(&overlays, &app.input_state.devices);
+        }
+
+        let (_, views) = xr_state.session.locate_views(
+            VIEW_TYPE,
+            xr_frame_state.predicted_display_time,
+            &xr_state.stage,
+        )?;
+
+        let ipd = helpers::ipd_from_views(&views);
+        if (app.input_state.ipd - ipd).abs() > 0.01 {
+            log::info!("IPD changed: {} -> {}", app.input_state.ipd, ipd);
+            app.input_state.ipd = ipd;
+            Toast::new(
+                ToastTopic::IpdChange,
+                "IPD".into(),
+                format!("{ipd:.1} mm").into(),
+            )
+            .submit(&mut app);
+        }
+
+        overlays
+            .iter_mut()
+            .for_each(|o| o.state.auto_movement(&mut app));
+
+        let lengths_haptics = interact(&mut overlays, &mut app);
+        for (idx, (len, haptics)) in lengths_haptics.iter().enumerate() {
+            lines.draw_from(
+                pointer_lines[idx],
+                app.input_state.pointers[idx].pose,
+                *len,
+                app.input_state.pointers[idx].interaction.mode as usize + 1,
+                &app.input_state.hmd,
+            );
+            if let Some(haptics) = haptics {
+                input_source.haptics(&xr_state, idx, haptics);
+            }
+        }
+
+        app.hid_provider.commit();
+
+        let watch = overlays.mut_by_id(watch_id).unwrap(); // want panic
+        let watch_transform = watch.state.transform;
+        if !watch.state.want_visible {
+            watch.state.want_visible = true;
+            watch.state.transform = Affine3A::from_scale(Vec3 {
+                x: 0.001,
+                y: 0.001,
+                z: 0.001,
+            });
+        }
+
+        #[cfg(feature = "wayvr")]
+        if let Err(e) =
+            crate::overlays::wayvr::tick_events::<OpenXrOverlayData>(&mut app, &mut overlays)
+        {
+            log::error!("WayVR tick_events failed: {e:?}");
+        }
+
+        // Begin rendering
+        let mut buffers = CommandBuffers::default();
+
+        if !main_session_visible {
+            if let Some(skybox) = skybox.as_mut() {
+                skybox.render(&xr_state, &app, &mut buffers)?;
+            }
+        }
+
+        for o in overlays.iter_mut() {
+            o.data.cur_visible = false;
+            if !o.state.want_visible {
+                continue;
+            }
+
+            if !o.data.init {
+                o.init(&mut app)?;
+                o.data.init = true;
+            }
+
+            let should_render = match o.should_render(&mut app)? {
+                ShouldRender::Should => true,
+                ShouldRender::Can => (o.data.last_alpha - o.state.alpha).abs() > f32::EPSILON,
+                ShouldRender::Unable => false, //try show old image if exists
+            };
+
+            if should_render {
+                if !o.ensure_swapchain(&app, &xr_state)? {
+                    continue;
+                }
+                let tgt = o.data.swapchain.as_mut().unwrap().acquire_wait_image()?; // want
+                if !o.render(&mut app, tgt, &mut buffers, o.state.alpha)? {
+                    o.data.swapchain.as_mut().unwrap().ensure_image_released()?; // want
+                    continue;
+                }
+                o.data.last_alpha = o.state.alpha;
+            } else if o.data.swapchain.is_none() {
+                continue;
+            }
+            o.data.cur_visible = true;
+        }
+
+        lines.render(&app, &mut buffers)?;
+
+        let future = buffers.execute_now(app.gfx.queue_gfx.clone())?;
+        if let Some(mut future) = future {
+            if let Err(e) = future.flush() {
+                return Err(BackendError::Fatal(e.into()));
+            }
+            future.cleanup_finished();
+        }
+        // End rendering
+
+        // Layer composition
+        let mut layers = vec![];
+        if !main_session_visible {
+            if let Some(skybox) = skybox.as_mut() {
+                for (idx, layer) in skybox.present(&xr_state, &app)?.into_iter().enumerate() {
+                    layers.push(((idx as f32).mul_add(-50.0, 200.0), layer));
+                }
+            }
+        }
+
+        for o in overlays.iter_mut() {
+            if !o.data.cur_visible {
+                continue;
+            }
+            let dist_sq = (app.input_state.hmd.translation - o.state.transform.translation)
+                .length_squared()
+                + (100f32 - o.state.z_order as f32);
+            if !dist_sq.is_normal() {
+                o.data.swapchain.as_mut().unwrap().ensure_image_released()?;
+                continue;
+            }
+            let maybe_layer = o.present(&xr_state)?;
+            if matches!(maybe_layer, CompositionLayer::None) {
+                continue;
+            }
+            layers.push((dist_sq, maybe_layer));
+        }
+
+        for maybe_layer in lines.present(&xr_state)? {
+            if matches!(maybe_layer, CompositionLayer::None) {
+                continue;
+            }
+            layers.push((0.0, maybe_layer));
+        }
+        // End layer composition
+
+        #[cfg(feature = "wayvr")]
+        if let Some(wayvr) = &app.wayvr {
+            wayvr.borrow_mut().data.tick_finish()?;
+        }
+
+        // Begin layer submit
+        layers.sort_by(|a, b| b.0.total_cmp(&a.0));
+
+        let frame_ref = layers
+            .iter()
+            .map(|f| match f.1 {
+                CompositionLayer::Quad(ref l) => l as &xr::CompositionLayerBase<xr::Vulkan>,
+                CompositionLayer::Cylinder(ref l) => l as &xr::CompositionLayerBase<xr::Vulkan>,
+                CompositionLayer::Equirect2(ref l) => l as &xr::CompositionLayerBase<xr::Vulkan>,
+                CompositionLayer::None => unreachable!(),
+            })
+            .collect::<Vec<_>>();
+
+        frame_stream.end(
+            xr_state.predicted_display_time,
+            environment_blend_mode,
+            &frame_ref,
+        )?;
+        // End layer submit
+
+        let removed_overlays = overlays.update(&mut app)?;
+        for o in removed_overlays {
+            delete_queue.push((o, cur_frame + 5));
+        }
+
+        notifications.submit_pending(&mut app);
+
+        app.tasks.retrieve_due(&mut due_tasks);
+        while let Some(task) = due_tasks.pop_front() {
+            match task {
+                TaskType::Overlay(sel, f) => {
+                    if let Some(o) = overlays.mut_by_selector(&sel) {
+                        f(&mut app, &mut o.state);
+                    } else {
+                        log::warn!("Overlay not found for task: {sel:?}");
+                    }
+                }
+                TaskType::CreateOverlay(sel, f) => {
+                    let None = overlays.mut_by_selector(&sel) else {
+                        continue;
+                    };
+
+                    let Some((mut overlay_state, overlay_backend)) = f(&mut app) else {
+                        continue;
+                    };
+                    overlay_state.birthframe = cur_frame;
+
+                    overlays.add(OverlayData {
+                        state: overlay_state,
+                        backend: overlay_backend,
+                        ..Default::default()
+                    });
+                }
+                TaskType::DropOverlay(sel) => {
+                    if let Some(o) = overlays.mut_by_selector(&sel) {
+                        if o.state.birthframe < cur_frame {
+                            log::debug!("{}: destroy", o.state.name);
+                            if let Some(o) = overlays.remove_by_selector(&sel) {
+                                // set for deletion after all images are done showing
+                                delete_queue.push((o, cur_frame + 5));
+                            }
+                        }
+                    }
+                }
+                TaskType::System(task) => match task {
+                    SystemTask::FixFloor => {
+                        if let Some(ref mut playspace) = playspace {
+                            playspace.fix_floor(
+                                &app.input_state,
+                                monado.as_mut().unwrap(), // safe
+                            );
+                        }
+                    }
+                    SystemTask::ResetPlayspace => {
+                        if let Some(ref mut playspace) = playspace {
+                            playspace.reset_offset(monado.as_mut().unwrap()); // safe
+                        }
+                    }
+                    SystemTask::ShowHide => {
+                        overlays.show_hide(&mut app);
+                    }
+                    _ => {}
+                },
+                #[cfg(feature = "wayvr")]
+                TaskType::WayVR(action) => {
+                    wayvr_action(&mut app, &mut overlays, &action);
+                }
+            }
+        }
+
+        delete_queue.retain(|(_, frame)| *frame > cur_frame);
+
+        let watch = overlays.mut_by_id(watch_id).unwrap(); // want panic
+        watch.state.transform = watch_transform;
+    }
+
+    Ok(())
+}
+
+pub(super) enum CompositionLayer<'a> {
+    None,
+    Quad(xr::CompositionLayerQuad<'a, xr::Vulkan>),
+    Cylinder(xr::CompositionLayerCylinderKHR<'a, xr::Vulkan>),
+    Equirect2(xr::CompositionLayerEquirect2KHR<'a, xr::Vulkan>),
+}

wlx-overlay-s/src/backend/openxr/openxr_actions.json5

@@ -0,0 +1,301 @@
+// Available bindings:
+//
+// -- click --
+// primary click to interact with the watch or overlays. required
+//
+// -- grab --
+// used to manipulate position, size, orientation of overlays in 3D space
+//
+// -- show_hide --
+// used to quickly hide and show your last selection of screens + keyboard
+//
+// -- space_drag --
+// move your stage (playspace drag)
+//
+// -- toggle_dashboard --
+// run or toggle visibility of a previously configured WayVR-compatible dashboard
+//
+// -- space_rotate --
+// rotate your stage (playspace rotate, WIP)
+//
+// -- space_reset --
+// reset your stage (reset the offset from playspace drag)
+//
+// -- click_modifier_right --
+// while this is held, your pointer will turn ORANGE and your mouse clicks will be RIGHT clicks
+//
+// -- click_modifier_middle --
+// while this is held, your pointer will turn PURPLE and your mouse clicks will be MIDDLE clicks
+//
+// -- move_mouse --
+// when using `focus_follows_mouse_mode`, you need to hold this for the mouse to move
+//
+// -- pose, haptic --
+// do not mess with these, unless you know what you're doing
+
+[
+  // Fallback controller, intended for testing
+  {
+    profile: "/interaction_profiles/khr/simple_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    },
+    click: {
+      // left trigger is click
+      left: "/user/hand/left/input/select/click",
+    },
+    grab: {
+      // right trigger is grab
+      right: "/user/hand/right/input/select/click"
+    },
+    show_hide: {
+      left: "/user/hand/left/input/menu/click"
+    }
+  },
+
+  // Oculus Touch Controller. Compatible with Quest 2, Quest 3, Quest Pro
+  {
+    profile: "/interaction_profiles/oculus/touch_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    },
+    click: {
+      left: "/user/hand/left/input/trigger/value",
+      right: "/user/hand/right/input/trigger/value"
+    },
+    grab: {
+      left: "/user/hand/left/input/squeeze/value",
+      right: "/user/hand/right/input/squeeze/value"
+    },
+    scroll: {
+      left: "/user/hand/left/input/thumbstick/y",
+      right: "/user/hand/right/input/thumbstick/y"
+    },
+    scroll_horizontal: {
+      left: "/user/hand/left/input/thumbstick/x",
+      right: "/user/hand/right/input/thumbstick/x"
+    },
+    show_hide: {
+      double_click: true,
+      left: "/user/hand/left/input/y/click",
+    },
+    space_drag: {
+      left: "/user/hand/left/input/menu/click",
+    },
+    space_reset: {
+      double_click: true,
+      left: "/user/hand/left/input/menu/click",
+    },
+    click_modifier_right: {
+      left: "/user/hand/left/input/y/touch",
+      right: "/user/hand/right/input/b/touch"
+    },
+    click_modifier_middle: {
+      left: "/user/hand/left/input/x/touch",
+      right: "/user/hand/right/input/a/touch"
+    },
+    move_mouse: {
+      // used with focus_follows_mouse_mode
+      left: "/user/hand/left/input/trigger/touch",
+      right: "/user/hand/right/input/trigger/touch"
+    }
+  },
+
+  // Index controller
+  {
+    profile: "/interaction_profiles/valve/index_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    },
+    click: {
+      left: "/user/hand/left/input/trigger/value",
+      right: "/user/hand/right/input/trigger/value"
+    },
+    alt_click: {
+      // left trackpad is space_drag
+      right: "/user/hand/right/input/trackpad/force",
+    },
+    grab: {
+      left: "/user/hand/left/input/squeeze/force",
+      right: "/user/hand/right/input/squeeze/force"
+    },
+    scroll: {
+      left: "/user/hand/left/input/thumbstick/y",
+      right: "/user/hand/right/input/thumbstick/y"
+    },
+    scroll_horizontal: {
+      left: "/user/hand/left/input/thumbstick/x",
+      right: "/user/hand/right/input/thumbstick/x"
+    },
+    toggle_dashboard: {
+      double_click: false,
+      right: "/user/hand/right/input/system/click",
+    },
+    show_hide: {
+      double_click: true,
+      left: "/user/hand/left/input/b/click",
+    },
+    space_drag: {
+      left: "/user/hand/left/input/trackpad/force",
+      // right trackpad is alt_click
+    },
+    space_reset: {
+      left: "/user/hand/left/input/trackpad/force",
+      double_click: true,
+    },
+    click_modifier_right: {
+      left: "/user/hand/left/input/b/touch",
+      right: "/user/hand/right/input/b/touch"
+    },
+    click_modifier_middle: {
+      left: "/user/hand/left/input/a/touch",
+      right: "/user/hand/right/input/a/touch"
+    },
+    move_mouse: {
+      // used with focus_follows_mouse_mode
+      left: "/user/hand/left/input/trigger/touch",
+      right: "/user/hand/right/input/trigger/touch"
+    }
+  },
+
+  // Vive controller
+  {
+    profile: "/interaction_profiles/htc/vive_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    click: {
+      left: "/user/hand/left/input/trigger/value",
+      right: "/user/hand/right/input/trigger/value"
+    },
+    grab: {
+      left: "/user/hand/left/input/squeeze/click",
+      right: "/user/hand/right/input/squeeze/click"
+    },
+    scroll: {
+      left: "/user/hand/left/input/trackpad/y",
+      right: "/user/hand/right/input/trackpad/y"
+    },
+    scroll_horizontal: {
+      left: "/user/hand/left/input/trackpad/x",
+      right: "/user/hand/right/input/trackpad/x"
+    },
+    show_hide: {
+      left: "/user/hand/left/input/menu/click",
+    },
+    space_drag: {
+      right: "/user/hand/right/input/menu/click",
+    },
+    space_reset: {
+      double_click: true,
+      right: "/user/hand/right/input/menu/click",
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    }
+  },
+
+  // Windows Mixed Reality controller
+  {
+    profile: "/interaction_profiles/microsoft/motion_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    },
+    click: {
+      left: "/user/hand/left/input/trigger/value",
+      right: "/user/hand/right/input/trigger/value"
+    },
+    grab: {
+      left: "/user/hand/left/input/squeeze/click",
+      right: "/user/hand/right/input/squeeze/click"
+    },
+    scroll: {
+      left: "/user/hand/left/input/thumbstick/y",
+      right: "/user/hand/right/input/thumbstick/y"
+    },
+    scroll_horizontal: {
+      left: "/user/hand/left/input/thumbstick/x",
+      right: "/user/hand/right/input/thumbstick/x"
+    },
+    show_hide: {
+      left: "/user/hand/left/input/system/click",
+    },
+    space_drag: {
+      right: "/user/hand/right/input/system/click",
+    },
+    space_reset: {
+      double_click: true,
+      right: "/user/hand/right/input/system/click",
+    },
+    click_modifier_right: {
+      left: "/user/hand/left/input/trackpad/dpad_up",
+      right: "/user/hand/right/input/trackpad/dpad_up"
+    },
+    click_modifier_middle: {
+      left: "/user/hand/left/input/trackpad/dpad_down",
+      right: "/user/hand/right/input/trackpad/dpad_down"
+    },
+  },
+
+  // HP Reverb G2 controller
+  {
+    profile: "/interaction_profiles/hp/mixed_reality_controller",
+    pose: {
+      left: "/user/hand/left/input/aim/pose",
+      right: "/user/hand/right/input/aim/pose"
+    },
+    haptic: {
+      left: "/user/hand/left/output/haptic",
+      right: "/user/hand/right/output/haptic"
+    },
+    click: {
+      left: "/user/hand/left/input/trigger/value",
+      right: "/user/hand/right/input/trigger/value"
+    },
+    grab: {
+      left: "/user/hand/left/input/squeeze/value",
+      right: "/user/hand/right/input/squeeze/value"
+    },
+    scroll: {
+      left: "/user/hand/left/input/thumbstick/y",
+      right: "/user/hand/right/input/thumbstick/y"
+    },
+    scroll_horizontal: {
+      left: "/user/hand/left/input/thumbstick/x",
+      right: "/user/hand/right/input/thumbstick/x"
+    },
+    show_hide: {
+      left: "/user/hand/left/input/system/click",
+    },
+    space_drag: {
+      right: "/user/hand/right/input/system/click",
+    },
+    space_reset: {
+      double_click: true,
+      right: "/user/hand/right/input/system/click",
+    },
+  },
+
+]

wlx-overlay-s/src/backend/openxr/overlay.rs

@@ -0,0 +1,123 @@
+use glam::Vec3A;
+use openxr::{self as xr, CompositionLayerFlags};
+use std::f32::consts::PI;
+use xr::EyeVisibility;
+
+use super::{helpers, swapchain::WlxSwapchain, CompositionLayer, XrState};
+use crate::{
+    backend::{
+        openxr::swapchain::{create_swapchain, SwapchainOpts},
+        overlay::OverlayData,
+    },
+    state::AppState,
+};
+
+#[derive(Default)]
+pub struct OpenXrOverlayData {
+    last_visible: bool,
+    pub(super) swapchain: Option<WlxSwapchain>,
+    pub(super) init: bool,
+    pub(super) cur_visible: bool,
+    pub(super) last_alpha: f32,
+}
+
+impl OverlayData<OpenXrOverlayData> {
+    pub(super) fn ensure_swapchain<'a>(
+        &'a mut self,
+        app: &AppState,
+        xr: &'a XrState,
+    ) -> anyhow::Result<bool> {
+        if self.data.swapchain.is_some() {
+            return Ok(true);
+        }
+
+        let Some(meta) = self.frame_meta() else {
+            log::warn!(
+                "{}: swapchain cannot be created due to missing metadata",
+                self.state.name
+            );
+            return Ok(false);
+        };
+
+        let extent = meta.extent;
+        self.data.swapchain = Some(create_swapchain(
+            xr,
+            app.gfx.clone(),
+            extent,
+            SwapchainOpts::new(),
+        )?);
+        Ok(true)
+    }
+
+    pub(super) fn present<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+    ) -> anyhow::Result<CompositionLayer<'a>> {
+        let Some(swapchain) = self.data.swapchain.as_mut() else {
+            log::warn!("{}: swapchain not ready", self.state.name);
+            return Ok(CompositionLayer::None);
+        };
+        if !swapchain.ever_acquired {
+            log::warn!("{}: swapchain not rendered", self.state.name);
+            return Ok(CompositionLayer::None);
+        }
+        swapchain.ensure_image_released()?;
+
+        let sub_image = swapchain.get_subimage();
+        let transform = self.state.transform * self.backend.frame_meta().unwrap().transform; // contract
+
+        let aspect_ratio = swapchain.extent[1] as f32 / swapchain.extent[0] as f32;
+        let (scale_x, scale_y) = if aspect_ratio < 1.0 {
+            let major = transform.matrix3.col(0).length();
+            (major, major * aspect_ratio)
+        } else {
+            let major = transform.matrix3.col(1).length();
+            (major / aspect_ratio, major)
+        };
+
+        if let Some(curvature) = self.state.curvature {
+            let radius = scale_x / (2.0 * PI * curvature);
+            let quat = helpers::transform_to_norm_quat(&transform);
+            let center_point = transform.translation + quat.mul_vec3a(Vec3A::Z * radius);
+
+            let posef = helpers::translation_rotation_to_posef(center_point, quat);
+            let angle = 2.0 * (scale_x / (2.0 * radius));
+
+            let cylinder = xr::CompositionLayerCylinderKHR::new()
+                .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA)
+                .pose(posef)
+                .sub_image(sub_image)
+                .eye_visibility(EyeVisibility::BOTH)
+                .space(&xr.stage)
+                .radius(radius)
+                .central_angle(angle)
+                .aspect_ratio(aspect_ratio);
+            Ok(CompositionLayer::Cylinder(cylinder))
+        } else {
+            let posef = helpers::transform_to_posef(&transform);
+            let quad = xr::CompositionLayerQuad::new()
+                .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA)
+                .pose(posef)
+                .sub_image(sub_image)
+                .eye_visibility(EyeVisibility::BOTH)
+                .space(&xr.stage)
+                .size(xr::Extent2Df {
+                    width: scale_x,
+                    height: scale_y,
+                });
+            Ok(CompositionLayer::Quad(quad))
+        }
+    }
+
+    pub(super) fn after_input(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        if self.data.last_visible != self.state.want_visible {
+            if self.state.want_visible {
+                self.backend.resume(app)?;
+            } else {
+                self.backend.pause(app)?;
+            }
+        }
+        self.data.last_visible = self.state.want_visible;
+        Ok(())
+    }
+}

wlx-overlay-s/src/backend/openxr/playspace.rs

@@ -0,0 +1,199 @@
+use glam::{Affine3A, Quat, Vec3A};
+use libmonado::{Monado, Pose, ReferenceSpaceType};
+
+use crate::{
+    backend::{common::OverlayContainer, input::InputState},
+    state::AppState,
+};
+
+use super::overlay::OpenXrOverlayData;
+
+struct MoverData<T> {
+    pose: Affine3A,
+    hand: usize,
+    hand_pose: T,
+}
+
+pub(super) struct PlayspaceMover {
+    last_transform: Affine3A,
+    drag: Option<MoverData<Vec3A>>,
+    rotate: Option<MoverData<Quat>>,
+}
+
+impl PlayspaceMover {
+    pub fn new(monado: &mut Monado) -> anyhow::Result<Self> {
+        log::info!("Monado: using space offset API");
+
+        let Ok(stage) = monado.get_reference_space_offset(ReferenceSpaceType::Stage) else {
+            anyhow::bail!("Space offsets not supported.");
+        };
+
+        log::debug!("STAGE is at {:?}, {:?}", stage.position, stage.orientation);
+
+        // initial offset
+        let last_transform =
+            Affine3A::from_rotation_translation(stage.orientation.into(), stage.position.into());
+
+        Ok(Self {
+            last_transform,
+
+            drag: None,
+            rotate: None,
+        })
+    }
+
+    pub fn update(
+        &mut self,
+        overlays: &mut OverlayContainer<OpenXrOverlayData>,
+        state: &AppState,
+        monado: &mut Monado,
+    ) {
+        for pointer in &state.input_state.pointers {
+            if pointer.now.space_reset {
+                if !pointer.before.space_reset {
+                    log::info!("Space reset");
+                    self.reset_offset(monado);
+                }
+                return;
+            }
+        }
+
+        if let Some(mut data) = self.rotate.take() {
+            let pointer = &state.input_state.pointers[data.hand];
+            if !pointer.now.space_rotate {
+                self.last_transform = data.pose;
+                log::info!("End space rotate");
+                return;
+            }
+
+            let new_hand =
+                Quat::from_affine3(&(data.pose * state.input_state.pointers[data.hand].raw_pose));
+
+            let dq = new_hand * data.hand_pose.conjugate();
+            let mut space_transform = if state.session.config.space_rotate_unlocked {
+                Affine3A::from_quat(dq)
+            } else {
+                let rel_y = f32::atan2(
+                    2.0 * dq.y.mul_add(dq.w, dq.x * dq.z),
+                    2.0f32.mul_add(dq.w.mul_add(dq.w, dq.x * dq.x), -1.0),
+                );
+
+                Affine3A::from_rotation_y(rel_y)
+            };
+            let offset = (space_transform.transform_vector3a(state.input_state.hmd.translation)
+                - state.input_state.hmd.translation)
+                * -1.0;
+
+            space_transform.translation = offset;
+
+            data.pose *= space_transform;
+            data.hand_pose = new_hand;
+
+            apply_offset(data.pose, monado);
+            self.rotate = Some(data);
+        } else {
+            for (i, pointer) in state.input_state.pointers.iter().enumerate() {
+                if pointer.now.space_rotate {
+                    let hand_pose = Quat::from_affine3(&(self.last_transform * pointer.raw_pose));
+                    self.rotate = Some(MoverData {
+                        pose: self.last_transform,
+                        hand: i,
+                        hand_pose,
+                    });
+                    self.drag = None;
+                    log::info!("Start space rotate");
+                    return;
+                }
+            }
+        }
+
+        if let Some(mut data) = self.drag.take() {
+            let pointer = &state.input_state.pointers[data.hand];
+            if !pointer.now.space_drag {
+                self.last_transform = data.pose;
+                log::info!("End space drag");
+                return;
+            }
+
+            let new_hand = data
+                .pose
+                .transform_point3a(state.input_state.pointers[data.hand].raw_pose.translation);
+            let relative_pos =
+                (new_hand - data.hand_pose) * state.session.config.space_drag_multiplier;
+
+            if relative_pos.length_squared() > 1000.0 {
+                log::warn!("Space drag too fast, ignoring");
+                return;
+            }
+
+            let overlay_offset = data.pose.inverse().transform_vector3a(relative_pos) * -1.0;
+
+            overlays.iter_mut().for_each(|overlay| {
+                if overlay.state.grabbable {
+                    overlay.state.dirty = true;
+                    overlay.state.transform.translation += overlay_offset;
+                }
+            });
+
+            data.pose.translation += relative_pos;
+            data.hand_pose = new_hand;
+
+            apply_offset(data.pose, monado);
+            self.drag = Some(data);
+        } else {
+            for (i, pointer) in state.input_state.pointers.iter().enumerate() {
+                if pointer.now.space_drag {
+                    let hand_pos = self
+                        .last_transform
+                        .transform_point3a(pointer.raw_pose.translation);
+                    self.drag = Some(MoverData {
+                        pose: self.last_transform,
+                        hand: i,
+                        hand_pose: hand_pos,
+                    });
+                    log::info!("Start space drag");
+                    return;
+                }
+            }
+        }
+    }
+
+    pub fn reset_offset(&mut self, monado: &mut Monado) {
+        if self.drag.is_some() {
+            log::info!("Space drag interrupted by manual reset");
+            self.drag = None;
+        }
+        if self.rotate.is_some() {
+            log::info!("Space rotate interrupted by manual reset");
+            self.rotate = None;
+        }
+
+        self.last_transform = Affine3A::IDENTITY;
+        apply_offset(self.last_transform, monado);
+    }
+
+    pub fn fix_floor(&mut self, input: &InputState, monado: &mut Monado) {
+        if self.drag.is_some() {
+            log::info!("Space drag interrupted by fix floor");
+            self.drag = None;
+        }
+        if self.rotate.is_some() {
+            log::info!("Space rotate interrupted by fix floor");
+            self.rotate = None;
+        }
+
+        let y1 = input.pointers[0].raw_pose.translation.y;
+        let y2 = input.pointers[1].raw_pose.translation.y;
+        let delta = y1.min(y2) - 0.03;
+        self.last_transform.translation.y += delta;
+        apply_offset(self.last_transform, monado);
+    }
+}
+
+fn apply_offset(transform: Affine3A, monado: &mut Monado) {
+    let pose = Pose {
+        position: transform.translation.into(),
+        orientation: Quat::from_affine3(&transform).into(),
+    };
+    let _ = monado.set_reference_space_offset(ReferenceSpaceType::Stage, pose);
+}

wlx-overlay-s/src/backend/openxr/skybox.rs

@@ -0,0 +1,252 @@
+use std::{
+    f32::consts::PI,
+    fs::File,
+    sync::{Arc, LazyLock},
+};
+
+use glam::{Quat, Vec3A};
+use openxr as xr;
+use vulkano::{
+    command_buffer::CommandBufferUsage,
+    image::view::ImageView,
+    pipeline::graphics::{color_blend::AttachmentBlend, input_assembly::PrimitiveTopology},
+};
+
+use crate::{
+    backend::openxr::{helpers::translation_rotation_to_posef, swapchain::SwapchainOpts},
+    config_io,
+    graphics::{dds::WlxCommandBufferDds, CommandBuffers, ExtentExt},
+    state::AppState,
+};
+
+use super::{
+    swapchain::{create_swapchain, WlxSwapchain},
+    CompositionLayer, XrState,
+};
+
+pub(super) struct Skybox {
+    view: Arc<ImageView>,
+    sky: Option<WlxSwapchain>,
+    grid: Option<WlxSwapchain>,
+}
+
+impl Skybox {
+    pub fn new(app: &AppState) -> anyhow::Result<Self> {
+        let mut command_buffer = app
+            .gfx
+            .create_xfer_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+
+        let mut maybe_image = None;
+
+        'custom_tex: {
+            if app.session.config.skybox_texture.is_empty() {
+                break 'custom_tex;
+            }
+
+            let real_path = config_io::get_config_root().join(&*app.session.config.skybox_texture);
+            let Ok(f) = File::open(real_path) else {
+                log::warn!(
+                    "Could not open custom skybox texture at: {}",
+                    app.session.config.skybox_texture
+                );
+                break 'custom_tex;
+            };
+            match command_buffer.upload_image_dds(f) {
+                Ok(image) => {
+                    maybe_image = Some(image);
+                }
+                Err(e) => {
+                    log::warn!(
+                        "Could not use custom skybox texture at: {}",
+                        app.session.config.skybox_texture
+                    );
+                    log::warn!("{e:?}");
+                }
+            }
+        }
+
+        if maybe_image.is_none() {
+            let p = include_bytes!("../../res/table_mountain_2.dds");
+            maybe_image = Some(command_buffer.upload_image_dds(p.as_slice())?);
+        }
+
+        command_buffer.build_and_execute_now()?;
+
+        let view = ImageView::new_default(maybe_image.unwrap())?; // safe unwrap
+
+        Ok(Self {
+            view,
+            sky: None,
+            grid: None,
+        })
+    }
+
+    fn prepare_sky<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+        app: &AppState,
+        buf: &mut CommandBuffers,
+    ) -> anyhow::Result<()> {
+        if self.sky.is_some() {
+            return Ok(());
+        }
+        let opts = SwapchainOpts::new().immutable();
+
+        let extent = self.view.image().extent();
+        let mut swapchain = create_swapchain(xr, app.gfx.clone(), extent, opts)?;
+        let tgt = swapchain.acquire_wait_image()?;
+        let pipeline = app.gfx.create_pipeline(
+            app.gfx_extras.shaders.get("vert_quad").unwrap().clone(), // want panic
+            app.gfx_extras.shaders.get("frag_srgb").unwrap().clone(), // want panic
+            app.gfx.surface_format,
+            None,
+            PrimitiveTopology::TriangleStrip,
+            false,
+        )?;
+
+        let set0 = pipeline.uniform_sampler(0, self.view.clone(), app.gfx.texture_filter)?;
+        let set1 = pipeline.uniform_buffer_upload(1, vec![1f32])?;
+        let pass = pipeline.create_pass(
+            tgt.extent_f32(),
+            app.gfx_extras.quad_verts.clone(),
+            0..4,
+            0..1,
+            vec![set0, set1],
+        )?;
+
+        let mut cmd_buffer = app
+            .gfx
+            .create_gfx_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+        cmd_buffer.begin_rendering(tgt)?;
+        cmd_buffer.run_ref(&pass)?;
+        cmd_buffer.end_rendering()?;
+
+        buf.push(cmd_buffer.build()?);
+
+        self.sky = Some(swapchain);
+        Ok(())
+    }
+
+    fn prepare_grid<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+        app: &AppState,
+        buf: &mut CommandBuffers,
+    ) -> anyhow::Result<()> {
+        if self.grid.is_some() {
+            return Ok(());
+        }
+
+        let extent = [1024, 1024, 1];
+        let mut swapchain = create_swapchain(
+            xr,
+            app.gfx.clone(),
+            extent,
+            SwapchainOpts::new().immutable(),
+        )?;
+        let pipeline = app.gfx.create_pipeline(
+            app.gfx_extras.shaders.get("vert_quad").unwrap().clone(), // want panic
+            app.gfx_extras.shaders.get("frag_grid").unwrap().clone(), // want panic
+            app.gfx.surface_format,
+            Some(AttachmentBlend::alpha()),
+            PrimitiveTopology::TriangleStrip,
+            false,
+        )?;
+
+        let tgt = swapchain.acquire_wait_image()?;
+        let pass = pipeline.create_pass(
+            tgt.extent_f32(),
+            app.gfx_extras.quad_verts.clone(),
+            0..4,
+            0..1,
+            vec![],
+        )?;
+
+        let mut cmd_buffer = app
+            .gfx
+            .create_gfx_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+        cmd_buffer.begin_rendering(tgt)?;
+        cmd_buffer.run_ref(&pass)?;
+        cmd_buffer.end_rendering()?;
+
+        buf.push(cmd_buffer.build()?);
+
+        self.grid = Some(swapchain);
+        Ok(())
+    }
+
+    pub(super) fn render(
+        &mut self,
+        xr: &XrState,
+        app: &AppState,
+        buf: &mut CommandBuffers,
+    ) -> anyhow::Result<()> {
+        self.prepare_sky(xr, app, buf)?;
+        self.prepare_grid(xr, app, buf)?;
+        Ok(())
+    }
+
+    pub(super) fn present<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+        app: &AppState,
+    ) -> anyhow::Result<Vec<CompositionLayer<'a>>> {
+        // cover the entire sphere
+        const HORIZ_ANGLE: f32 = 2.0 * PI;
+        const HI_VERT_ANGLE: f32 = 0.5 * PI;
+        const LO_VERT_ANGLE: f32 = -0.5 * PI;
+
+        static GRID_POSE: LazyLock<xr::Posef> = LazyLock::new(|| {
+            translation_rotation_to_posef(Vec3A::ZERO, Quat::from_rotation_x(PI * -0.5))
+        });
+
+        let pose = xr::Posef {
+            orientation: xr::Quaternionf::IDENTITY,
+            position: xr::Vector3f {
+                x: app.input_state.hmd.translation.x,
+                y: app.input_state.hmd.translation.y,
+                z: app.input_state.hmd.translation.z,
+            },
+        };
+
+        self.sky.as_mut().unwrap().ensure_image_released()?;
+
+        let sky = xr::CompositionLayerEquirect2KHR::new()
+            .layer_flags(xr::CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA)
+            .pose(pose)
+            .radius(10.0)
+            .sub_image(self.sky.as_ref().unwrap().get_subimage())
+            .eye_visibility(xr::EyeVisibility::BOTH)
+            .space(&xr.stage)
+            .central_horizontal_angle(HORIZ_ANGLE)
+            .upper_vertical_angle(HI_VERT_ANGLE)
+            .lower_vertical_angle(LO_VERT_ANGLE);
+
+        self.grid.as_mut().unwrap().ensure_image_released()?;
+        let grid = xr::CompositionLayerQuad::new()
+            .layer_flags(xr::CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA)
+            .pose(*GRID_POSE)
+            .size(xr::Extent2Df {
+                width: 10.0,
+                height: 10.0,
+            })
+            .sub_image(self.grid.as_ref().unwrap().get_subimage())
+            .eye_visibility(xr::EyeVisibility::BOTH)
+            .space(&xr.stage);
+
+        Ok(vec![
+            CompositionLayer::Equirect2(sky),
+            CompositionLayer::Quad(grid),
+        ])
+    }
+}
+
+pub(super) fn create_skybox(xr: &XrState, app: &AppState) -> Option<Skybox> {
+    if !app.session.config.use_skybox {
+        return None;
+    }
+    xr.instance
+        .exts()
+        .khr_composition_layer_equirect2
+        .and_then(|_| Skybox::new(app).ok())
+}

wlx-overlay-s/src/backend/openxr/swapchain.rs

@@ -0,0 +1,125 @@
+use std::sync::Arc;
+
+use ash::vk;
+use openxr as xr;
+
+use smallvec::SmallVec;
+use vulkano::{
+    image::{sys::RawImage, view::ImageView, ImageCreateInfo, ImageUsage},
+    Handle,
+};
+use wgui::gfx::WGfx;
+
+use super::XrState;
+
+#[derive(Default)]
+pub(super) struct SwapchainOpts {
+    pub immutable: bool,
+}
+
+impl SwapchainOpts {
+    pub fn new() -> Self {
+        Self::default()
+    }
+    pub const fn immutable(mut self) -> Self {
+        self.immutable = true;
+        self
+    }
+}
+
+pub(super) fn create_swapchain(
+    xr: &XrState,
+    gfx: Arc<WGfx>,
+    extent: [u32; 3],
+    opts: SwapchainOpts,
+) -> anyhow::Result<WlxSwapchain> {
+    let create_flags = if opts.immutable {
+        xr::SwapchainCreateFlags::STATIC_IMAGE
+    } else {
+        xr::SwapchainCreateFlags::EMPTY
+    };
+
+    let swapchain = xr.session.create_swapchain(&xr::SwapchainCreateInfo {
+        create_flags,
+        usage_flags: xr::SwapchainUsageFlags::COLOR_ATTACHMENT | xr::SwapchainUsageFlags::SAMPLED,
+        format: gfx.surface_format as _,
+        sample_count: 1,
+        width: extent[0],
+        height: extent[1],
+        face_count: 1,
+        array_size: 1,
+        mip_count: 1,
+    })?;
+
+    let images = swapchain
+        .enumerate_images()?
+        .into_iter()
+        .map(|handle| {
+            let vk_image = vk::Image::from_raw(handle);
+            // thanks @yshui
+            let raw_image = unsafe {
+                RawImage::from_handle_borrowed(
+                    gfx.device.clone(),
+                    vk_image,
+                    ImageCreateInfo {
+                        format: gfx.surface_format as _,
+                        extent,
+                        usage: ImageUsage::COLOR_ATTACHMENT,
+                        ..Default::default()
+                    },
+                )?
+            };
+            // SAFETY: OpenXR guarantees that the image is a swapchain image, thus has memory backing it.
+            let image = Arc::new(unsafe { raw_image.assume_bound() });
+            Ok(ImageView::new_default(image)?)
+        })
+        .collect::<anyhow::Result<SmallVec<[Arc<ImageView>; 4]>>>()?;
+
+    Ok(WlxSwapchain {
+        acquired: false,
+        ever_acquired: false,
+        swapchain,
+        images,
+        extent,
+    })
+}
+
+pub(super) struct WlxSwapchain {
+    acquired: bool,
+    pub(super) ever_acquired: bool,
+    pub(super) swapchain: xr::Swapchain<xr::Vulkan>,
+    pub(super) extent: [u32; 3],
+    pub(super) images: SmallVec<[Arc<ImageView>; 4]>,
+}
+
+impl WlxSwapchain {
+    pub(super) fn acquire_wait_image(&mut self) -> anyhow::Result<Arc<ImageView>> {
+        let idx = self.swapchain.acquire_image()? as usize;
+        self.swapchain.wait_image(xr::Duration::INFINITE)?;
+        self.ever_acquired = true;
+        self.acquired = true;
+        Ok(self.images[idx].clone())
+    }
+
+    pub(super) fn ensure_image_released(&mut self) -> anyhow::Result<()> {
+        if self.acquired {
+            self.swapchain.release_image()?;
+            self.acquired = false;
+        }
+        Ok(())
+    }
+
+    pub(super) fn get_subimage(&self) -> xr::SwapchainSubImage<xr::Vulkan> {
+        debug_assert!(self.ever_acquired, "swapchain was never acquired!");
+        xr::SwapchainSubImage::new()
+            .swapchain(&self.swapchain)
+            .image_rect(xr::Rect2Di {
+                offset: xr::Offset2Di { x: 0, y: 0 },
+                extent: xr::Extent2Di {
+                    width: self.extent[0] as _,
+                    height: self.extent[1] as _,
+                },
+            })
+            .image_array_index(0)
+    }
+}

wlx-overlay-s/src/backend/osc.rs

@@ -0,0 +1,183 @@
+use std::{
+    net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
+    time::Instant,
+};
+
+use anyhow::bail;
+use rosc::{OscMessage, OscPacket, OscType};
+
+use crate::overlays::{keyboard::KEYBOARD_NAME, watch::WATCH_NAME};
+
+use crate::backend::input::TrackedDeviceRole;
+
+use super::{common::OverlayContainer, input::TrackedDevice};
+
+pub struct OscSender {
+    last_sent_overlay: Instant,
+    last_sent_battery: Instant,
+    upstream: UdpSocket,
+}
+
+impl OscSender {
+    pub fn new(send_port: u16) -> anyhow::Result<Self> {
+        let ip = IpAddr::V4(Ipv4Addr::LOCALHOST);
+
+        let Ok(upstream) = UdpSocket::bind("0.0.0.0:0") else {
+            bail!("Failed to bind UDP socket - OSC will not function.");
+        };
+
+        let Ok(()) = upstream.connect(SocketAddr::new(ip, send_port)) else {
+            bail!("Failed to connect UDP socket - OSC will not function.");
+        };
+
+        Ok(Self {
+            upstream,
+            last_sent_overlay: Instant::now(),
+            last_sent_battery: Instant::now(),
+        })
+    }
+
+    pub fn send_message(&self, addr: String, args: Vec<OscType>) -> anyhow::Result<()> {
+        let packet = OscPacket::Message(OscMessage { addr, args });
+        let Ok(bytes) = rosc::encoder::encode(&packet) else {
+            bail!("Could not encode OSC packet.");
+        };
+
+        let Ok(_) = self.upstream.send(&bytes) else {
+            bail!("Could not send OSC packet.");
+        };
+
+        Ok(())
+    }
+
+    pub fn send_params<D>(
+        &mut self,
+        overlays: &OverlayContainer<D>,
+        devices: &Vec<TrackedDevice>,
+    ) -> anyhow::Result<()>
+    where
+        D: Default,
+    {
+        // send overlay data every 0.1 seconds
+        if self.last_sent_overlay.elapsed().as_millis() >= 100 {
+            self.last_sent_overlay = Instant::now();
+
+            let mut num_overlays = 0;
+            let mut has_keyboard = false;
+            let mut has_wrist = false;
+
+            for o in overlays.iter() {
+                if !o.state.want_visible {
+                    continue;
+                }
+                match o.state.name.as_ref() {
+                    WATCH_NAME => has_wrist = true,
+                    KEYBOARD_NAME => has_keyboard = true,
+                    _ => {
+                        if o.state.interactable {
+                            num_overlays += 1;
+                        }
+                    }
+                }
+            }
+
+            self.send_message(
+                "/avatar/parameters/isOverlayOpen".into(),
+                vec![OscType::Bool(num_overlays > 0)],
+            )?;
+            self.send_message(
+                "/avatar/parameters/isKeyboardOpen".into(),
+                vec![OscType::Bool(has_keyboard)],
+            )?;
+            self.send_message(
+                "/avatar/parameters/isWristVisible".into(),
+                vec![OscType::Bool(has_wrist)],
+            )?;
+            self.send_message(
+                "/avatar/parameters/openOverlayCount".into(),
+                vec![OscType::Int(num_overlays)],
+            )?;
+        }
+
+        // send battery levels every 10 seconds
+        if self.last_sent_battery.elapsed().as_millis() >= 10000 {
+            self.last_sent_battery = Instant::now();
+
+            let mut tracker_count: i8 = 0;
+            let mut controller_count: i8 = 0;
+            let mut tracker_total_bat = 0.0;
+            let mut controller_total_bat = 0.0;
+
+            for device in devices {
+                let tracker_param;
+
+                // soc is the battery level (set to device status.charge)
+                let level = device.soc.unwrap_or(-1.0);
+                let parameter = match device.role {
+                    TrackedDeviceRole::None => continue,
+                    TrackedDeviceRole::Hmd => {
+                        // legacy OVR Toolkit style (int)
+                        // as of 20 Nov 2024 OVR Toolkit uses int 0-100, but this may change in a future update.
+                        //TODO: update this once their implementation matches their docs
+                        self.send_message(
+                            "/avatar/parameters/hmdBattery".into(),
+                            vec![OscType::Int((level * 100.0f32).round() as i32)],
+                        )?;
+
+                        "headset"
+                    }
+                    TrackedDeviceRole::LeftHand => {
+                        controller_count += 1;
+                        controller_total_bat += level;
+                        "leftController"
+                    }
+                    TrackedDeviceRole::RightHand => {
+                        controller_count += 1;
+                        controller_total_bat += level;
+                        "rightController"
+                    }
+                    TrackedDeviceRole::Tracker => {
+                        tracker_count += 1;
+                        tracker_total_bat += level;
+                        tracker_param = format!("tracker{tracker_count}");
+                        tracker_param.as_str()
+                    }
+                };
+
+                // send device battery parameters
+                self.send_message(
+                    format!("/avatar/parameters/{parameter}Battery"),
+                    vec![OscType::Float(level)],
+                )?;
+                self.send_message(
+                    format!("/avatar/parameters/{parameter}Charging"),
+                    vec![OscType::Bool(device.charging)],
+                )?;
+            }
+
+            // send average controller and tracker battery parameters
+            self.send_message(
+                String::from("/avatar/parameters/averageControllerBattery"),
+                vec![OscType::Float(
+                    controller_total_bat / f32::from(controller_count),
+                )],
+            )?;
+            self.send_message(
+                String::from("/avatar/parameters/averageTrackerBattery"),
+                vec![OscType::Float(tracker_total_bat / f32::from(tracker_count))],
+            )?;
+        }
+
+        Ok(())
+    }
+
+    pub fn send_single_param(
+        &mut self,
+        parameter: String,
+        values: Vec<OscType>,
+    ) -> anyhow::Result<()> {
+        self.send_message(parameter, values)?;
+
+        Ok(())
+    }
+}

wlx-overlay-s/src/backend/overlay.rs

@@ -0,0 +1,465 @@
+use std::{
+    f32::consts::PI,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+};
+
+use anyhow::Ok;
+use glam::{Affine2, Affine3A, Mat3A, Quat, Vec2, Vec3, Vec3A};
+use serde::Deserialize;
+use vulkano::{format::Format, image::view::ImageView};
+
+use crate::{
+    config::AStrMapExt,
+    graphics::CommandBuffers,
+    state::{AppState, KeyboardFocus},
+};
+
+use super::{
+    common::snap_upright,
+    input::{DummyInteractionHandler, Haptics, InteractionHandler, PointerHit},
+};
+
+static OVERLAY_AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(0);
+
+pub trait OverlayBackend: OverlayRenderer + InteractionHandler {
+    fn set_renderer(&mut self, renderer: Box<dyn OverlayRenderer>);
+    fn set_interaction(&mut self, interaction: Box<dyn InteractionHandler>);
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)]
+pub struct OverlayID(pub usize);
+
+pub const Z_ORDER_TOAST: u32 = 70;
+pub const Z_ORDER_LINES: u32 = 69;
+pub const Z_ORDER_WATCH: u32 = 68;
+pub const Z_ORDER_ANCHOR: u32 = 67;
+pub const Z_ORDER_DEFAULT: u32 = 0;
+pub const Z_ORDER_DASHBOARD: u32 = Z_ORDER_DEFAULT;
+
+pub struct OverlayState {
+    pub id: OverlayID,
+    pub name: Arc<str>,
+    pub want_visible: bool,
+    pub show_hide: bool,
+    pub grabbable: bool,
+    pub interactable: bool,
+    pub recenter: bool,
+    pub keyboard_focus: Option<KeyboardFocus>,
+    pub dirty: bool,
+    pub alpha: f32,
+    pub z_order: u32,
+    pub transform: Affine3A,
+    pub spawn_scale: f32, // aka width
+    pub spawn_point: Vec3A,
+    pub spawn_rotation: Quat,
+    pub saved_transform: Option<Affine3A>,
+    pub positioning: Positioning,
+    pub curvature: Option<f32>,
+    pub interaction_transform: Affine2,
+    pub birthframe: usize,
+}
+
+impl Default for OverlayState {
+    fn default() -> Self {
+        Self {
+            id: OverlayID(OVERLAY_AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed)),
+            name: Arc::from(""),
+            want_visible: false,
+            show_hide: false,
+            grabbable: false,
+            recenter: false,
+            interactable: false,
+            keyboard_focus: None,
+            dirty: true,
+            alpha: 1.0,
+            z_order: Z_ORDER_DEFAULT,
+            positioning: Positioning::Floating,
+            curvature: None,
+            spawn_scale: 1.0,
+            spawn_point: Vec3A::NEG_Z,
+            spawn_rotation: Quat::IDENTITY,
+            saved_transform: None,
+            transform: Affine3A::IDENTITY,
+            interaction_transform: Affine2::IDENTITY,
+            birthframe: 0,
+        }
+    }
+}
+
+pub struct OverlayData<T>
+where
+    T: Default,
+{
+    pub state: OverlayState,
+    pub backend: Box<dyn OverlayBackend>,
+    pub primary_pointer: Option<usize>,
+    pub data: T,
+}
+
+impl<T> Default for OverlayData<T>
+where
+    T: Default,
+{
+    fn default() -> Self {
+        Self {
+            state: OverlayState::default(),
+            backend: Box::<SplitOverlayBackend>::default(),
+            primary_pointer: None,
+            data: Default::default(),
+        }
+    }
+}
+
+impl OverlayState {
+    fn get_transform(&self) -> Affine3A {
+        self.saved_transform.unwrap_or_else(|| {
+            Affine3A::from_scale_rotation_translation(
+                Vec3::ONE * self.spawn_scale,
+                self.spawn_rotation,
+                self.spawn_point.into(),
+            )
+        })
+    }
+
+    pub fn auto_movement(&mut self, app: &mut AppState) {
+        let (target_transform, lerp) = match self.positioning {
+            Positioning::FollowHead { lerp } => (app.input_state.hmd * self.get_transform(), lerp),
+            Positioning::FollowHand { hand, lerp } => (
+                app.input_state.pointers[hand].pose * self.get_transform(),
+                lerp,
+            ),
+            _ => return,
+        };
+
+        self.transform = match lerp {
+            1.0 => target_transform,
+            lerp => {
+                let scale = target_transform.matrix3.x_axis.length();
+
+                let rot_from = Quat::from_mat3a(&self.transform.matrix3.div_scalar(scale));
+                let rot_to = Quat::from_mat3a(&target_transform.matrix3.div_scalar(scale));
+
+                let rotation = rot_from.slerp(rot_to, lerp);
+                let translation = self
+                    .transform
+                    .translation
+                    .slerp(target_transform.translation, lerp);
+
+                Affine3A::from_scale_rotation_translation(
+                    Vec3::ONE * scale,
+                    rotation,
+                    translation.into(),
+                )
+            }
+        };
+
+        self.dirty = true;
+    }
+
+    pub fn reset(&mut self, app: &mut AppState, hard_reset: bool) {
+        let parent_transform = match self.positioning {
+            Positioning::Floating
+            | Positioning::FollowHead { .. }
+            | Positioning::FollowHeadPaused { .. } => app.input_state.hmd,
+            Positioning::FollowHand { hand, .. } | Positioning::FollowHandPaused { hand, .. } => {
+                app.input_state.pointers[hand].pose
+            }
+            Positioning::Anchored => app.anchor,
+            Positioning::Static => return,
+        };
+
+        if hard_reset {
+            self.saved_transform = None;
+        }
+
+        self.transform = parent_transform * self.get_transform();
+
+        if self.grabbable && hard_reset {
+            self.realign(&app.input_state.hmd);
+        }
+        self.dirty = true;
+    }
+
+    pub fn save_transform(&mut self, app: &mut AppState) -> bool {
+        let parent_transform = match self.positioning {
+            Positioning::Floating => snap_upright(app.input_state.hmd, Vec3A::Y),
+            Positioning::FollowHead { .. } | Positioning::FollowHeadPaused { .. } => {
+                app.input_state.hmd
+            }
+            Positioning::FollowHand { hand, .. } | Positioning::FollowHandPaused { hand, .. } => {
+                app.input_state.pointers[hand].pose
+            }
+            Positioning::Anchored => snap_upright(app.anchor, Vec3A::Y),
+            Positioning::Static => return false,
+        };
+
+        self.saved_transform = Some(parent_transform.inverse() * self.transform);
+
+        true
+    }
+
+    pub fn realign(&mut self, hmd: &Affine3A) {
+        let to_hmd = hmd.translation - self.transform.translation;
+        let up_dir: Vec3A;
+
+        if hmd.x_axis.dot(Vec3A::Y).abs() > 0.2 {
+            // Snap upright
+            up_dir = hmd.y_axis;
+        } else {
+            let dot = to_hmd.normalize().dot(hmd.z_axis);
+            let z_dist = to_hmd.length();
+            let y_dist = (self.transform.translation.y - hmd.translation.y).abs();
+            let x_angle = (y_dist / z_dist).asin();
+
+            if dot < -f32::EPSILON {
+                // facing down
+                let up_point = hmd.translation + z_dist / x_angle.cos() * Vec3A::Y;
+                up_dir = (up_point - self.transform.translation).normalize();
+            } else if dot > f32::EPSILON {
+                // facing up
+                let dn_point = hmd.translation + z_dist / x_angle.cos() * Vec3A::NEG_Y;
+                up_dir = (self.transform.translation - dn_point).normalize();
+            } else {
+                // perfectly upright
+                up_dir = Vec3A::Y;
+            }
+        }
+
+        let scale = self.transform.x_axis.length();
+
+        let col_z = (self.transform.translation - hmd.translation).normalize();
+        let col_y = up_dir;
+        let col_x = col_y.cross(col_z);
+        let col_y = col_z.cross(col_x).normalize();
+        let col_x = col_x.normalize();
+
+        let rot = Mat3A::from_quat(self.spawn_rotation)
+            * Mat3A::from_quat(Quat::from_axis_angle(Vec3::Y, PI));
+        self.transform.matrix3 = Mat3A::from_cols(col_x, col_y, col_z).mul_scalar(scale) * rot;
+    }
+}
+
+impl<T> OverlayData<T>
+where
+    T: Default,
+{
+    pub fn init(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        if self.state.curvature.is_none() {
+            self.state.curvature = app
+                .session
+                .config
+                .curve_values
+                .arc_get(self.state.name.as_ref())
+                .copied();
+        }
+
+        if matches!(
+            self.state.positioning,
+            Positioning::Floating | Positioning::Anchored
+        ) {
+            let hard_reset;
+            if let Some(transform) = app
+                .session
+                .config
+                .transform_values
+                .arc_get(self.state.name.as_ref())
+            {
+                self.state.saved_transform = Some(*transform);
+                hard_reset = false;
+            } else {
+                hard_reset = true;
+            }
+            self.state.reset(app, hard_reset);
+        }
+        self.backend.init(app)
+    }
+    pub fn should_render(&mut self, app: &mut AppState) -> anyhow::Result<ShouldRender> {
+        self.backend.should_render(app)
+    }
+    pub fn render(
+        &mut self,
+        app: &mut AppState,
+        tgt: Arc<ImageView>,
+        buf: &mut CommandBuffers,
+        alpha: f32,
+    ) -> anyhow::Result<bool> {
+        self.backend.render(app, tgt, buf, alpha)
+    }
+    pub fn frame_meta(&mut self) -> Option<FrameMeta> {
+        self.backend.frame_meta()
+    }
+}
+
+#[derive(Default, Clone, Copy)]
+pub struct FrameMeta {
+    pub extent: [u32; 3],
+    pub transform: Affine3A,
+    pub format: Format,
+}
+
+pub enum ShouldRender {
+    /// The overlay is dirty and needs to be rendered.
+    Should,
+    /// The overlay is not dirty but is ready to be rendered.
+    Can,
+    /// The overlay is not ready to be rendered.
+    Unable,
+}
+
+pub trait OverlayRenderer {
+    /// Called once, before the first frame is rendered
+    fn init(&mut self, app: &mut AppState) -> anyhow::Result<()>;
+    fn pause(&mut self, app: &mut AppState) -> anyhow::Result<()>;
+    fn resume(&mut self, app: &mut AppState) -> anyhow::Result<()>;
+
+    /// Called when the presentation layer is ready to present a new frame
+    fn should_render(&mut self, app: &mut AppState) -> anyhow::Result<ShouldRender>;
+
+    /// Called when the contents need to be rendered to the swapchain
+    fn render(
+        &mut self,
+        app: &mut AppState,
+        tgt: Arc<ImageView>,
+        buf: &mut CommandBuffers,
+        alpha: f32,
+    ) -> anyhow::Result<bool>;
+
+    /// Called to retrieve the effective extent of the image
+    /// Used for creating swapchains.
+    ///
+    /// Must be true if should_render was also true on the same frame.
+    fn frame_meta(&mut self) -> Option<FrameMeta>;
+}
+
+pub struct FallbackRenderer;
+
+impl OverlayRenderer for FallbackRenderer {
+    fn init(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn pause(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn resume(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn should_render(&mut self, _app: &mut AppState) -> anyhow::Result<ShouldRender> {
+        Ok(ShouldRender::Unable)
+    }
+    fn render(
+        &mut self,
+        _app: &mut AppState,
+        _tgt: Arc<ImageView>,
+        _buf: &mut CommandBuffers,
+        _alpha: f32,
+    ) -> anyhow::Result<bool> {
+        Ok(false)
+    }
+    fn frame_meta(&mut self) -> Option<FrameMeta> {
+        None
+    }
+}
+// Boilerplate and dummies
+
+#[derive(Clone, Copy, Debug, Default)]
+pub enum Positioning {
+    /// Stays in place unless recentered, recenters relative to HMD
+    #[default]
+    Floating,
+    /// Stays in place unless recentered, recenters relative to anchor
+    Anchored,
+    /// Following HMD
+    FollowHead { lerp: f32 },
+    /// Normally follows HMD, but paused due to interaction
+    FollowHeadPaused { lerp: f32 },
+    /// Following hand
+    FollowHand { hand: usize, lerp: f32 },
+    /// Normally follows hand, but paused due to interaction
+    FollowHandPaused { hand: usize, lerp: f32 },
+    /// Stays in place, no recentering
+    Static,
+}
+
+pub struct SplitOverlayBackend {
+    pub renderer: Box<dyn OverlayRenderer>,
+    pub interaction: Box<dyn InteractionHandler>,
+}
+
+impl Default for SplitOverlayBackend {
+    fn default() -> Self {
+        Self {
+            renderer: Box::new(FallbackRenderer),
+            interaction: Box::new(DummyInteractionHandler),
+        }
+    }
+}
+
+impl OverlayBackend for SplitOverlayBackend {
+    fn set_renderer(&mut self, renderer: Box<dyn OverlayRenderer>) {
+        self.renderer = renderer;
+    }
+    fn set_interaction(&mut self, interaction: Box<dyn InteractionHandler>) {
+        self.interaction = interaction;
+    }
+}
+impl OverlayRenderer for SplitOverlayBackend {
+    fn init(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.renderer.init(app)
+    }
+    fn pause(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.renderer.pause(app)
+    }
+    fn resume(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.renderer.resume(app)
+    }
+    fn should_render(&mut self, app: &mut AppState) -> anyhow::Result<ShouldRender> {
+        self.renderer.should_render(app)
+    }
+    fn render(
+        &mut self,
+        app: &mut AppState,
+        tgt: Arc<ImageView>,
+        buf: &mut CommandBuffers,
+        alpha: f32,
+    ) -> anyhow::Result<bool> {
+        self.renderer.render(app, tgt, buf, alpha)
+    }
+    fn frame_meta(&mut self) -> Option<FrameMeta> {
+        self.renderer.frame_meta()
+    }
+}
+
+impl InteractionHandler for SplitOverlayBackend {
+    fn on_left(&mut self, app: &mut AppState, pointer: usize) {
+        self.interaction.on_left(app, pointer);
+    }
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> Option<Haptics> {
+        self.interaction.on_hover(app, hit)
+    }
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta_y: f32, delta_x: f32) {
+        self.interaction.on_scroll(app, hit, delta_y, delta_x);
+    }
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
+        self.interaction.on_pointer(app, hit, pressed);
+    }
+}
+
+pub fn ui_transform(extent: [u32; 2]) -> Affine2 {
+    let aspect = extent[0] as f32 / extent[1] as f32;
+    let scale = if aspect < 1.0 {
+        Vec2 {
+            x: 1.0 / aspect,
+            y: -1.0,
+        }
+    } else {
+        Vec2 {
+            x: 1.0,
+            y: -1.0 * aspect,
+        }
+    };
+    let center = Vec2 { x: 0.5, y: 0.5 };
+    Affine2::from_scale_angle_translation(scale, 0.0, center)
+}

wlx-overlay-s/src/backend/task.rs

@@ -0,0 +1,118 @@
+use std::{
+    cmp,
+    collections::{BinaryHeap, VecDeque},
+    sync::atomic::{self, AtomicUsize},
+    time::Instant,
+};
+
+use serde::Deserialize;
+
+use crate::state::AppState;
+
+#[cfg(feature = "wayvr")]
+use crate::backend::wayvr::WayVRAction;
+
+use super::{
+    common::OverlaySelector,
+    overlay::{OverlayBackend, OverlayState},
+};
+
+static TASK_AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(0);
+
+struct AppTask {
+    pub not_before: Instant,
+    pub id: usize,
+    pub task: TaskType,
+}
+
+impl PartialEq<Self> for AppTask {
+    fn eq(&self, other: &Self) -> bool {
+        self.cmp(other) == cmp::Ordering::Equal
+    }
+}
+impl PartialOrd<Self> for AppTask {
+    fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+impl Eq for AppTask {}
+impl Ord for AppTask {
+    fn cmp(&self, other: &Self) -> cmp::Ordering {
+        self.not_before
+            .cmp(&other.not_before)
+            .then(self.id.cmp(&other.id))
+            .reverse()
+    }
+}
+
+pub enum SystemTask {
+    ColorGain(ColorChannel, f32),
+    ResetPlayspace,
+    FixFloor,
+    ShowHide,
+}
+
+pub type OverlayTask = dyn FnOnce(&mut AppState, &mut OverlayState) + Send;
+pub type CreateOverlayTask =
+    dyn FnOnce(&mut AppState) -> Option<(OverlayState, Box<dyn OverlayBackend>)> + Send;
+
+pub enum TaskType {
+    Overlay(OverlaySelector, Box<OverlayTask>),
+    CreateOverlay(OverlaySelector, Box<CreateOverlayTask>),
+    DropOverlay(OverlaySelector),
+    System(SystemTask),
+    #[cfg(feature = "wayvr")]
+    WayVR(WayVRAction),
+}
+
+#[derive(Deserialize, Clone, Copy)]
+pub enum ColorChannel {
+    R,
+    G,
+    B,
+    All,
+}
+
+pub struct TaskContainer {
+    tasks: BinaryHeap<AppTask>,
+}
+
+impl TaskContainer {
+    pub const fn new() -> Self {
+        Self {
+            tasks: BinaryHeap::new(),
+        }
+    }
+
+    pub fn enqueue(&mut self, task: TaskType) {
+        self.tasks.push(AppTask {
+            not_before: Instant::now(),
+            id: TASK_AUTO_INCREMENT.fetch_add(1, atomic::Ordering::Relaxed),
+            task,
+        });
+    }
+
+    /// Enqueue a task to be executed at a specific time.
+    /// If the time is in the past, the task will be executed immediately.
+    /// Multiple tasks enqueued for the same instant will be executed in order of submission.
+    pub fn enqueue_at(&mut self, task: TaskType, not_before: Instant) {
+        self.tasks.push(AppTask {
+            not_before,
+            id: TASK_AUTO_INCREMENT.fetch_add(1, atomic::Ordering::Relaxed),
+            task,
+        });
+    }
+
+    pub fn retrieve_due(&mut self, dest_buf: &mut VecDeque<TaskType>) {
+        let now = Instant::now();
+
+        while let Some(task) = self.tasks.peek() {
+            if task.not_before > now {
+                break;
+            }
+
+            // Safe unwrap because we peeked.
+            dest_buf.push_back(self.tasks.pop().unwrap().task);
+        }
+    }
+}

wlx-overlay-s/src/backend/wayvr/client.rs

@@ -0,0 +1,252 @@
+use std::{io::Read, os::unix::net::UnixStream, path::PathBuf, sync::Arc};
+
+use smithay::{
+    backend::input::Keycode,
+    input::{keyboard::KeyboardHandle, pointer::PointerHandle},
+    reexports::wayland_server,
+    utils::SerialCounter,
+};
+
+use crate::backend::wayvr::{ExternalProcessRequest, WayVRTask};
+
+use super::{
+    comp::{self, ClientState},
+    display, process, ProcessWayVREnv,
+};
+
+pub struct WayVRClient {
+    pub client: wayland_server::Client,
+    pub display_handle: display::DisplayHandle,
+    pub pid: u32,
+}
+
+pub struct WayVRCompositor {
+    pub state: comp::Application,
+    pub seat_keyboard: KeyboardHandle<comp::Application>,
+    pub seat_pointer: PointerHandle<comp::Application>,
+    pub serial_counter: SerialCounter,
+    pub wayland_env: super::WaylandEnv,
+
+    display: wayland_server::Display<comp::Application>,
+    listener: wayland_server::ListeningSocket,
+
+    toplevel_surf_count: u32, // for logging purposes
+
+    pub clients: Vec<WayVRClient>,
+}
+
+fn get_wayvr_env_from_pid(pid: i32) -> anyhow::Result<ProcessWayVREnv> {
+    let path = format!("/proc/{pid}/environ");
+    let mut env_data = String::new();
+    std::fs::File::open(path)?.read_to_string(&mut env_data)?;
+
+    let lines: Vec<&str> = env_data.split('\0').filter(|s| !s.is_empty()).collect();
+
+    let mut env = ProcessWayVREnv {
+        display_auth: None,
+        display_name: None,
+    };
+
+    for line in lines {
+        if let Some((key, value)) = line.split_once('=') {
+            if key == "WAYVR_DISPLAY_AUTH" {
+                env.display_auth = Some(String::from(value));
+            } else if key == "WAYVR_DISPLAY_NAME" {
+                env.display_name = Some(String::from(value));
+            }
+        }
+    }
+
+    Ok(env)
+}
+
+impl WayVRCompositor {
+    pub fn new(
+        state: comp::Application,
+        display: wayland_server::Display<comp::Application>,
+        seat_keyboard: KeyboardHandle<comp::Application>,
+        seat_pointer: PointerHandle<comp::Application>,
+    ) -> anyhow::Result<Self> {
+        let (wayland_env, listener) = create_wayland_listener()?;
+
+        Ok(Self {
+            state,
+            display,
+            seat_keyboard,
+            seat_pointer,
+            listener,
+            wayland_env,
+            serial_counter: SerialCounter::new(),
+            clients: Vec::new(),
+            toplevel_surf_count: 0,
+        })
+    }
+
+    pub fn add_client(&mut self, client: WayVRClient) {
+        self.clients.push(client);
+    }
+
+    pub fn cleanup_clients(&mut self) {
+        self.clients.retain(|client| {
+            let Some(data) = client.client.get_data::<ClientState>() else {
+                return false;
+            };
+
+            if *data.disconnected.lock().unwrap() {
+                return false;
+            }
+
+            true
+        });
+    }
+
+    fn accept_connection(
+        &mut self,
+        stream: UnixStream,
+        displays: &mut display::DisplayVec,
+        processes: &mut process::ProcessVec,
+    ) -> anyhow::Result<()> {
+        let client = self
+            .display
+            .handle()
+            .insert_client(stream, Arc::new(comp::ClientState::default()))
+            .unwrap();
+
+        let creds = client.get_credentials(&self.display.handle())?;
+
+        let process_env = get_wayvr_env_from_pid(creds.pid)?;
+
+        // Find suitable auth key from the process list
+        for p in processes.vec.iter().flatten() {
+            if let process::Process::Managed(process) = &p.obj {
+                if let Some(auth_key) = &process_env.display_auth {
+                    // Find process with matching auth key
+                    if process.auth_key.as_str() == auth_key {
+                        // Check if display handle is valid
+                        if displays.get(&process.display_handle).is_some() {
+                            // Add client
+                            self.add_client(WayVRClient {
+                                client,
+                                display_handle: process.display_handle,
+                                pid: creds.pid as u32,
+                            });
+                            return Ok(());
+                        }
+                    }
+                }
+            }
+        }
+
+        // This is a new process which we didn't met before.
+        // Treat external processes exclusively (spawned by the user or external program)
+        log::warn!(
+            "External process ID {} connected to this Wayland server",
+            creds.pid
+        );
+
+        self.state
+            .wayvr_tasks
+            .send(WayVRTask::NewExternalProcess(ExternalProcessRequest {
+                env: process_env,
+                client,
+                pid: creds.pid as u32,
+            }));
+
+        Ok(())
+    }
+
+    fn accept_connections(
+        &mut self,
+        displays: &mut display::DisplayVec,
+        processes: &mut process::ProcessVec,
+    ) -> anyhow::Result<()> {
+        if let Some(stream) = self.listener.accept()? {
+            if let Err(e) = self.accept_connection(stream, displays, processes) {
+                log::error!("Failed to accept connection: {e}");
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn tick_wayland(
+        &mut self,
+        displays: &mut display::DisplayVec,
+        processes: &mut process::ProcessVec,
+    ) -> anyhow::Result<()> {
+        if let Err(e) = self.accept_connections(displays, processes) {
+            log::error!("accept_connections failed: {e}");
+        }
+
+        self.display.dispatch_clients(&mut self.state)?;
+        self.display.flush_clients()?;
+
+        let surf_count = self.state.xdg_shell.toplevel_surfaces().len() as u32;
+        if surf_count != self.toplevel_surf_count {
+            self.toplevel_surf_count = surf_count;
+            log::info!("Toplevel surface count changed: {surf_count}");
+        }
+
+        Ok(())
+    }
+
+    pub fn send_key(&mut self, virtual_key: u32, down: bool) {
+        let state = if down {
+            smithay::backend::input::KeyState::Pressed
+        } else {
+            smithay::backend::input::KeyState::Released
+        };
+
+        self.seat_keyboard.input::<(), _>(
+            &mut self.state,
+            Keycode::new(virtual_key),
+            state,
+            self.serial_counter.next_serial(),
+            0,
+            |_, _, _| smithay::input::keyboard::FilterResult::Forward,
+        );
+    }
+}
+
+const STARTING_WAYLAND_ADDR_IDX: u32 = 20;
+
+fn export_display_number(display_num: u32) -> anyhow::Result<()> {
+    let mut path =
+        std::env::var("XDG_RUNTIME_DIR").map_or_else(|_| PathBuf::from("/tmp"), PathBuf::from);
+    path.push("wayvr.disp");
+    std::fs::write(path, format!("{display_num}\n"))?;
+    Ok(())
+}
+
+fn create_wayland_listener() -> anyhow::Result<(super::WaylandEnv, wayland_server::ListeningSocket)>
+{
+    let mut env = super::WaylandEnv {
+        display_num: STARTING_WAYLAND_ADDR_IDX,
+    };
+
+    let listener = loop {
+        let display_str = env.display_num_string();
+        log::debug!("Trying to open socket \"{display_str}\"");
+        match wayland_server::ListeningSocket::bind(display_str.as_str()) {
+            Ok(listener) => {
+                log::debug!("Listening to {display_str}");
+                break listener;
+            }
+            Err(e) => {
+                log::debug!(
+                    "Failed to open socket \"{display_str}\" (reason: {e}), trying next..."
+                );
+
+                env.display_num += 1;
+                if env.display_num > STARTING_WAYLAND_ADDR_IDX + 20 {
+                    // Highly unlikely for the user to have 20 Wayland displays enabled at once. Return error instead.
+                    anyhow::bail!("Failed to create wayland-server socket")
+                }
+            }
+        }
+    };
+
+    let _ = export_display_number(env.display_num);
+
+    Ok((env, listener))
+}

wlx-overlay-s/src/backend/wayvr/comp.rs

@@ -0,0 +1,236 @@
+use smithay::backend::allocator::dmabuf::Dmabuf;
+use smithay::backend::renderer::gles::GlesRenderer;
+use smithay::backend::renderer::utils::on_commit_buffer_handler;
+use smithay::backend::renderer::ImportDma;
+use smithay::input::{Seat, SeatHandler, SeatState};
+use smithay::reexports::wayland_protocols::xdg::shell::server::xdg_toplevel;
+use smithay::reexports::wayland_server;
+use smithay::reexports::wayland_server::protocol::{wl_buffer, wl_seat, wl_surface};
+use smithay::reexports::wayland_server::Resource;
+use smithay::wayland::buffer::BufferHandler;
+use smithay::wayland::dmabuf::{
+    DmabufFeedback, DmabufGlobal, DmabufHandler, DmabufState, ImportNotifier,
+};
+use smithay::wayland::output::OutputHandler;
+use smithay::wayland::shm::{ShmHandler, ShmState};
+use smithay::{
+    delegate_compositor, delegate_data_device, delegate_dmabuf, delegate_output, delegate_seat,
+    delegate_shm, delegate_xdg_shell,
+};
+use std::collections::HashSet;
+use std::os::fd::OwnedFd;
+use std::sync::{Arc, Mutex};
+
+use smithay::utils::Serial;
+use smithay::wayland::compositor::{
+    self, with_surface_tree_downward, SurfaceAttributes, TraversalAction,
+};
+
+use smithay::wayland::selection::data_device::{
+    ClientDndGrabHandler, DataDeviceHandler, DataDeviceState, ServerDndGrabHandler,
+};
+use smithay::wayland::selection::SelectionHandler;
+use smithay::wayland::shell::xdg::{
+    PopupSurface, PositionerState, ToplevelSurface, XdgShellHandler, XdgShellState,
+};
+use wayland_server::backend::{ClientData, ClientId, DisconnectReason};
+use wayland_server::protocol::wl_surface::WlSurface;
+use wayland_server::Client;
+
+use super::event_queue::SyncEventQueue;
+use super::WayVRTask;
+
+pub struct Application {
+    pub gles_renderer: GlesRenderer,
+    pub dmabuf_state: (DmabufState, DmabufGlobal, Option<DmabufFeedback>),
+    pub compositor: compositor::CompositorState,
+    pub xdg_shell: XdgShellState,
+    pub seat_state: SeatState<Application>,
+    pub shm: ShmState,
+    pub data_device: DataDeviceState,
+    pub wayvr_tasks: SyncEventQueue<WayVRTask>,
+    pub redraw_requests: HashSet<wayland_server::backend::ObjectId>,
+}
+
+impl Application {
+    pub fn check_redraw(&mut self, surface: &WlSurface) -> bool {
+        self.redraw_requests.remove(&surface.id())
+    }
+}
+
+impl compositor::CompositorHandler for Application {
+    fn compositor_state(&mut self) -> &mut compositor::CompositorState {
+        &mut self.compositor
+    }
+
+    fn client_compositor_state<'a>(
+        &self,
+        client: &'a Client,
+    ) -> &'a compositor::CompositorClientState {
+        &client.get_data::<ClientState>().unwrap().compositor_state
+    }
+
+    fn commit(&mut self, surface: &WlSurface) {
+        on_commit_buffer_handler::<Self>(surface);
+        self.redraw_requests.insert(surface.id());
+    }
+}
+
+impl SeatHandler for Application {
+    type KeyboardFocus = WlSurface;
+    type PointerFocus = WlSurface;
+    type TouchFocus = WlSurface;
+
+    fn seat_state(&mut self) -> &mut SeatState<Self> {
+        &mut self.seat_state
+    }
+
+    fn focus_changed(&mut self, _seat: &Seat<Self>, _focused: Option<&WlSurface>) {}
+    fn cursor_image(
+        &mut self,
+        _seat: &Seat<Self>,
+        _image: smithay::input::pointer::CursorImageStatus,
+    ) {
+    }
+}
+
+impl BufferHandler for Application {
+    fn buffer_destroyed(&mut self, _buffer: &wl_buffer::WlBuffer) {}
+}
+
+impl ClientDndGrabHandler for Application {}
+
+impl ServerDndGrabHandler for Application {
+    fn send(&mut self, _mime_type: String, _fd: OwnedFd, _seat: Seat<Self>) {}
+}
+
+impl DataDeviceHandler for Application {
+    fn data_device_state(&self) -> &DataDeviceState {
+        &self.data_device
+    }
+}
+
+impl SelectionHandler for Application {
+    type SelectionUserData = ();
+}
+
+#[derive(Default)]
+pub struct ClientState {
+    compositor_state: compositor::CompositorClientState,
+    pub disconnected: Arc<Mutex<bool>>,
+}
+
+impl ClientData for ClientState {
+    fn initialized(&self, client_id: ClientId) {
+        log::debug!("Client ID {client_id:?} connected");
+    }
+
+    fn disconnected(&self, client_id: ClientId, reason: DisconnectReason) {
+        *self.disconnected.lock().unwrap() = true;
+        log::debug!("Client ID {client_id:?} disconnected. Reason: {reason:?}");
+    }
+}
+
+impl AsMut<compositor::CompositorState> for Application {
+    fn as_mut(&mut self) -> &mut compositor::CompositorState {
+        &mut self.compositor
+    }
+}
+
+impl XdgShellHandler for Application {
+    fn xdg_shell_state(&mut self) -> &mut XdgShellState {
+        &mut self.xdg_shell
+    }
+
+    fn new_toplevel(&mut self, surface: ToplevelSurface) {
+        if let Some(client) = surface.wl_surface().client() {
+            self.wayvr_tasks
+                .send(WayVRTask::NewToplevel(client.id(), surface.clone()));
+        }
+        surface.with_pending_state(|state| {
+            state.states.set(xdg_toplevel::State::Activated);
+        });
+        surface.send_configure();
+    }
+
+    fn toplevel_destroyed(&mut self, surface: ToplevelSurface) {
+        if let Some(client) = surface.wl_surface().client() {
+            self.wayvr_tasks
+                .send(WayVRTask::DropToplevel(client.id(), surface.clone()));
+        }
+    }
+
+    fn new_popup(&mut self, _surface: PopupSurface, _positioner: PositionerState) {
+        // Handle popup creation here
+    }
+
+    fn grab(&mut self, _surface: PopupSurface, _seat: wl_seat::WlSeat, _serial: Serial) {
+        // Handle popup grab here
+    }
+
+    fn reposition_request(
+        &mut self,
+        _surface: PopupSurface,
+        _positioner: PositionerState,
+        _token: u32,
+    ) {
+        // Handle popup reposition here
+    }
+}
+
+impl ShmHandler for Application {
+    fn shm_state(&self) -> &ShmState {
+        &self.shm
+    }
+}
+
+impl OutputHandler for Application {}
+
+impl DmabufHandler for Application {
+    fn dmabuf_state(&mut self) -> &mut DmabufState {
+        &mut self.dmabuf_state.0
+    }
+
+    fn dmabuf_imported(
+        &mut self,
+        _global: &DmabufGlobal,
+        dmabuf: Dmabuf,
+        notifier: ImportNotifier,
+    ) {
+        if self.gles_renderer.import_dmabuf(&dmabuf, None).is_ok() {
+            let _ = notifier.successful::<Self>();
+        } else {
+            notifier.failed();
+        }
+    }
+}
+
+delegate_dmabuf!(Application);
+delegate_xdg_shell!(Application);
+delegate_compositor!(Application);
+delegate_shm!(Application);
+delegate_seat!(Application);
+delegate_data_device!(Application);
+delegate_output!(Application);
+
+pub fn send_frames_surface_tree(surface: &wl_surface::WlSurface, time: u32) {
+    with_surface_tree_downward(
+        surface,
+        (),
+        |_, _, &()| TraversalAction::DoChildren(()),
+        |_surf, states, &()| {
+            // the surface may not have any user_data if it is a subsurface and has not
+            // yet been commited
+            for callback in states
+                .cached_state
+                .get::<SurfaceAttributes>()
+                .current()
+                .frame_callbacks
+                .drain(..)
+            {
+                callback.done(time);
+            }
+        },
+        |_, _, &()| true,
+    );
+}

wlx-overlay-s/src/backend/wayvr/display.rs

@@ -0,0 +1,605 @@
+use std::{cell::RefCell, rc::Rc, sync::Arc};
+
+use smithay::{
+    backend::renderer::{
+        element::{
+            surface::{render_elements_from_surface_tree, WaylandSurfaceRenderElement},
+            Kind,
+        },
+        gles::{ffi, GlesRenderer, GlesTexture},
+        utils::draw_render_elements,
+        Bind, Color32F, Frame, Renderer,
+    },
+    input,
+    utils::{Logical, Point, Rectangle, Size, Transform},
+    wayland::shell::xdg::ToplevelSurface,
+};
+use wayvr_ipc::packet_server;
+
+use crate::{
+    backend::{overlay::OverlayID, wayvr::time::get_millis},
+    gen_id,
+};
+
+use super::{
+    client::WayVRCompositor, comp::send_frames_surface_tree, egl_data, event_queue::SyncEventQueue,
+    process, smithay_wrapper, time, window, BlitMethod, WayVRSignal,
+};
+
+fn generate_auth_key() -> String {
+    let uuid = uuid::Uuid::new_v4();
+    uuid.to_string()
+}
+
+#[derive(Debug)]
+pub struct DisplayWindow {
+    pub window_handle: window::WindowHandle,
+    pub toplevel: ToplevelSurface,
+    pub process_handle: process::ProcessHandle,
+}
+
+pub struct SpawnProcessResult {
+    pub auth_key: String,
+    pub child: std::process::Child,
+}
+
+#[derive(Debug)]
+pub enum DisplayTask {
+    ProcessCleanup(process::ProcessHandle),
+}
+
+const MAX_DISPLAY_SIZE: u16 = 8192;
+
+#[derive(Debug)]
+pub struct Display {
+    // Display info stuff
+    pub width: u16,
+    pub height: u16,
+    pub name: String,
+    pub visible: bool,
+    pub layout: packet_server::WvrDisplayWindowLayout,
+    pub overlay_id: Option<OverlayID>,
+    pub wants_redraw: bool,
+    pub rendered_frame_count: u32,
+    pub primary: bool,
+    pub wm: Rc<RefCell<window::WindowManager>>,
+    pub displayed_windows: Vec<DisplayWindow>,
+    wayland_env: super::WaylandEnv,
+    last_pressed_time_ms: u64,
+    pub no_windows_since: Option<u64>,
+
+    // Render data stuff
+    gles_texture: GlesTexture, // TODO: drop texture
+    egl_image: khronos_egl::Image,
+    egl_data: Rc<egl_data::EGLData>,
+
+    pub render_data: egl_data::RenderData,
+
+    pub tasks: SyncEventQueue<DisplayTask>,
+}
+
+impl Drop for Display {
+    fn drop(&mut self) {
+        let _ = self
+            .egl_data
+            .egl
+            .destroy_image(self.egl_data.display, self.egl_image);
+    }
+}
+
+pub struct DisplayInitParams<'a> {
+    pub wm: Rc<RefCell<window::WindowManager>>,
+    pub config: &'a super::Config,
+    pub renderer: &'a mut GlesRenderer,
+    pub egl_data: Rc<egl_data::EGLData>,
+    pub wayland_env: super::WaylandEnv,
+    pub width: u16,
+    pub height: u16,
+    pub name: &'a str,
+    pub primary: bool,
+}
+
+impl Display {
+    pub fn new(params: DisplayInitParams) -> anyhow::Result<Self> {
+        if params.width > MAX_DISPLAY_SIZE {
+            anyhow::bail!(
+                "display width ({}) is larger than {}",
+                params.width,
+                MAX_DISPLAY_SIZE
+            );
+        }
+
+        if params.height > MAX_DISPLAY_SIZE {
+            anyhow::bail!(
+                "display height ({}) is larger than {}",
+                params.height,
+                MAX_DISPLAY_SIZE
+            );
+        }
+
+        let tex_format = ffi::RGBA;
+        let internal_format = ffi::RGBA8;
+
+        let tex_id = params.renderer.with_context(|gl| {
+            smithay_wrapper::create_framebuffer_texture(
+                gl,
+                u32::from(params.width),
+                u32::from(params.height),
+                tex_format,
+                internal_format,
+            )
+        })?;
+
+        let egl_image = params.egl_data.create_egl_image(tex_id)?;
+
+        let render_data = match params.config.blit_method {
+            BlitMethod::Dmabuf => match params.egl_data.create_dmabuf_data(&egl_image) {
+                Ok(dmabuf_data) => egl_data::RenderData::Dmabuf(dmabuf_data),
+                Err(e) => {
+                    log::error!("create_dmabuf_data failed: {e:?}. Using software blitting (This will be slow!)");
+                    egl_data::RenderData::Software(None)
+                }
+            },
+            BlitMethod::Software => egl_data::RenderData::Software(None),
+        };
+
+        let opaque = false;
+        let size = (i32::from(params.width), i32::from(params.height)).into();
+        let gles_texture = unsafe {
+            GlesTexture::from_raw(params.renderer, Some(tex_format), opaque, tex_id, size)
+        };
+
+        Ok(Self {
+            egl_data: params.egl_data,
+            width: params.width,
+            height: params.height,
+            name: String::from(params.name),
+            primary: params.primary,
+            wayland_env: params.wayland_env,
+            wm: params.wm,
+            displayed_windows: Vec::new(),
+            render_data,
+            egl_image,
+            gles_texture,
+            last_pressed_time_ms: 0,
+            no_windows_since: None,
+            overlay_id: None,
+            tasks: SyncEventQueue::new(),
+            visible: true,
+            wants_redraw: true,
+            rendered_frame_count: 0,
+            layout: packet_server::WvrDisplayWindowLayout::Tiling,
+        })
+    }
+
+    pub fn as_packet(&self, handle: DisplayHandle) -> packet_server::WvrDisplay {
+        packet_server::WvrDisplay {
+            width: self.width,
+            height: self.height,
+            name: self.name.clone(),
+            visible: self.visible,
+            handle: handle.as_packet(),
+        }
+    }
+
+    pub fn add_window(
+        &mut self,
+        window_handle: window::WindowHandle,
+        process_handle: process::ProcessHandle,
+        toplevel: &ToplevelSurface,
+    ) {
+        log::debug!("Attaching toplevel surface into display");
+        self.displayed_windows.push(DisplayWindow {
+            window_handle,
+            process_handle,
+            toplevel: toplevel.clone(),
+        });
+        self.reposition_windows();
+    }
+
+    pub fn remove_window(&mut self, window_handle: window::WindowHandle) {
+        self.displayed_windows
+            .retain(|disp| disp.window_handle != window_handle);
+    }
+
+    pub fn reposition_windows(&mut self) {
+        let window_count = self.displayed_windows.len();
+
+        match &self.layout {
+            packet_server::WvrDisplayWindowLayout::Tiling => {
+                let mut i = 0;
+                for win in &mut self.displayed_windows {
+                    if let Some(window) = self.wm.borrow_mut().windows.get_mut(&win.window_handle) {
+                        if !window.visible {
+                            continue;
+                        }
+                        let d_cur = i as f32 / window_count as f32;
+                        let d_next = (i + 1) as f32 / window_count as f32;
+
+                        let left = (d_cur * f32::from(self.width)) as i32;
+                        let right = (d_next * f32::from(self.width)) as i32;
+
+                        window.set_pos(left, 0);
+                        window.set_size((right - left) as u32, u32::from(self.height));
+                        i += 1;
+                    }
+                }
+            }
+            packet_server::WvrDisplayWindowLayout::Stacking(opts) => {
+                let do_margins = |margins: &packet_server::Margins, window: &mut window::Window| {
+                    let top = i32::from(margins.top);
+                    let bottom = i32::from(self.height) - i32::from(margins.bottom);
+                    let left = i32::from(margins.left);
+                    let right = i32::from(self.width) - i32::from(margins.right);
+                    let width = right - left;
+                    let height = bottom - top;
+                    if width < 0 || height < 0 {
+                        return; // wrong parameters, do nothing!
+                    }
+
+                    window.set_pos(left, top);
+                    window.set_size(width as u32, height as u32);
+                };
+
+                let mut i = 0;
+                for win in &mut self.displayed_windows {
+                    if let Some(window) = self.wm.borrow_mut().windows.get_mut(&win.window_handle) {
+                        if !window.visible {
+                            continue;
+                        }
+                        do_margins(
+                            if i == 0 {
+                                &opts.margins_first
+                            } else {
+                                &opts.margins_rest
+                            },
+                            window,
+                        );
+                        i += 1;
+                    }
+                }
+            }
+        }
+    }
+
+    pub fn tick(
+        &mut self,
+        config: &super::Config,
+        handle: &DisplayHandle,
+        signals: &mut SyncEventQueue<WayVRSignal>,
+    ) {
+        if self.visible {
+            if !self.displayed_windows.is_empty() {
+                self.no_windows_since = None;
+            } else if let Some(auto_hide_delay) = config.auto_hide_delay {
+                if let Some(s) = self.no_windows_since {
+                    if s + u64::from(auto_hide_delay) < get_millis() {
+                        // Auto-hide after specific time
+                        signals.send(WayVRSignal::DisplayVisibility(*handle, false));
+                    }
+                }
+            }
+        }
+
+        while let Some(task) = self.tasks.read() {
+            match task {
+                DisplayTask::ProcessCleanup(process_handle) => {
+                    let count = self.displayed_windows.len();
+                    self.displayed_windows
+                        .retain(|win| win.process_handle != process_handle);
+                    log::info!(
+                        "Cleanup finished for display \"{}\". Current window count: {}",
+                        self.name,
+                        self.displayed_windows.len()
+                    );
+                    self.no_windows_since = Some(get_millis());
+
+                    if count != self.displayed_windows.len() {
+                        signals.send(WayVRSignal::BroadcastStateChanged(
+                            packet_server::WvrStateChanged::WindowRemoved,
+                        ));
+                    }
+
+                    self.reposition_windows();
+                }
+            }
+        }
+    }
+
+    pub fn tick_render(&mut self, renderer: &mut GlesRenderer, time_ms: u64) -> anyhow::Result<()> {
+        renderer.bind(self.gles_texture.clone())?;
+
+        let size = Size::from((i32::from(self.width), i32::from(self.height)));
+        let damage: Rectangle<i32, smithay::utils::Physical> = Rectangle::from_size(size);
+
+        let elements: Vec<WaylandSurfaceRenderElement<GlesRenderer>> = self
+            .displayed_windows
+            .iter()
+            .flat_map(|display_window| {
+                let wm = self.wm.borrow_mut();
+                if let Some(window) = wm.windows.get(&display_window.window_handle) {
+                    if !window.visible {
+                        return vec![];
+                    }
+                    render_elements_from_surface_tree(
+                        renderer,
+                        display_window.toplevel.wl_surface(),
+                        (window.pos_x, window.pos_y),
+                        1.0,
+                        1.0,
+                        Kind::Unspecified,
+                    )
+                } else {
+                    // Failed to fetch window
+                    vec![]
+                }
+            })
+            .collect();
+
+        let mut frame = renderer.render(size, Transform::Normal)?;
+
+        let clear_color = if self.displayed_windows.is_empty() {
+            Color32F::new(0.5, 0.5, 0.5, 0.5)
+        } else {
+            Color32F::new(0.0, 0.0, 0.0, 0.0)
+        };
+
+        frame.clear(clear_color, &[damage])?;
+
+        draw_render_elements(&mut frame, 1.0, &elements, &[damage])?;
+
+        let _sync_point = frame.finish()?;
+
+        for window in &self.displayed_windows {
+            send_frames_surface_tree(window.toplevel.wl_surface(), time_ms as u32);
+        }
+
+        if let egl_data::RenderData::Software(_) = &self.render_data {
+            // Read OpenGL texture into memory. Slow!
+            let pixel_data = renderer.with_context(|gl| unsafe {
+                gl.BindTexture(ffi::TEXTURE_2D, self.gles_texture.tex_id());
+
+                let len = self.width as usize * self.height as usize * 4;
+                let mut data: Box<[u8]> = Box::new_uninit_slice(len).assume_init();
+                gl.ReadPixels(
+                    0,
+                    0,
+                    i32::from(self.width),
+                    i32::from(self.height),
+                    ffi::RGBA,
+                    ffi::UNSIGNED_BYTE,
+                    data.as_mut_ptr().cast(),
+                );
+
+                let data: Arc<[u8]> = Arc::from(data);
+                data
+            })?;
+
+            self.render_data =
+                egl_data::RenderData::Software(Some(egl_data::RenderSoftwarePixelsData {
+                    data: pixel_data,
+                    width: self.width,
+                    height: self.height,
+                }));
+        }
+
+        self.rendered_frame_count += 1;
+
+        Ok(())
+    }
+
+    fn get_hovered_window(&self, cursor_x: u32, cursor_y: u32) -> Option<window::WindowHandle> {
+        let wm = self.wm.borrow();
+
+        for cell in self.displayed_windows.iter().rev() {
+            if let Some(window) = wm.windows.get(&cell.window_handle) {
+                if !window.visible {
+                    continue;
+                }
+
+                if (cursor_x as i32) >= window.pos_x
+                    && (cursor_x as i32) < window.pos_x + window.size_x as i32
+                    && (cursor_y as i32) >= window.pos_y
+                    && (cursor_y as i32) < window.pos_y + window.size_y as i32
+                {
+                    return Some(cell.window_handle);
+                }
+            }
+        }
+        None
+    }
+
+    pub const fn trigger_rerender(&mut self) {
+        self.wants_redraw = true;
+    }
+
+    pub fn set_visible(&mut self, visible: bool) {
+        log::info!("Display \"{}\" visible: {}", self.name.as_str(), visible);
+        if self.visible == visible {
+            return;
+        }
+        self.visible = visible;
+        if visible {
+            self.no_windows_since = None;
+            self.trigger_rerender();
+        }
+    }
+
+    pub fn set_layout(&mut self, layout: packet_server::WvrDisplayWindowLayout) {
+        log::info!("Display \"{}\" layout: {:?}", self.name.as_str(), layout);
+        if self.layout == layout {
+            return;
+        }
+        self.layout = layout;
+        self.trigger_rerender();
+        self.reposition_windows();
+    }
+
+    pub fn send_mouse_move(
+        &self,
+        config: &super::Config,
+        manager: &mut WayVRCompositor,
+        x: u32,
+        y: u32,
+    ) {
+        let current_ms = time::get_millis();
+        if self.last_pressed_time_ms + u64::from(config.click_freeze_time_ms) > current_ms {
+            return;
+        }
+
+        if let Some(window_handle) = self.get_hovered_window(x, y) {
+            let wm = self.wm.borrow();
+            if let Some(window) = wm.windows.get(&window_handle) {
+                let surf = window.toplevel.wl_surface().clone();
+                let point = Point::<f64, Logical>::from((
+                    f64::from(x as i32 - window.pos_x),
+                    f64::from(y as i32 - window.pos_y),
+                ));
+
+                manager.seat_pointer.motion(
+                    &mut manager.state,
+                    Some((surf, Point::from((0.0, 0.0)))),
+                    &input::pointer::MotionEvent {
+                        serial: manager.serial_counter.next_serial(),
+                        time: 0,
+                        location: point,
+                    },
+                );
+
+                manager.seat_pointer.frame(&mut manager.state);
+            }
+        }
+    }
+
+    const fn get_mouse_index_number(index: super::MouseIndex) -> u32 {
+        match index {
+            super::MouseIndex::Left => 0x110,   /* BTN_LEFT */
+            super::MouseIndex::Center => 0x112, /* BTN_MIDDLE */
+            super::MouseIndex::Right => 0x111,  /* BTN_RIGHT */
+        }
+    }
+
+    pub fn send_mouse_down(&mut self, manager: &mut WayVRCompositor, index: super::MouseIndex) {
+        // Change keyboard focus to pressed window
+        let loc = manager.seat_pointer.current_location();
+
+        self.last_pressed_time_ms = time::get_millis();
+
+        if let Some(window_handle) =
+            self.get_hovered_window(loc.x.max(0.0) as u32, loc.y.max(0.0) as u32)
+        {
+            let wm = self.wm.borrow();
+            if let Some(window) = wm.windows.get(&window_handle) {
+                let surf = window.toplevel.wl_surface().clone();
+
+                manager.seat_keyboard.set_focus(
+                    &mut manager.state,
+                    Some(surf),
+                    manager.serial_counter.next_serial(),
+                );
+            }
+        }
+
+        manager.seat_pointer.button(
+            &mut manager.state,
+            &input::pointer::ButtonEvent {
+                button: Self::get_mouse_index_number(index),
+                serial: manager.serial_counter.next_serial(),
+                time: 0,
+                state: smithay::backend::input::ButtonState::Pressed,
+            },
+        );
+
+        manager.seat_pointer.frame(&mut manager.state);
+    }
+
+    pub fn send_mouse_up(manager: &mut WayVRCompositor, index: super::MouseIndex) {
+        manager.seat_pointer.button(
+            &mut manager.state,
+            &input::pointer::ButtonEvent {
+                button: Self::get_mouse_index_number(index),
+                serial: manager.serial_counter.next_serial(),
+                time: 0,
+                state: smithay::backend::input::ButtonState::Released,
+            },
+        );
+
+        manager.seat_pointer.frame(&mut manager.state);
+    }
+
+    pub fn send_mouse_scroll(manager: &mut WayVRCompositor, delta_y: f32, delta_x: f32) {
+        manager.seat_pointer.axis(
+            &mut manager.state,
+            input::pointer::AxisFrame {
+                source: None,
+                relative_direction: (
+                    smithay::backend::input::AxisRelativeDirection::Identical,
+                    smithay::backend::input::AxisRelativeDirection::Identical,
+                ),
+                time: 0,
+                axis: (f64::from(delta_x), f64::from(-delta_y)),
+                v120: Some((0, (delta_y * -120.0) as i32)),
+                stop: (false, false),
+            },
+        );
+        manager.seat_pointer.frame(&mut manager.state);
+    }
+
+    fn configure_env(&self, cmd: &mut std::process::Command, auth_key: &str) {
+        cmd.env_remove("DISPLAY"); // Goodbye X11
+        cmd.env("WAYLAND_DISPLAY", self.wayland_env.display_num_string());
+        cmd.env("WAYVR_DISPLAY_AUTH", auth_key);
+    }
+
+    pub fn spawn_process(
+        &mut self,
+        exec_path: &str,
+        args: &[&str],
+        env: &[(&str, &str)],
+        working_dir: Option<&str>,
+    ) -> anyhow::Result<SpawnProcessResult> {
+        log::info!("Spawning subprocess with exec path \"{exec_path}\"");
+
+        let auth_key = generate_auth_key();
+
+        let mut cmd = std::process::Command::new(exec_path);
+        self.configure_env(&mut cmd, auth_key.as_str());
+        cmd.args(args);
+        if let Some(working_dir) = working_dir {
+            cmd.current_dir(working_dir);
+        }
+
+        for e in env {
+            cmd.env(e.0, e.1);
+        }
+
+        match cmd.spawn() {
+            Ok(child) => Ok(SpawnProcessResult { auth_key, child }),
+            Err(e) => {
+                anyhow::bail!(
+					"Failed to launch process with path \"{}\": {}. Make sure your exec path exists.",
+					exec_path,
+					e
+				);
+            }
+        }
+    }
+}
+
+gen_id!(DisplayVec, Display, DisplayCell, DisplayHandle);
+
+impl DisplayHandle {
+    pub const fn from_packet(handle: packet_server::WvrDisplayHandle) -> Self {
+        Self {
+            generation: handle.generation,
+            idx: handle.idx,
+        }
+    }
+
+    pub const fn as_packet(&self) -> packet_server::WvrDisplayHandle {
+        packet_server::WvrDisplayHandle {
+            idx: self.idx,
+            generation: self.generation,
+        }
+    }
+}

wlx-overlay-s/src/backend/wayvr/egl_data.rs

@@ -0,0 +1,315 @@
+use std::sync::Arc;
+
+use crate::backend::wayvr::egl_ex::{
+    PFNEGLGETPLATFORMDISPLAYEXTPROC, PFNEGLQUERYDMABUFFORMATSEXTPROC,
+    PFNEGLQUERYDMABUFMODIFIERSEXTPROC,
+};
+
+use super::egl_ex;
+use anyhow::anyhow;
+
+#[derive(Debug)]
+pub struct EGLData {
+    pub egl: khronos_egl::Instance<khronos_egl::Static>,
+    pub display: khronos_egl::Display,
+    pub config: khronos_egl::Config,
+    pub context: khronos_egl::Context,
+}
+
+#[macro_export]
+macro_rules! bind_egl_function {
+    ($func_type:ident, $func:expr) => {
+        std::mem::transmute_copy::<_, $func_type>($func).unwrap()
+    };
+}
+
+#[derive(Debug, Clone)]
+pub struct DMAbufModifierInfo {
+    pub modifiers: Vec<u64>,
+    pub fourcc: u32,
+}
+
+#[derive(Debug, Clone)]
+pub struct RenderDMAbufData {
+    pub fd: i32,
+    pub stride: i32,
+    pub offset: i32,
+    pub mod_info: DMAbufModifierInfo,
+}
+
+#[derive(Debug, Clone)]
+pub struct RenderSoftwarePixelsData {
+    pub data: Arc<[u8]>,
+    pub width: u16,
+    pub height: u16,
+}
+
+#[derive(Debug, Clone)]
+pub enum RenderData {
+    Dmabuf(RenderDMAbufData),
+    Software(Option<RenderSoftwarePixelsData>), // will be set if the next image data is available
+}
+
+fn load_egl_func(
+    egl: &khronos_egl::Instance<khronos_egl::Static>,
+    func_name: &str,
+) -> anyhow::Result<extern "system" fn()> {
+    let raw_fn = egl
+        .get_proc_address(func_name)
+        .ok_or_else(|| anyhow::anyhow!("Required EGL function {} not found", func_name))?;
+    Ok(raw_fn)
+}
+
+fn get_disp(
+    egl: &khronos_egl::Instance<khronos_egl::Static>,
+) -> anyhow::Result<khronos_egl::Display> {
+    unsafe {
+        if let Ok(func) = load_egl_func(egl, "eglGetPlatformDisplayEXT") {
+            let egl_get_platform_display_ext =
+                bind_egl_function!(PFNEGLGETPLATFORMDISPLAYEXTPROC, &func);
+
+            let display_ext = egl_get_platform_display_ext(
+                egl_ex::EGL_PLATFORM_WAYLAND_EXT, // platform
+                std::ptr::null_mut(),             // void *native_display
+                std::ptr::null_mut(),             // EGLint *attrib_list
+            );
+
+            if display_ext.is_null() {
+                log::warn!("eglGetPlatformDisplayEXT failed, using eglGetDisplay instead");
+            } else {
+                return Ok(khronos_egl::Display::from_ptr(display_ext));
+            }
+        }
+
+        egl
+            .get_display(khronos_egl::DEFAULT_DISPLAY)
+            .ok_or_else(|| anyhow!(
+                "Both eglGetPlatformDisplayEXT and eglGetDisplay failed. This shouldn't happen unless you don't have any display manager running. Cannot continue, check your EGL installation."
+            ))
+    }
+}
+
+impl EGLData {
+    pub fn new() -> anyhow::Result<Self> {
+        let egl = khronos_egl::Instance::new(khronos_egl::Static);
+        let display = get_disp(&egl)?;
+
+        let (major, minor) = egl.initialize(display)?;
+        log::debug!("EGL version: {major}.{minor}");
+
+        let attrib_list = [
+            khronos_egl::RED_SIZE,
+            8,
+            khronos_egl::GREEN_SIZE,
+            8,
+            khronos_egl::BLUE_SIZE,
+            8,
+            khronos_egl::SURFACE_TYPE,
+            khronos_egl::WINDOW_BIT,
+            khronos_egl::RENDERABLE_TYPE,
+            khronos_egl::OPENGL_BIT,
+            khronos_egl::NONE,
+        ];
+
+        let config = egl
+            .choose_first_config(display, &attrib_list)?
+            .ok_or_else(|| anyhow!("Failed to get EGL config"))?;
+
+        egl.bind_api(khronos_egl::OPENGL_ES_API)?;
+
+        log::debug!("eglCreateContext");
+
+        // Require OpenGL ES 3.0
+        let context_attrib_list = [
+            khronos_egl::CONTEXT_MAJOR_VERSION,
+            3,
+            khronos_egl::CONTEXT_MINOR_VERSION,
+            0,
+            khronos_egl::NONE,
+        ];
+
+        let context = egl.create_context(display, config, None, &context_attrib_list)?;
+
+        log::debug!("eglMakeCurrent");
+
+        egl.make_current(display, None, None, Some(context))?;
+
+        Ok(Self {
+            egl,
+            display,
+            config,
+            context,
+        })
+    }
+
+    fn query_dmabuf_mod_info(&self) -> anyhow::Result<DMAbufModifierInfo> {
+        let target_fourcc = 0x3432_4258; //XB24
+
+        unsafe {
+            let egl_query_dmabuf_formats_ext = bind_egl_function!(
+                PFNEGLQUERYDMABUFFORMATSEXTPROC,
+                &load_egl_func(&self.egl, "eglQueryDmaBufFormatsEXT")?
+            );
+
+            // Query format count
+            let mut num_formats: khronos_egl::Int = 0;
+            egl_query_dmabuf_formats_ext(
+                self.display.as_ptr(),
+                0,
+                std::ptr::null_mut(),
+                &mut num_formats,
+            );
+
+            // Retrieve formt list
+            let mut formats: Vec<i32> = vec![0; num_formats as usize];
+            egl_query_dmabuf_formats_ext(
+                self.display.as_ptr(),
+                num_formats,
+                formats.as_mut_ptr(),
+                &mut num_formats,
+            );
+
+            /*for (idx, format) in formats.iter().enumerate() {
+                let bytes = format.to_le_bytes();
+                log::trace!(
+                    "idx {}, format {}{}{}{} (hex {:#x})",
+                    idx,
+                    bytes[0] as char,
+                    bytes[1] as char,
+                    bytes[2] as char,
+                    bytes[3] as char,
+                    format
+                );
+            }*/
+
+            let egl_query_dmabuf_modifiers_ext = bind_egl_function!(
+                PFNEGLQUERYDMABUFMODIFIERSEXTPROC,
+                &load_egl_func(&self.egl, "eglQueryDmaBufModifiersEXT")?
+            );
+
+            let mut num_mods: khronos_egl::Int = 0;
+
+            // Query modifier count
+            egl_query_dmabuf_modifiers_ext(
+                self.display.as_ptr(),
+                target_fourcc,
+                0,
+                std::ptr::null_mut(),
+                std::ptr::null_mut(),
+                &mut num_mods,
+            );
+
+            if num_mods == 0 {
+                anyhow::bail!("eglQueryDmaBufModifiersEXT modifier count is zero");
+            }
+
+            let mut mods: Vec<u64> = vec![0; num_mods as usize];
+            egl_query_dmabuf_modifiers_ext(
+                self.display.as_ptr(),
+                target_fourcc,
+                num_mods,
+                mods.as_mut_ptr(),
+                std::ptr::null_mut(),
+                &mut num_mods,
+            );
+
+            if mods[0] == 0xFFFF_FFFF_FFFF_FFFF {
+                anyhow::bail!("modifier is -1")
+            }
+
+            log::trace!("Modifier list:");
+            for modifier in &mods {
+                log::trace!("{modifier:#x}");
+            }
+
+            // We should not change these modifier values. Passing all of them to the Vulkan dmabuf
+            // texture system causes significant graphical corruption due to invalid memory layout and
+            // tiling on this specific GPU model (very probably others also have the same issue).
+            // It is not guaranteed that this modifier will be present in other models.
+            // If not, the full list of modifiers will be passed. Further testing is required.
+            // For now, it looks like only NAVI32-based gpus have this problem.
+            let mod_whitelist: [u64; 2] = [
+                0x200_0000_2086_bf04, /* AMD RX 7800 XT, Navi32 */
+                0x200_0000_1866_bf04, /* AMD RX 7600 XT, Navi33 */
+            ];
+
+            for modifier in &mod_whitelist {
+                if mods.contains(modifier) {
+                    log::warn!("Using whitelisted dmabuf tiling modifier: {modifier:#x}");
+                    mods = vec![*modifier, 0x0 /* also important (???) */];
+                    break;
+                }
+            }
+
+            Ok(DMAbufModifierInfo {
+                modifiers: mods,
+                fourcc: target_fourcc as u32,
+            })
+        }
+    }
+
+    pub fn create_dmabuf_data(
+        &self,
+        egl_image: &khronos_egl::Image,
+    ) -> anyhow::Result<RenderDMAbufData> {
+        use egl_ex::PFNEGLEXPORTDMABUFIMAGEMESAPROC as FUNC;
+        unsafe {
+            let egl_export_dmabuf_image_mesa =
+                bind_egl_function!(FUNC, &load_egl_func(&self.egl, "eglExportDMABUFImageMESA")?);
+
+            let mut fds: [i32; 3] = [0; 3];
+            let mut strides: [i32; 3] = [0; 3];
+            let mut offsets: [i32; 3] = [0; 3];
+
+            let ret = egl_export_dmabuf_image_mesa(
+                self.display.as_ptr(),
+                egl_image.as_ptr(),
+                fds.as_mut_ptr(),
+                strides.as_mut_ptr(),
+                offsets.as_mut_ptr(),
+            );
+
+            if ret != khronos_egl::TRUE {
+                anyhow::bail!("eglExportDMABUFImageMESA failed with return code {ret}");
+            }
+
+            if fds[0] <= 0 {
+                anyhow::bail!("fd is <=0 (got {})", fds[0]);
+            }
+
+            // many planes in RGB data?
+            if fds[1] != 0 || strides[1] != 0 || offsets[1] != 0 {
+                anyhow::bail!("multi-planar data received, packed RGB expected");
+            }
+
+            if strides[0] < 0 {
+                anyhow::bail!("strides is < 0");
+            }
+
+            if offsets[0] < 0 {
+                anyhow::bail!("offsets is < 0");
+            }
+
+            let mod_info = self.query_dmabuf_mod_info()?;
+
+            Ok(RenderDMAbufData {
+                fd: fds[0],
+                stride: strides[0],
+                offset: offsets[0],
+                mod_info,
+            })
+        }
+    }
+
+    pub fn create_egl_image(&self, gl_tex_id: u32) -> anyhow::Result<khronos_egl::Image> {
+        unsafe {
+            Ok(self.egl.create_image(
+                self.display,
+                self.context,
+                khronos_egl::GL_TEXTURE_2D as std::ffi::c_uint,
+                khronos_egl::ClientBuffer::from_ptr(gl_tex_id as *mut std::ffi::c_void),
+                &[khronos_egl::ATTRIB_NONE],
+            )?)
+        }
+    }
+}

wlx-overlay-s/src/backend/wayvr/egl_ex.rs

@@ -0,0 +1,49 @@
+#![allow(clippy::all)]
+
+pub const EGL_PLATFORM_WAYLAND_EXT: khronos_egl::Enum = 0x31D8;
+
+// eglGetPlatformDisplayEXT
+// https://registry.khronos.org/EGL/extensions/EXT/EGL_EXT_platform_base.txt
+pub type PFNEGLGETPLATFORMDISPLAYEXTPROC = Option<
+    unsafe extern "C" fn(
+        platform: khronos_egl::Enum,
+        native_display: *mut std::ffi::c_void,
+        attrib_list: *mut khronos_egl::Enum,
+    ) -> khronos_egl::EGLDisplay,
+>;
+
+// eglExportDMABUFImageMESA
+// https://registry.khronos.org/EGL/extensions/MESA/EGL_MESA_image_dma_buf_export.txt
+pub type PFNEGLEXPORTDMABUFIMAGEMESAPROC = Option<
+    unsafe extern "C" fn(
+        dpy: khronos_egl::EGLDisplay,
+        image: khronos_egl::EGLImage,
+        fds: *mut i32,
+        strides: *mut khronos_egl::Int,
+        offsets: *mut khronos_egl::Int,
+    ) -> khronos_egl::Boolean,
+>;
+
+// eglQueryDmaBufModifiersEXT
+// https://registry.khronos.org/EGL/extensions/EXT/EGL_EXT_image_dma_buf_import_modifiers.txt
+pub type PFNEGLQUERYDMABUFMODIFIERSEXTPROC = Option<
+    unsafe extern "C" fn(
+        dpy: khronos_egl::EGLDisplay,
+        format: khronos_egl::Int,
+        max_modifiers: khronos_egl::Int,
+        modifiers: *mut u64,
+        external_only: *mut khronos_egl::Boolean,
+        num_modifiers: *mut khronos_egl::Int,
+    ) -> khronos_egl::Boolean,
+>;
+
+// eglQueryDmaBufFormatsEXT
+// https://registry.khronos.org/EGL/extensions/EXT/EGL_EXT_image_dma_buf_import_modifiers.txt
+pub type PFNEGLQUERYDMABUFFORMATSEXTPROC = Option<
+    unsafe extern "C" fn(
+        dpy: khronos_egl::EGLDisplay,
+        max_formats: khronos_egl::Int,
+        formats: *mut khronos_egl::Int,
+        num_formats: *mut khronos_egl::Int,
+    ) -> khronos_egl::Boolean,
+>;

wlx-overlay-s/src/backend/wayvr/event_queue.rs

@@ -0,0 +1,33 @@
+#![allow(dead_code)]
+
+use std::{cell::RefCell, collections::VecDeque, rc::Rc};
+
+#[derive(Debug)]
+struct Data<DataType> {
+    queue: VecDeque<DataType>,
+}
+
+#[derive(Debug, Clone)]
+pub struct SyncEventQueue<DataType> {
+    data: Rc<RefCell<Data<DataType>>>,
+}
+
+impl<DataType> SyncEventQueue<DataType> {
+    pub fn new() -> Self {
+        Self {
+            data: Rc::new(RefCell::new(Data {
+                queue: VecDeque::default(),
+            })),
+        }
+    }
+
+    pub fn send(&self, message: DataType) {
+        let mut data = self.data.borrow_mut();
+        data.queue.push_back(message);
+    }
+
+    pub fn read(&self) -> Option<DataType> {
+        let mut data = self.data.borrow_mut();
+        data.queue.pop_front()
+    }
+}

wlx-overlay-s/src/backend/wayvr/handle.rs

@@ -0,0 +1,176 @@
+#[macro_export]
+macro_rules! gen_id {
+    (
+		$container_name:ident,
+		$instance_name:ident,
+		$cell_name:ident,
+		$handle_name:ident) => {
+        //ThingCell
+        #[derive(Debug)]
+        pub struct $cell_name {
+            pub obj: $instance_name,
+            pub generation: u64,
+        }
+
+        //ThingVec
+        #[derive(Debug)]
+        pub struct $container_name {
+            // Vec<Option<ThingCell>>
+            pub vec: Vec<Option<$cell_name>>,
+
+            cur_generation: u64,
+        }
+
+        //ThingHandle
+        #[derive(Default, Debug, Clone, Copy, PartialEq, Hash, Eq)]
+        pub struct $handle_name {
+            idx: u32,
+            generation: u64,
+        }
+
+        #[allow(dead_code)]
+        impl $handle_name {
+            pub const fn reset(&mut self) {
+                self.generation = 0;
+            }
+
+            pub const fn is_set(&self) -> bool {
+                self.generation > 0
+            }
+
+            pub const fn id(&self) -> u32 {
+                self.idx
+            }
+
+            pub const fn new(idx: u32, generation: u64) -> Self {
+                Self { idx, generation }
+            }
+        }
+
+        //ThingVec
+        impl $container_name {
+            pub const fn new() -> Self {
+                Self {
+                    vec: Vec::new(),
+                    cur_generation: 0,
+                }
+            }
+
+            pub fn iter(&self) -> impl Iterator<Item = ($handle_name, &$instance_name)> {
+                self.vec.iter().enumerate().filter_map(|(idx, opt_cell)| {
+                    opt_cell.as_ref().map(|cell| {
+                        let handle = $container_name::get_handle(&cell, idx);
+                        (handle, &cell.obj)
+                    })
+                })
+            }
+
+            pub fn iter_mut(
+                &mut self,
+            ) -> impl Iterator<Item = ($handle_name, &mut $instance_name)> {
+                self.vec
+                    .iter_mut()
+                    .enumerate()
+                    .filter_map(|(idx, opt_cell)| {
+                        opt_cell.as_mut().map(|cell| {
+                            let handle = $container_name::get_handle(&cell, idx);
+                            (handle, &mut cell.obj)
+                        })
+                    })
+            }
+
+            pub const fn get_handle(cell: &$cell_name, idx: usize) -> $handle_name {
+                $handle_name {
+                    idx: idx as u32,
+                    generation: cell.generation,
+                }
+            }
+
+            fn find_unused_idx(&mut self) -> Option<u32> {
+                for (num, obj) in self.vec.iter().enumerate() {
+                    if obj.is_none() {
+                        return Some(num as u32);
+                    }
+                }
+                None
+            }
+
+            pub fn add(&mut self, obj: $instance_name) -> $handle_name {
+                self.cur_generation += 1;
+                let generation = self.cur_generation;
+
+                let unused_idx = self.find_unused_idx();
+
+                let idx = if let Some(idx) = unused_idx {
+                    idx
+                } else {
+                    self.vec.len() as u32
+                };
+
+                let handle = $handle_name { idx, generation };
+
+                let cell = $cell_name { obj, generation };
+
+                if let Some(idx) = unused_idx {
+                    self.vec[idx as usize] = Some(cell);
+                } else {
+                    self.vec.push(Some(cell))
+                }
+
+                handle
+            }
+
+            pub fn remove(&mut self, handle: &$handle_name) {
+                // Out of bounds, ignore
+                if handle.idx as usize >= self.vec.len() {
+                    return;
+                }
+
+                // Remove only if the generation matches
+                if let Some(cell) = &self.vec[handle.idx as usize] {
+                    if cell.generation == handle.generation {
+                        self.vec[handle.idx as usize] = None;
+                    }
+                }
+            }
+
+            pub fn get(&self, handle: &$handle_name) -> Option<&$instance_name> {
+                // Out of bounds, ignore
+                if handle.idx as usize >= self.vec.len() {
+                    return None;
+                }
+
+                if let Some(cell) = &self.vec[handle.idx as usize] {
+                    if cell.generation == handle.generation {
+                        return Some(&cell.obj);
+                    }
+                }
+
+                None
+            }
+
+            pub fn get_mut(&mut self, handle: &$handle_name) -> Option<&mut $instance_name> {
+                // Out of bounds, ignore
+                if handle.idx as usize >= self.vec.len() {
+                    return None;
+                }
+
+                if let Some(cell) = &mut self.vec[handle.idx as usize] {
+                    if cell.generation == handle.generation {
+                        return Some(&mut cell.obj);
+                    }
+                }
+
+                None
+            }
+        }
+    };
+}
+
+/* Example usage:
+gen_id!(ThingVec, ThingInstance, ThingCell, ThingHandle);
+
+struct ThingInstance {}
+
+impl ThingInstance {}
+ */

wlx-overlay-s/src/backend/wayvr/mod.rs

@@ -0,0 +1,740 @@
+pub mod client;
+mod comp;
+pub mod display;
+pub mod egl_data;
+mod egl_ex;
+pub mod event_queue;
+mod handle;
+mod process;
+pub mod server_ipc;
+mod smithay_wrapper;
+mod time;
+mod window;
+use comp::Application;
+use display::{Display, DisplayInitParams, DisplayVec};
+use event_queue::SyncEventQueue;
+use process::ProcessVec;
+use serde::Deserialize;
+use server_ipc::WayVRServer;
+use smallvec::SmallVec;
+use smithay::{
+    backend::{
+        egl,
+        renderer::{gles::GlesRenderer, ImportDma},
+    },
+    input::{keyboard::XkbConfig, SeatState},
+    output::{Mode, Output},
+    reexports::wayland_server::{self, backend::ClientId},
+    wayland::{
+        compositor,
+        dmabuf::{DmabufFeedbackBuilder, DmabufState},
+        selection::data_device::DataDeviceState,
+        shell::xdg::{ToplevelSurface, XdgShellState},
+        shm::ShmState,
+    },
+};
+use std::{
+    cell::RefCell,
+    collections::{HashMap, HashSet},
+    rc::Rc,
+    sync::Arc,
+};
+use time::get_millis;
+use wayvr_ipc::{packet_client, packet_server};
+
+use crate::{hid::MODS_TO_KEYS, state::AppState};
+
+const STR_INVALID_HANDLE_DISP: &str = "Invalid display handle";
+
+#[derive(Debug, Clone)]
+pub struct WaylandEnv {
+    pub display_num: u32,
+}
+
+impl WaylandEnv {
+    pub fn display_num_string(&self) -> String {
+        // e.g. "wayland-20"
+        format!("wayland-{}", self.display_num)
+    }
+}
+
+#[derive(Clone)]
+pub struct ProcessWayVREnv {
+    pub display_auth: Option<String>,
+    pub display_name: Option<String>, // Externally spawned process by a user script
+}
+
+#[derive(Clone)]
+pub struct ExternalProcessRequest {
+    pub env: ProcessWayVREnv,
+    pub client: wayland_server::Client,
+    pub pid: u32,
+}
+
+#[derive(Clone)]
+pub enum WayVRTask {
+    NewToplevel(ClientId, ToplevelSurface),
+    DropToplevel(ClientId, ToplevelSurface),
+    NewExternalProcess(ExternalProcessRequest),
+    ProcessTerminationRequest(process::ProcessHandle),
+}
+
+#[derive(Clone)]
+pub enum WayVRSignal {
+    DisplayVisibility(display::DisplayHandle, bool),
+    DisplayWindowLayout(
+        display::DisplayHandle,
+        packet_server::WvrDisplayWindowLayout,
+    ),
+    BroadcastStateChanged(packet_server::WvrStateChanged),
+    DropOverlay(super::overlay::OverlayID),
+    Haptics(super::input::Haptics),
+}
+
+pub enum BlitMethod {
+    Dmabuf,
+    Software,
+}
+
+impl BlitMethod {
+    pub fn from_string(str: &str) -> Option<Self> {
+        match str {
+            "dmabuf" => Some(Self::Dmabuf),
+            "software" => Some(Self::Software),
+            _ => None,
+        }
+    }
+}
+
+pub struct Config {
+    pub click_freeze_time_ms: u32,
+    pub keyboard_repeat_delay_ms: u32,
+    pub keyboard_repeat_rate: u32,
+    pub auto_hide_delay: Option<u32>, // if None, auto-hide is disabled
+    pub blit_method: BlitMethod,
+}
+
+pub struct WayVRState {
+    time_start: u64,
+    pub displays: display::DisplayVec,
+    pub manager: client::WayVRCompositor,
+    wm: Rc<RefCell<window::WindowManager>>,
+    egl_data: Rc<egl_data::EGLData>,
+    pub processes: process::ProcessVec,
+    pub config: Config,
+    dashboard_display: Option<display::DisplayHandle>,
+    pub tasks: SyncEventQueue<WayVRTask>,
+    pub signals: SyncEventQueue<WayVRSignal>,
+    ticks: u64,
+    cur_modifiers: u8,
+}
+
+pub struct WayVR {
+    pub state: WayVRState,
+    pub ipc_server: WayVRServer,
+}
+
+pub enum MouseIndex {
+    Left,
+    Center,
+    Right,
+}
+
+pub enum TickTask {
+    NewExternalProcess(ExternalProcessRequest), // Call WayVRCompositor::add_client after receiving this message
+    NewDisplay(
+        packet_client::WvrDisplayCreateParams,
+        Option<display::DisplayHandle>, /* existing handle? */
+    ),
+}
+
+impl WayVR {
+    #[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
+    pub fn new(config: Config) -> anyhow::Result<Self> {
+        log::info!("Initializing WayVR");
+        let display: wayland_server::Display<Application> = wayland_server::Display::new()?;
+        let dh = display.handle();
+        let compositor = compositor::CompositorState::new::<Application>(&dh);
+        let xdg_shell = XdgShellState::new::<Application>(&dh);
+        let mut seat_state = SeatState::new();
+        let shm = ShmState::new::<Application>(&dh, Vec::new());
+        let data_device = DataDeviceState::new::<Application>(&dh);
+        let mut seat = seat_state.new_wl_seat(&dh, "wayvr");
+
+        let dummy_width = 1280;
+        let dummy_height = 720;
+        let dummy_milli_hz = 60000; /* refresh rate in millihertz */
+
+        let output = Output::new(
+            String::from("wayvr_display"),
+            smithay::output::PhysicalProperties {
+                size: (dummy_width, dummy_height).into(),
+                subpixel: smithay::output::Subpixel::None,
+                make: String::from("Completely Legit"),
+                model: String::from("Virtual WayVR Display"),
+            },
+        );
+
+        let mode = Mode {
+            refresh: dummy_milli_hz,
+            size: (dummy_width, dummy_height).into(),
+        };
+
+        let _global = output.create_global::<Application>(&dh);
+        output.change_current_state(Some(mode), None, None, None);
+        output.set_preferred(mode);
+
+        let egl_data = egl_data::EGLData::new()?;
+
+        let smithay_display = smithay_wrapper::get_egl_display(&egl_data)?;
+        let smithay_context = smithay_wrapper::get_egl_context(&egl_data, &smithay_display)?;
+
+        let render_node = egl::EGLDevice::device_for_display(&smithay_display)
+            .and_then(|device| device.try_get_render_node());
+
+        let gles_renderer = unsafe { GlesRenderer::new(smithay_context)? };
+
+        let dmabuf_default_feedback = match render_node {
+            Ok(Some(node)) => {
+                let dmabuf_formats = gles_renderer.dmabuf_formats();
+                let dmabuf_default_feedback =
+                    DmabufFeedbackBuilder::new(node.dev_id(), dmabuf_formats)
+                        .build()
+                        .unwrap();
+                Some(dmabuf_default_feedback)
+            }
+            Ok(None) => {
+                log::warn!("dmabuf: Failed to query render node");
+                None
+            }
+            Err(err) => {
+                log::warn!("dmabuf: Failed to get egl device for display: {err}");
+                None
+            }
+        };
+
+        let dmabuf_state = dmabuf_default_feedback.map_or_else(
+            || {
+                let dmabuf_formats = gles_renderer.dmabuf_formats();
+                let mut dmabuf_state = DmabufState::new();
+                let dmabuf_global =
+                    dmabuf_state.create_global::<Application>(&display.handle(), dmabuf_formats);
+                (dmabuf_state, dmabuf_global, None)
+            },
+            |default_feedback| {
+                let mut dmabuf_state = DmabufState::new();
+                let dmabuf_global = dmabuf_state
+                    .create_global_with_default_feedback::<Application>(
+                        &display.handle(),
+                        &default_feedback,
+                    );
+                (dmabuf_state, dmabuf_global, Some(default_feedback))
+            },
+        );
+
+        let seat_keyboard = seat.add_keyboard(
+            XkbConfig::default(),
+            config.keyboard_repeat_delay_ms as i32,
+            config.keyboard_repeat_rate as i32,
+        )?;
+        let seat_pointer = seat.add_pointer();
+
+        let tasks = SyncEventQueue::new();
+
+        let state = Application {
+            compositor,
+            xdg_shell,
+            seat_state,
+            shm,
+            data_device,
+            wayvr_tasks: tasks.clone(),
+            redraw_requests: HashSet::new(),
+            dmabuf_state,
+            gles_renderer,
+        };
+
+        let time_start = get_millis();
+
+        let ipc_server = WayVRServer::new()?;
+
+        let state = WayVRState {
+            time_start,
+            manager: client::WayVRCompositor::new(state, display, seat_keyboard, seat_pointer)?,
+            displays: DisplayVec::new(),
+            processes: ProcessVec::new(),
+            egl_data: Rc::new(egl_data),
+            wm: Rc::new(RefCell::new(window::WindowManager::new())),
+            config,
+            dashboard_display: None,
+            ticks: 0,
+            tasks,
+            signals: SyncEventQueue::new(),
+            cur_modifiers: 0,
+        };
+
+        Ok(Self { state, ipc_server })
+    }
+
+    pub fn render_display(&mut self, display: display::DisplayHandle) -> anyhow::Result<bool> {
+        let display = self
+            .state
+            .displays
+            .get_mut(&display)
+            .ok_or_else(|| anyhow::anyhow!(STR_INVALID_HANDLE_DISP))?;
+
+        /* Buffer warm-up is required, always two first calls of this function are always rendered */
+        if !display.wants_redraw && display.rendered_frame_count >= 2 {
+            // Nothing changed, do not render
+            return Ok(false);
+        }
+
+        if !display.visible {
+            // Display is invisible, do not render
+            return Ok(false);
+        }
+
+        // millis since the start of wayvr
+        let time_ms = get_millis() - self.state.time_start;
+
+        display.tick_render(&mut self.state.manager.state.gles_renderer, time_ms)?;
+        display.wants_redraw = false;
+
+        Ok(true)
+    }
+
+    #[allow(clippy::too_many_lines, clippy::cognitive_complexity)]
+    pub fn tick_events(&mut self, app: &AppState) -> anyhow::Result<Vec<TickTask>> {
+        let mut tasks: Vec<TickTask> = Vec::new();
+
+        self.ipc_server.tick(&mut server_ipc::TickParams {
+            state: &mut self.state,
+            tasks: &mut tasks,
+            app,
+        });
+
+        // Check for redraw events
+        for (_, disp) in self.state.displays.iter_mut() {
+            for disp_window in &disp.displayed_windows {
+                if self
+                    .state
+                    .manager
+                    .state
+                    .check_redraw(disp_window.toplevel.wl_surface())
+                {
+                    disp.wants_redraw = true;
+                }
+            }
+        }
+
+        // Tick all child processes
+        let mut to_remove: SmallVec<[(process::ProcessHandle, display::DisplayHandle); 2]> =
+            SmallVec::new();
+
+        for (handle, process) in self.state.processes.iter_mut() {
+            if !process.is_running() {
+                to_remove.push((handle, process.display_handle()));
+            }
+        }
+
+        for (p_handle, disp_handle) in &to_remove {
+            self.state.processes.remove(p_handle);
+
+            if let Some(display) = self.state.displays.get_mut(disp_handle) {
+                display
+                    .tasks
+                    .send(display::DisplayTask::ProcessCleanup(*p_handle));
+                display.wants_redraw = true;
+            }
+        }
+
+        for (handle, display) in self.state.displays.iter_mut() {
+            display.tick(&self.state.config, &handle, &mut self.state.signals);
+        }
+
+        if !to_remove.is_empty() {
+            self.state.signals.send(WayVRSignal::BroadcastStateChanged(
+                packet_server::WvrStateChanged::ProcessRemoved,
+            ));
+        }
+
+        while let Some(task) = self.state.tasks.read() {
+            match task {
+                WayVRTask::NewExternalProcess(req) => {
+                    tasks.push(TickTask::NewExternalProcess(req));
+                }
+                WayVRTask::NewToplevel(client_id, toplevel) => {
+                    // Attach newly created toplevel surfaces to displays
+                    for client in &self.state.manager.clients {
+                        if client.client.id() != client_id {
+                            continue;
+                        }
+
+                        let Some(process_handle) =
+                            process::find_by_pid(&self.state.processes, client.pid)
+                        else {
+                            log::error!(
+                                    "WayVR window creation failed: Unexpected process ID {}. It wasn't registered before.",
+                                    client.pid
+                                );
+                            continue;
+                        };
+
+                        let window_handle = self
+                            .state
+                            .wm
+                            .borrow_mut()
+                            .create_window(client.display_handle, &toplevel);
+
+                        let Some(display) = self.state.displays.get_mut(&client.display_handle)
+                        else {
+                            // This shouldn't happen, scream if it does
+                            log::error!("Could not attach window handle into display");
+                            continue;
+                        };
+
+                        display.add_window(window_handle, process_handle, &toplevel);
+                        self.state.signals.send(WayVRSignal::BroadcastStateChanged(
+                            packet_server::WvrStateChanged::WindowCreated,
+                        ));
+                    }
+                }
+                WayVRTask::DropToplevel(client_id, toplevel) => {
+                    for client in &self.state.manager.clients {
+                        if client.client.id() != client_id {
+                            continue;
+                        }
+
+                        let mut wm = self.state.wm.borrow_mut();
+                        let Some(window_handle) = wm.find_window_handle(&toplevel) else {
+                            log::warn!("DropToplevel: Couldn't find matching window handle");
+                            continue;
+                        };
+
+                        let Some(display) = self.state.displays.get_mut(&client.display_handle)
+                        else {
+                            log::warn!("DropToplevel: Couldn't find matching display");
+                            continue;
+                        };
+
+                        display.remove_window(window_handle);
+                        wm.remove_window(window_handle);
+
+                        drop(wm);
+
+                        display.reposition_windows();
+                    }
+                }
+                WayVRTask::ProcessTerminationRequest(process_handle) => {
+                    if let Some(process) = self.state.processes.get_mut(&process_handle) {
+                        process.terminate();
+                    }
+                }
+            }
+        }
+
+        self.state
+            .manager
+            .tick_wayland(&mut self.state.displays, &mut self.state.processes)?;
+
+        if self.state.ticks % 200 == 0 {
+            self.state.manager.cleanup_clients();
+        }
+
+        self.state.ticks += 1;
+
+        Ok(tasks)
+    }
+
+    pub fn tick_finish(&mut self) -> anyhow::Result<()> {
+        self.state
+            .manager
+            .state
+            .gles_renderer
+            .with_context(|gl| unsafe {
+                gl.Flush();
+                gl.Finish();
+            })?;
+        Ok(())
+    }
+
+    #[allow(dead_code)]
+    pub fn get_primary_display(displays: &DisplayVec) -> Option<display::DisplayHandle> {
+        for (idx, cell) in displays.vec.iter().enumerate() {
+            if let Some(cell) = cell {
+                if cell.obj.primary {
+                    return Some(DisplayVec::get_handle(cell, idx));
+                }
+            }
+        }
+        None
+    }
+
+    pub fn get_display_by_name(
+        displays: &DisplayVec,
+        name: &str,
+    ) -> Option<display::DisplayHandle> {
+        for (idx, cell) in displays.vec.iter().enumerate() {
+            if let Some(cell) = cell {
+                if cell.obj.name == name {
+                    return Some(DisplayVec::get_handle(cell, idx));
+                }
+            }
+        }
+        None
+    }
+
+    pub fn terminate_process(&mut self, process_handle: process::ProcessHandle) {
+        self.state
+            .tasks
+            .send(WayVRTask::ProcessTerminationRequest(process_handle));
+    }
+}
+
+impl WayVRState {
+    pub fn send_mouse_move(&mut self, display: display::DisplayHandle, x: u32, y: u32) {
+        if let Some(display) = self.displays.get(&display) {
+            display.send_mouse_move(&self.config, &mut self.manager, x, y);
+        }
+    }
+
+    pub fn send_mouse_down(&mut self, display: display::DisplayHandle, index: MouseIndex) {
+        if let Some(display) = self.displays.get_mut(&display) {
+            display.send_mouse_down(&mut self.manager, index);
+        }
+    }
+
+    pub fn send_mouse_up(&mut self, index: MouseIndex) {
+        Display::send_mouse_up(&mut self.manager, index);
+    }
+
+    pub fn send_mouse_scroll(&mut self, delta_y: f32, delta_x: f32) {
+        Display::send_mouse_scroll(&mut self.manager, delta_y, delta_x);
+    }
+
+    pub fn send_key(&mut self, virtual_key: u32, down: bool) {
+        self.manager.send_key(virtual_key, down);
+    }
+
+    pub fn set_modifiers(&mut self, modifiers: u8) {
+        let changed = self.cur_modifiers ^ modifiers;
+        for i in 0..8 {
+            let m = 1 << i;
+            if changed & m != 0 {
+                if let Some(vk) = MODS_TO_KEYS.get(m).into_iter().flatten().next() {
+                    self.send_key(*vk as u32, modifiers & m != 0);
+                }
+            }
+        }
+        self.cur_modifiers = modifiers;
+    }
+
+    pub fn set_display_visible(&mut self, display: display::DisplayHandle, visible: bool) {
+        if let Some(display) = self.displays.get_mut(&display) {
+            display.set_visible(visible);
+        }
+    }
+
+    pub fn set_display_layout(
+        &mut self,
+        display: display::DisplayHandle,
+        layout: packet_server::WvrDisplayWindowLayout,
+    ) {
+        if let Some(display) = self.displays.get_mut(&display) {
+            display.set_layout(layout);
+        }
+    }
+
+    pub fn get_render_data(
+        &self,
+        display: display::DisplayHandle,
+    ) -> Option<&egl_data::RenderData> {
+        self.displays
+            .get(&display)
+            .map(|display| &display.render_data)
+    }
+
+    pub fn create_display(
+        &mut self,
+        width: u16,
+        height: u16,
+        name: &str,
+        primary: bool,
+    ) -> anyhow::Result<display::DisplayHandle> {
+        let display = display::Display::new(DisplayInitParams {
+            wm: self.wm.clone(),
+            egl_data: self.egl_data.clone(),
+            renderer: &mut self.manager.state.gles_renderer,
+            wayland_env: self.manager.wayland_env.clone(),
+            config: &self.config,
+            width,
+            height,
+            name,
+            primary,
+        })?;
+        let handle = self.displays.add(display);
+
+        self.signals.send(WayVRSignal::BroadcastStateChanged(
+            packet_server::WvrStateChanged::DisplayCreated,
+        ));
+
+        Ok(handle)
+    }
+
+    pub fn destroy_display(&mut self, handle: display::DisplayHandle) -> anyhow::Result<()> {
+        let Some(display) = self.displays.get(&handle) else {
+            anyhow::bail!("Display not found");
+        };
+
+        if let Some(overlay_id) = display.overlay_id {
+            self.signals.send(WayVRSignal::DropOverlay(overlay_id));
+        } else {
+            log::warn!("Destroying display without OverlayID set"); // This shouldn't happen, but log it anyways.
+        }
+
+        let mut process_names = Vec::<String>::new();
+
+        for (_, process) in self.processes.iter_mut() {
+            if process.display_handle() == handle {
+                process_names.push(process.get_name());
+            }
+        }
+
+        if !display.displayed_windows.is_empty() || !process_names.is_empty() {
+            anyhow::bail!(
+                "Display is not empty. Attached processes: {}",
+                process_names.join(", ")
+            );
+        }
+
+        self.manager.cleanup_clients();
+
+        for client in &self.manager.clients {
+            if client.display_handle == handle {
+                // This shouldn't happen, but make sure we are all set to destroy this display
+                anyhow::bail!("Wayland client still exists");
+            }
+        }
+
+        self.displays.remove(&handle);
+
+        self.signals.send(WayVRSignal::BroadcastStateChanged(
+            packet_server::WvrStateChanged::DisplayRemoved,
+        ));
+
+        Ok(())
+    }
+
+    pub fn get_or_create_dashboard_display(
+        &mut self,
+        width: u16,
+        height: u16,
+        name: &str,
+    ) -> anyhow::Result<(bool /* newly created? */, display::DisplayHandle)> {
+        if let Some(handle) = &self.dashboard_display {
+            // ensure it still exists
+            if self.displays.get(handle).is_some() {
+                return Ok((false, *handle));
+            }
+        }
+
+        let new_disp = self.create_display(width, height, name, false)?;
+        self.dashboard_display = Some(new_disp);
+
+        Ok((true, new_disp))
+    }
+
+    // Check if process with given arguments already exists
+    pub fn process_query(
+        &self,
+        display_handle: display::DisplayHandle,
+        exec_path: &str,
+        args: &[&str],
+        _env: &[(&str, &str)],
+    ) -> Option<process::ProcessHandle> {
+        for (idx, cell) in self.processes.vec.iter().enumerate() {
+            if let Some(cell) = &cell {
+                if let process::Process::Managed(process) = &cell.obj {
+                    if process.display_handle != display_handle
+                        || process.exec_path != exec_path
+                        || process.args != args
+                    {
+                        continue;
+                    }
+                    return Some(process::ProcessVec::get_handle(cell, idx));
+                }
+            }
+        }
+
+        None
+    }
+
+    pub fn add_external_process(
+        &mut self,
+        display_handle: display::DisplayHandle,
+        pid: u32,
+    ) -> process::ProcessHandle {
+        self.processes
+            .add(process::Process::External(process::ExternalProcess {
+                pid,
+                display_handle,
+            }))
+    }
+
+    pub fn spawn_process(
+        &mut self,
+        display_handle: display::DisplayHandle,
+        exec_path: &str,
+        args: &[&str],
+        env: &[(&str, &str)],
+        working_dir: Option<&str>,
+        userdata: HashMap<String, String>,
+    ) -> anyhow::Result<process::ProcessHandle> {
+        let display = self
+            .displays
+            .get_mut(&display_handle)
+            .ok_or_else(|| anyhow::anyhow!(STR_INVALID_HANDLE_DISP))?;
+
+        let res = display.spawn_process(exec_path, args, env, working_dir)?;
+
+        let handle = self
+            .processes
+            .add(process::Process::Managed(process::WayVRProcess {
+                auth_key: res.auth_key,
+                child: res.child,
+                display_handle,
+                exec_path: String::from(exec_path),
+                userdata,
+                args: args.iter().map(|x| String::from(*x)).collect(),
+                working_dir: working_dir.map(String::from),
+                env: env
+                    .iter()
+                    .map(|(a, b)| (String::from(*a), String::from(*b)))
+                    .collect(),
+            }));
+
+        self.signals.send(WayVRSignal::BroadcastStateChanged(
+            packet_server::WvrStateChanged::ProcessCreated,
+        ));
+
+        Ok(handle)
+    }
+}
+
+#[derive(Deserialize, Clone)]
+pub enum WayVRDisplayClickAction {
+    ToggleVisibility,
+    Reset,
+}
+
+#[derive(Deserialize, Clone)]
+pub enum WayVRAction {
+    AppClick {
+        catalog_name: Arc<str>,
+        app_name: Arc<str>,
+    },
+    DisplayClick {
+        display_name: Arc<str>,
+        action: WayVRDisplayClickAction,
+    },
+    ToggleDashboard,
+}

wlx-overlay-s/src/backend/wayvr/process.rs

@@ -0,0 +1,228 @@
+use std::{collections::HashMap, io::Read};
+
+use wayvr_ipc::packet_server;
+
+use crate::gen_id;
+
+use super::display;
+
+#[derive(Debug)]
+#[allow(dead_code)]
+pub struct WayVRProcess {
+    pub auth_key: String,
+    pub child: std::process::Child,
+    pub display_handle: display::DisplayHandle,
+
+    pub exec_path: String,
+    pub args: Vec<String>,
+    pub env: Vec<(String, String)>,
+    pub working_dir: Option<String>,
+
+    pub userdata: HashMap<String, String>,
+}
+
+#[derive(Debug)]
+pub struct ExternalProcess {
+    pub pid: u32,
+    pub display_handle: display::DisplayHandle,
+}
+
+#[derive(Debug)]
+pub enum Process {
+    Managed(WayVRProcess),     // Process spawned by WayVR
+    External(ExternalProcess), // External process not directly controlled by us
+}
+
+impl Process {
+    pub const fn display_handle(&self) -> display::DisplayHandle {
+        match self {
+            Self::Managed(p) => p.display_handle,
+            Self::External(p) => p.display_handle,
+        }
+    }
+
+    pub fn is_running(&mut self) -> bool {
+        match self {
+            Self::Managed(p) => p.is_running(),
+            Self::External(p) => p.is_running(),
+        }
+    }
+
+    pub fn terminate(&mut self) {
+        match self {
+            Self::Managed(p) => p.terminate(),
+            Self::External(p) => p.terminate(),
+        }
+    }
+
+    pub fn get_name(&self) -> String {
+        match self {
+            Self::Managed(p) => p.get_name().unwrap_or_else(|| String::from("unknown")),
+            Self::External(p) => p.get_name().unwrap_or_else(|| String::from("unknown")),
+        }
+    }
+
+    pub fn to_packet(&self, handle: ProcessHandle) -> packet_server::WvrProcess {
+        match self {
+            Self::Managed(p) => packet_server::WvrProcess {
+                name: p.get_name().unwrap_or_else(|| String::from("unknown")),
+                userdata: p.userdata.clone(),
+                display_handle: p.display_handle.as_packet(),
+                handle: handle.as_packet(),
+            },
+            Self::External(p) => packet_server::WvrProcess {
+                name: p.get_name().unwrap_or_else(|| String::from("unknown")),
+                userdata: HashMap::default(),
+                display_handle: p.display_handle.as_packet(),
+                handle: handle.as_packet(),
+            },
+        }
+    }
+}
+
+impl Drop for WayVRProcess {
+    fn drop(&mut self) {
+        log::info!(
+            "Sending SIGTERM (graceful exit) to process {}",
+            self.exec_path.as_str()
+        );
+        self.terminate();
+    }
+}
+
+fn get_process_env_value(pid: i32, key: &str) -> anyhow::Result<Option<String>> {
+    let path = format!("/proc/{pid}/environ");
+    let mut env_data = String::new();
+    std::fs::File::open(path)?.read_to_string(&mut env_data)?;
+    let lines: Vec<&str> = env_data.split('\0').filter(|s| !s.is_empty()).collect();
+
+    for line in lines {
+        if let Some(cell) = line.split_once('=') {
+            if cell.0 == key {
+                return Ok(Some(String::from(cell.1)));
+            }
+        }
+    }
+
+    Ok(None)
+}
+
+impl WayVRProcess {
+    fn is_running(&mut self) -> bool {
+        match self.child.try_wait() {
+            Ok(Some(_exit_status)) => false,
+            Ok(None) => true,
+            Err(e) => {
+                // this shouldn't happen
+                log::error!("Child::try_wait failed: {e}");
+                false
+            }
+        }
+    }
+
+    fn terminate(&mut self) {
+        unsafe {
+            // Gracefully stop process
+            libc::kill(self.child.id() as i32, libc::SIGTERM);
+        }
+    }
+
+    pub fn get_name(&self) -> Option<String> {
+        get_exec_name_from_pid(self.child.id())
+    }
+}
+
+fn get_exec_name_from_pid(pid: u32) -> Option<String> {
+    let path = format!("/proc/{pid}/exe");
+    match std::fs::read_link(&path) {
+        Ok(buf) => {
+            if let Some(process_name) = buf.file_name().and_then(|s| s.to_str()) {
+                return Some(String::from(process_name));
+            }
+            None
+        }
+        Err(_) => None,
+    }
+}
+
+impl ExternalProcess {
+    fn is_running(&self) -> bool {
+        if self.pid == 0 {
+            false
+        } else {
+            std::fs::metadata(format!("/proc/{}", self.pid)).is_ok()
+        }
+    }
+
+    fn terminate(&mut self) {
+        if self.pid != 0 {
+            unsafe {
+                // send SIGINT (^C)
+                libc::kill(self.pid as i32, libc::SIGINT);
+            }
+        }
+        self.pid = 0;
+    }
+
+    pub fn get_name(&self) -> Option<String> {
+        get_exec_name_from_pid(self.pid)
+    }
+}
+
+gen_id!(ProcessVec, Process, ProcessCell, ProcessHandle);
+
+pub fn find_by_pid(processes: &ProcessVec, pid: u32) -> Option<ProcessHandle> {
+    log::debug!("Finding process with PID {pid}");
+
+    for (idx, cell) in processes.vec.iter().enumerate() {
+        let Some(cell) = cell else {
+            continue;
+        };
+        match &cell.obj {
+            Process::Managed(wayvr_process) => {
+                if wayvr_process.child.id() == pid {
+                    return Some(ProcessVec::get_handle(cell, idx));
+                }
+            }
+            Process::External(external_process) => {
+                if external_process.pid == pid {
+                    return Some(ProcessVec::get_handle(cell, idx));
+                }
+            }
+        }
+    }
+
+    log::debug!("Finding by PID failed, trying WAYVR_DISPLAY_AUTH...");
+
+    if let Ok(Some(value)) = get_process_env_value(pid as i32, "WAYVR_DISPLAY_AUTH") {
+        for (idx, cell) in processes.vec.iter().enumerate() {
+            let Some(cell) = cell else {
+                continue;
+            };
+            if let Process::Managed(wayvr_process) = &cell.obj {
+                if wayvr_process.auth_key == value {
+                    return Some(ProcessVec::get_handle(cell, idx));
+                }
+            }
+        }
+    }
+
+    log::debug!("Process find with PID {pid} failed");
+    None
+}
+
+impl ProcessHandle {
+    pub const fn from_packet(handle: packet_server::WvrProcessHandle) -> Self {
+        Self {
+            generation: handle.generation,
+            idx: handle.idx,
+        }
+    }
+
+    pub const fn as_packet(&self) -> packet_server::WvrProcessHandle {
+        packet_server::WvrProcessHandle {
+            idx: self.idx,
+            generation: self.generation,
+        }
+    }
+}

wlx-overlay-s/src/backend/wayvr/server_ipc.rs

@@ -0,0 +1,663 @@
+use crate::state::AppState;
+
+use super::{display, process, window, TickTask, WayVRSignal};
+use bytes::BufMut;
+use glam::Vec3A;
+use interprocess::local_socket::{self, traits::Listener, ToNsName};
+use smallvec::SmallVec;
+use std::io::{Read, Write};
+use wayvr_ipc::{
+    ipc::{self},
+    packet_client::{self, PacketClient},
+    packet_server::{self, PacketServer, WlxInputStatePointer},
+};
+
+pub struct AuthInfo {
+    pub client_name: String,
+    pub protocol_version: u32, // client protocol version
+}
+
+pub struct Connection {
+    alive: bool,
+    conn: local_socket::Stream,
+    next_packet: Option<u32>,
+    auth: Option<AuthInfo>,
+}
+
+pub fn send_packet(conn: &mut local_socket::Stream, data: &[u8]) -> anyhow::Result<()> {
+    let mut bytes = bytes::BytesMut::new();
+
+    // packet size
+    bytes.put_u32(data.len() as u32);
+
+    // packet data
+    bytes.put_slice(data);
+
+    conn.write_all(&bytes)?;
+
+    Ok(())
+}
+
+fn read_check(expected_size: u32, res: std::io::Result<usize>) -> bool {
+    match res {
+        Ok(count) => {
+            if count == 0 {
+                return false;
+            }
+            if count as u32 == expected_size {
+                true // read succeeded
+            } else {
+                log::error!("count {count} is not {expected_size}");
+                false
+            }
+        }
+        Err(_e) => {
+            //log::error!("failed to get packet size: {}", e);
+            false
+        }
+    }
+}
+
+type Payload = SmallVec<[u8; 64]>;
+
+fn read_payload(conn: &mut local_socket::Stream, size: u32) -> Option<Payload> {
+    let mut payload = Payload::new();
+    payload.resize(size as usize, 0);
+    if read_check(size, conn.read(&mut payload)) {
+        Some(payload)
+    } else {
+        None
+    }
+}
+
+pub struct TickParams<'a> {
+    pub state: &'a mut super::WayVRState,
+    pub tasks: &'a mut Vec<TickTask>,
+    pub app: &'a AppState,
+}
+
+pub fn gen_args_vec(input: &str) -> Vec<&str> {
+    input.split_whitespace().collect()
+}
+
+pub fn gen_env_vec(input: &[String]) -> Vec<(&str, &str)> {
+    let res = input
+        .iter()
+        .filter_map(|e| e.as_str().split_once('='))
+        .collect();
+    res
+}
+
+impl Connection {
+    const fn new(conn: local_socket::Stream) -> Self {
+        Self {
+            conn,
+            alive: true,
+            auth: None,
+            next_packet: None,
+        }
+    }
+
+    fn kill(&mut self, reason: &str) {
+        let _dont_care = send_packet(
+            &mut self.conn,
+            &ipc::data_encode(&PacketServer::Disconnect(packet_server::Disconnect {
+                reason: String::from(reason),
+            })),
+        );
+        self.alive = false;
+    }
+
+    fn process_handshake(&mut self, handshake: &packet_client::Handshake) -> anyhow::Result<()> {
+        if self.auth.is_some() {
+            anyhow::bail!("You were already authenticated");
+        }
+
+        if handshake.protocol_version != ipc::PROTOCOL_VERSION {
+            anyhow::bail!(
+                "Unsupported protocol version {}",
+                handshake.protocol_version
+            );
+        }
+
+        if handshake.magic != ipc::CONNECTION_MAGIC {
+            anyhow::bail!("Invalid magic");
+        }
+
+        match handshake.client_name.len() {
+            0 => anyhow::bail!("Client name is empty"),
+            1..32 => {}
+            _ => anyhow::bail!("Client name is too long"),
+        }
+
+        log::info!("IPC: Client \"{}\" connected.", handshake.client_name);
+
+        self.auth = Some(AuthInfo {
+            client_name: handshake.client_name.clone(),
+            protocol_version: handshake.protocol_version,
+        });
+
+        // Send auth response
+        send_packet(
+            &mut self.conn,
+            &ipc::data_encode(&PacketServer::HandshakeSuccess(
+                packet_server::HandshakeSuccess {
+                    runtime: String::from("wlx-overlay-s"),
+                },
+            )),
+        )?;
+
+        Ok(())
+    }
+
+    fn handle_wvr_display_list(
+        &mut self,
+        params: &TickParams,
+        serial: ipc::Serial,
+    ) -> anyhow::Result<()> {
+        let list: Vec<packet_server::WvrDisplay> = params
+            .state
+            .displays
+            .vec
+            .iter()
+            .enumerate()
+            .filter_map(|(idx, opt_cell)| {
+                let Some(cell) = opt_cell else {
+                    return None;
+                };
+                let display = &cell.obj;
+                Some(display.as_packet(display::DisplayHandle::new(idx as u32, cell.generation)))
+            })
+            .collect();
+
+        send_packet(
+            &mut self.conn,
+            &ipc::data_encode(&PacketServer::WvrDisplayListResponse(
+                serial,
+                packet_server::WvrDisplayList { list },
+            )),
+        )?;
+
+        Ok(())
+    }
+
+    fn handle_wlx_input_state(
+        &mut self,
+        params: &TickParams,
+        serial: ipc::Serial,
+    ) -> anyhow::Result<()> {
+        let input_state = &params.app.input_state;
+
+        let to_arr = |vec: &Vec3A| -> [f32; 3] { [vec.x, vec.y, vec.z] };
+
+        send_packet(
+            &mut self.conn,
+            &ipc::data_encode(&PacketServer::WlxInputStateResponse(
+                serial,
+                packet_server::WlxInputState {
+                    hmd_pos: to_arr(&input_state.hmd.translation),
+                    left: WlxInputStatePointer {
+                        pos: to_arr(&input_state.pointers[0].raw_pose.translation),
+                    },
+                    right: WlxInputStatePointer {
+                        pos: to_arr(&input_state.pointers[0].raw_pose.translation),
+                    },
+                },
+            )),
+        )?;
+
+        Ok(())
+    }
+
+    fn handle_wvr_display_create(
+        &mut self,
+        params: &mut TickParams,
+        serial: ipc::Serial,
+        packet_params: packet_client::WvrDisplayCreateParams,
+    ) -> anyhow::Result<()> {
+        let display_handle = params.state.create_display(
+            packet_params.width,
+            packet_params.height,
+            &packet_params.name,
+            false,
+        )?;
+
+        params
+            .tasks
+            .push(TickTask::NewDisplay(packet_params, Some(display_handle)));
+
+        send_packet(
+            &mut self.conn,
+            &ipc::data_encode(&PacketServer::WvrDisplayCreateResponse(
+                serial,
+                display_handle.as_packet(),
+            )),
+        )?;
+        Ok(())
+    }
+
+    fn handle_wvr_display_remove(
+        &mut self,
+        params: &mut TickParams,
+        serial: ipc::Serial,
+        handle: packet_server::WvrDisplayHandle,
+    ) -> anyhow::Result<()> {
+        let res = params
+            .state
+            .destroy_display(display::DisplayHandle::from_packet(handle))
+            .map_err(|e| format!("{e:?}"));
+
+        send_packet(
+            &mut self.conn,
+            &ipc::data_encode(&PacketServer::WvrDisplayRemoveResponse(serial, res)),
+        )?;
+        Ok(())
+    }
+
+    fn handle_wvr_display_set_visible(
+        params: &mut TickParams,
+        handle: packet_server::WvrDisplayHandle,
+        visible: bool,
+    ) {
+        params.state.signals.send(WayVRSignal::DisplayVisibility(
+            display::DisplayHandle::from_packet(handle),
+            visible,
+        ));
+    }
+
+    fn handle_wvr_display_set_window_layout(
+        params: &mut TickParams,
+        handle: packet_server::WvrDisplayHandle,
+        layout: packet_server::WvrDisplayWindowLayout,
+    ) {
+        params.state.signals.send(WayVRSignal::DisplayWindowLayout(
+            display::DisplayHandle::from_packet(handle),
+            layout,
+        ));
+    }
+
+    fn handle_wvr_display_window_list(
+        &mut self,
+        params: &mut TickParams,
+        serial: ipc::Serial,
+        display_handle: packet_server::WvrDisplayHandle,
+    ) -> anyhow::Result<()> {
+        let mut send = |list: Option<packet_server::WvrWindowList>| -> anyhow::Result<()> {
+            send_packet(
+                &mut self.conn,
+                &ipc::data_encode(&PacketServer::WvrDisplayWindowListResponse(serial, list)),
+            )
+        };
+
+        let Some(display) = params
+            .state
+            .displays
+            .get(&display::DisplayHandle::from_packet(display_handle.clone()))
+        else {
+            return send(None);
+        };
+
+        send(Some(packet_server::WvrWindowList {
+            list: display
+                .displayed_windows
+                .iter()
+                .filter_map(|disp_win| {
+                    params
+                        .state
+                        .wm
+                        .borrow_mut()
+                        .windows
+                        .get(&disp_win.window_handle)
+                        .map(|win| packet_server::WvrWindow {
+                            handle: window::WindowHandle::as_packet(&disp_win.window_handle),
+                            process_handle: process::ProcessHandle::as_packet(
+                                &disp_win.process_handle,
+                            ),
+                            pos_x: win.pos_x,
+                            pos_y: win.pos_y,
+                            size_x: win.size_x,
+                            size_y: win.size_y,
+                            visible: win.visible,
+                            display_handle: display_handle.clone(),
+                        })
+                })
+                .collect::<Vec<_>>(),
+        }))
+    }
+
+    fn handle_wvr_window_set_visible(
+        params: &mut TickParams,
+        handle: packet_server::WvrWindowHandle,
+        visible: bool,
+    ) {
+        let mut to_resize = None;
+
+        if let Some(window) = params
+            .state
+            .wm
+            .borrow_mut()
+            .windows
+            .get_mut(&window::WindowHandle::from_packet(handle))
+        {
+            window.visible = visible;
+            to_resize = Some(window.display_handle);
+        }
+
+        if let Some(to_resize) = to_resize {
+            if let Some(display) = params.state.displays.get_mut(&to_resize) {
+                display.reposition_windows();
+                display.trigger_rerender();
+            }
+        }
+    }
+
+    fn handle_wvr_process_launch(
+        &mut self,
+        params: &mut TickParams,
+        serial: ipc::Serial,
+        packet_params: packet_client::WvrProcessLaunchParams,
+    ) -> anyhow::Result<()> {
+        let args_vec = gen_args_vec(&packet_params.args);
+        let env_vec = gen_env_vec(&packet_params.env);
+
+        let res = params.state.spawn_process(
+            super::display::DisplayHandle::from_packet(packet_params.target_display),
+            &packet_params.exec,
+            &args_vec,
+            &env_vec,
+            None,
+            packet_params.userdata,
+        );
+
+        let res = res.map(|r| r.as_packet()).map_err(|e| e.to_string());
+
+        send_packet(
+            &mut self.conn,
+            &ipc::data_encode(&PacketServer::WvrProcessLaunchResponse(serial, res)),
+        )?;
+
+        Ok(())
+    }
+
+    fn handle_wvr_display_get(
+        &mut self,
+        params: &TickParams,
+        serial: ipc::Serial,
+        display_handle: packet_server::WvrDisplayHandle,
+    ) -> anyhow::Result<()> {
+        let native_handle = &display::DisplayHandle::from_packet(display_handle);
+        let disp = params
+            .state
+            .displays
+            .get(native_handle)
+            .map(|disp| disp.as_packet(*native_handle));
+
+        send_packet(
+            &mut self.conn,
+            &ipc::data_encode(&PacketServer::WvrDisplayGetResponse(serial, disp)),
+        )?;
+
+        Ok(())
+    }
+
+    fn handle_wvr_process_list(
+        &mut self,
+        params: &TickParams,
+        serial: ipc::Serial,
+    ) -> anyhow::Result<()> {
+        let list: Vec<packet_server::WvrProcess> = params
+            .state
+            .processes
+            .vec
+            .iter()
+            .enumerate()
+            .filter_map(|(idx, opt_cell)| {
+                let Some(cell) = opt_cell else {
+                    return None;
+                };
+                let process = &cell.obj;
+                Some(process.to_packet(process::ProcessHandle::new(idx as u32, cell.generation)))
+            })
+            .collect();
+
+        send_packet(
+            &mut self.conn,
+            &ipc::data_encode(&PacketServer::WvrProcessListResponse(
+                serial,
+                packet_server::WvrProcessList { list },
+            )),
+        )?;
+
+        Ok(())
+    }
+
+    // This request doesn't return anything to the client
+    fn handle_wvr_process_terminate(
+        params: &mut TickParams,
+        process_handle: packet_server::WvrProcessHandle,
+    ) {
+        let native_handle = &process::ProcessHandle::from_packet(process_handle);
+        let process = params.state.processes.get_mut(native_handle);
+
+        let Some(process) = process else {
+            return;
+        };
+
+        process.terminate();
+    }
+
+    fn handle_wvr_process_get(
+        &mut self,
+        params: &TickParams,
+        serial: ipc::Serial,
+        process_handle: packet_server::WvrProcessHandle,
+    ) -> anyhow::Result<()> {
+        let native_handle = &process::ProcessHandle::from_packet(process_handle);
+        let process = params
+            .state
+            .processes
+            .get(native_handle)
+            .map(|process| process.to_packet(*native_handle));
+
+        send_packet(
+            &mut self.conn,
+            &ipc::data_encode(&PacketServer::WvrProcessGetResponse(serial, process)),
+        )?;
+
+        Ok(())
+    }
+
+    fn handle_wlx_haptics(
+        params: &mut TickParams,
+        haptics_params: packet_client::WlxHapticsParams,
+    ) {
+        params.state.signals.send(super::WayVRSignal::Haptics(
+            crate::backend::input::Haptics {
+                duration: haptics_params.duration,
+                frequency: haptics_params.frequency,
+                intensity: haptics_params.intensity,
+            },
+        ));
+    }
+
+    fn process_payload(&mut self, params: &mut TickParams, payload: Payload) -> anyhow::Result<()> {
+        let packet: PacketClient = ipc::data_decode(&payload)?;
+
+        if let PacketClient::Handshake(handshake) = &packet {
+            self.process_handshake(handshake)?;
+            return Ok(());
+        }
+
+        match packet {
+            PacketClient::Handshake(_) => unreachable!(), // handled previously
+            PacketClient::WlxInputState(serial) => {
+                self.handle_wlx_input_state(params, serial)?;
+            }
+            PacketClient::WvrDisplayList(serial) => {
+                self.handle_wvr_display_list(params, serial)?;
+            }
+            PacketClient::WvrDisplayGet(serial, display_handle) => {
+                self.handle_wvr_display_get(params, serial, display_handle)?;
+            }
+            PacketClient::WvrDisplayRemove(serial, display_handle) => {
+                self.handle_wvr_display_remove(params, serial, display_handle)?;
+            }
+            PacketClient::WvrDisplaySetVisible(display_handle, visible) => {
+                Self::handle_wvr_display_set_visible(params, display_handle, visible);
+            }
+            PacketClient::WvrDisplaySetWindowLayout(display_handle, layout) => {
+                Self::handle_wvr_display_set_window_layout(params, display_handle, layout);
+            }
+            PacketClient::WvrDisplayWindowList(serial, display_handle) => {
+                self.handle_wvr_display_window_list(params, serial, display_handle)?;
+            }
+            PacketClient::WvrWindowSetVisible(window_handle, visible) => {
+                Self::handle_wvr_window_set_visible(params, window_handle, visible);
+            }
+            PacketClient::WvrProcessGet(serial, process_handle) => {
+                self.handle_wvr_process_get(params, serial, process_handle)?;
+            }
+            PacketClient::WvrProcessList(serial) => {
+                self.handle_wvr_process_list(params, serial)?;
+            }
+            PacketClient::WvrProcessLaunch(serial, packet_params) => {
+                self.handle_wvr_process_launch(params, serial, packet_params)?;
+            }
+            PacketClient::WvrDisplayCreate(serial, packet_params) => {
+                self.handle_wvr_display_create(params, serial, packet_params)?;
+            }
+            PacketClient::WvrProcessTerminate(process_handle) => {
+                Self::handle_wvr_process_terminate(params, process_handle);
+            }
+            PacketClient::WlxHaptics(haptics_params) => {
+                Self::handle_wlx_haptics(params, haptics_params);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn process_check_payload(&mut self, params: &mut TickParams, payload: Payload) -> bool {
+        log::debug!("payload size {}", payload.len());
+
+        if let Err(e) = self.process_payload(params, payload) {
+            log::error!("Invalid payload from the client, closing connection: {e}");
+            // send also error message directly to the client before disconnecting
+            self.kill(format!("{e}").as_str());
+            false
+        } else {
+            true
+        }
+    }
+
+    fn read_packet(&mut self, params: &mut TickParams) -> bool {
+        if let Some(payload_size) = self.next_packet {
+            let Some(payload) = read_payload(&mut self.conn, payload_size) else {
+                // still failed to read payload, try in next tick
+                return false;
+            };
+
+            if !self.process_check_payload(params, payload) {
+                return false;
+            }
+
+            self.next_packet = None;
+        }
+
+        let mut buf_packet_header: [u8; 4] = [0; 4];
+        if !read_check(4, self.conn.read(&mut buf_packet_header)) {
+            return false;
+        }
+
+        let payload_size = u32::from_be_bytes(buf_packet_header[0..4].try_into().unwrap()); // 0-3 bytes (u32 size)
+
+        let size_limit: u32 = 128 * 1024;
+
+        if payload_size > size_limit {
+            // over 128 KiB?
+            log::error!(
+                "Client sent a packet header with the size over {size_limit} bytes, closing connection."
+            );
+            self.kill("Too big packet received (over 128 KiB)");
+            return false;
+        }
+
+        let Some(payload) = read_payload(&mut self.conn, payload_size) else {
+            // failed to read payload, try in next tick
+            self.next_packet = Some(payload_size);
+            return false;
+        };
+
+        if !self.process_check_payload(params, payload) {
+            return false;
+        }
+
+        true
+    }
+
+    fn tick(&mut self, params: &mut TickParams) {
+        while self.read_packet(params) {}
+    }
+}
+
+impl Drop for Connection {
+    fn drop(&mut self) {
+        log::info!("Connection closed");
+    }
+}
+
+pub struct WayVRServer {
+    listener: local_socket::Listener,
+    connections: Vec<Connection>,
+}
+
+impl WayVRServer {
+    pub fn new() -> anyhow::Result<Self> {
+        let printname = "/tmp/wayvr_ipc.sock";
+        let name = printname.to_ns_name::<local_socket::GenericNamespaced>()?;
+        let opts = local_socket::ListenerOptions::new()
+            .name(name)
+            .nonblocking(local_socket::ListenerNonblockingMode::Both);
+        let listener = match opts.create_sync() {
+            Ok(listener) => listener,
+            Err(e) => anyhow::bail!("Failed to start WayVRServer IPC listener. Reason: {}", e),
+        };
+
+        log::info!("WayVRServer IPC running at {printname}");
+
+        Ok(Self {
+            listener,
+            connections: Vec::new(),
+        })
+    }
+
+    fn accept_connections(&mut self) {
+        let Ok(conn) = self.listener.accept() else {
+            return; // No new connection or other error
+        };
+
+        self.connections.push(Connection::new(conn));
+    }
+
+    fn tick_connections(&mut self, params: &mut TickParams) {
+        for c in &mut self.connections {
+            c.tick(params);
+        }
+
+        // remove killed connections
+        self.connections.retain(|c| c.alive);
+    }
+
+    pub fn tick(&mut self, params: &mut TickParams) {
+        self.accept_connections();
+        self.tick_connections(params);
+    }
+
+    pub fn broadcast(&mut self, packet: packet_server::PacketServer) {
+        for connection in &mut self.connections {
+            if let Err(e) = send_packet(&mut connection.conn, &ipc::data_encode(&packet)) {
+                log::error!("failed to broadcast packet: {e:?}");
+            }
+        }
+    }
+}

wlx-overlay-s/src/backend/wayvr/smithay_wrapper.rs

@@ -0,0 +1,54 @@
+use super::egl_data;
+use smithay::backend::{egl as smithay_egl, renderer::gles::ffi};
+
+pub fn get_egl_display(data: &egl_data::EGLData) -> anyhow::Result<smithay_egl::EGLDisplay> {
+    Ok(unsafe { smithay_egl::EGLDisplay::from_raw(data.display.as_ptr(), data.config.as_ptr())? })
+}
+
+pub fn get_egl_context(
+    data: &egl_data::EGLData,
+    display: &smithay_egl::EGLDisplay,
+) -> anyhow::Result<smithay_egl::EGLContext> {
+    let display_ptr = display.get_display_handle().handle;
+    debug_assert!(std::ptr::eq(display_ptr, data.display.as_ptr()));
+    let config_ptr = data.config.as_ptr();
+    let context_ptr = data.context.as_ptr();
+    Ok(unsafe { smithay_egl::EGLContext::from_raw(display_ptr, config_ptr, context_ptr)? })
+}
+
+pub fn create_framebuffer_texture(
+    gl: &ffi::Gles2,
+    width: u32,
+    height: u32,
+    tex_format: u32,
+    internal_format: u32,
+) -> u32 {
+    unsafe {
+        let mut tex = 0;
+        gl.GenTextures(1, &mut tex);
+        gl.BindTexture(ffi::TEXTURE_2D, tex);
+        gl.TexParameteri(
+            ffi::TEXTURE_2D,
+            ffi::TEXTURE_MIN_FILTER,
+            ffi::NEAREST as i32,
+        );
+        gl.TexParameteri(
+            ffi::TEXTURE_2D,
+            ffi::TEXTURE_MAG_FILTER,
+            ffi::NEAREST as i32,
+        );
+        gl.TexImage2D(
+            ffi::TEXTURE_2D,
+            0,
+            internal_format as i32,
+            width as i32,
+            height as i32,
+            0,
+            tex_format,
+            ffi::UNSIGNED_BYTE,
+            std::ptr::null(),
+        );
+        gl.BindTexture(ffi::TEXTURE_2D, 0);
+        tex
+    }
+}

wlx-overlay-s/src/backend/wayvr/time.rs

@@ -0,0 +1,9 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+// Returns milliseconds since unix epoch
+pub fn get_millis() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_millis() as u64
+}

wlx-overlay-s/src/backend/wayvr/window.rs

@@ -0,0 +1,102 @@
+use smithay::wayland::shell::xdg::ToplevelSurface;
+use wayvr_ipc::packet_server;
+
+use crate::gen_id;
+
+use super::display;
+
+#[derive(Debug)]
+pub struct Window {
+    pub pos_x: i32,
+    pub pos_y: i32,
+    pub size_x: u32,
+    pub size_y: u32,
+    pub visible: bool,
+    pub toplevel: ToplevelSurface,
+    pub display_handle: display::DisplayHandle,
+}
+
+impl Window {
+    pub fn new(display_handle: display::DisplayHandle, toplevel: &ToplevelSurface) -> Self {
+        Self {
+            pos_x: 0,
+            pos_y: 0,
+            size_x: 0,
+            size_y: 0,
+            visible: true,
+            toplevel: toplevel.clone(),
+            display_handle,
+        }
+    }
+
+    pub const fn set_pos(&mut self, pos_x: i32, pos_y: i32) {
+        self.pos_x = pos_x;
+        self.pos_y = pos_y;
+    }
+
+    pub fn set_size(&mut self, size_x: u32, size_y: u32) {
+        self.toplevel.with_pending_state(|state| {
+            //state.bounds = Some((size_x as i32, size_y as i32).into());
+            state.size = Some((size_x as i32, size_y as i32).into());
+        });
+        self.toplevel.send_configure();
+
+        self.size_x = size_x;
+        self.size_y = size_y;
+    }
+}
+
+#[derive(Debug)]
+pub struct WindowManager {
+    pub windows: WindowVec,
+}
+
+impl WindowManager {
+    pub const fn new() -> Self {
+        Self {
+            windows: WindowVec::new(),
+        }
+    }
+
+    pub fn find_window_handle(&self, toplevel: &ToplevelSurface) -> Option<WindowHandle> {
+        for (idx, cell) in self.windows.vec.iter().enumerate() {
+            if let Some(cell) = cell {
+                let window = &cell.obj;
+                if window.toplevel == *toplevel {
+                    return Some(WindowVec::get_handle(cell, idx));
+                }
+            }
+        }
+        None
+    }
+
+    pub fn create_window(
+        &mut self,
+        display_handle: display::DisplayHandle,
+        toplevel: &ToplevelSurface,
+    ) -> WindowHandle {
+        self.windows.add(Window::new(display_handle, toplevel))
+    }
+
+    pub fn remove_window(&mut self, window_handle: WindowHandle) {
+        self.windows.remove(&window_handle);
+    }
+}
+
+gen_id!(WindowVec, Window, WindowCell, WindowHandle);
+
+impl WindowHandle {
+    pub const fn from_packet(handle: packet_server::WvrWindowHandle) -> Self {
+        Self {
+            generation: handle.generation,
+            idx: handle.idx,
+        }
+    }
+
+    pub const fn as_packet(&self) -> packet_server::WvrWindowHandle {
+        packet_server::WvrWindowHandle {
+            idx: self.idx,
+            generation: self.generation,
+        }
+    }
+}

wlx-overlay-s/src/config.rs

@@ -0,0 +1,508 @@
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use crate::config_io;
+use crate::overlays::toast::DisplayMethod;
+use crate::overlays::toast::ToastTopic;
+use crate::state::LeftRight;
+use chrono::Offset;
+use config::Config;
+use config::File;
+use glam::vec3a;
+use glam::Affine3A;
+use glam::Quat;
+use glam::Vec3A;
+use idmap::IdMap;
+use log::error;
+use serde::Deserialize;
+use serde::Serialize;
+
+pub type AStrMap<V> = Vec<(Arc<str>, V)>;
+
+pub trait AStrMapExt<V> {
+    fn arc_set(&mut self, key: Arc<str>, value: V) -> bool;
+    fn arc_get(&self, key: &str) -> Option<&V>;
+    fn arc_rm(&mut self, key: &str) -> Option<V>;
+}
+
+impl<V> AStrMapExt<V> for AStrMap<V> {
+    fn arc_set(&mut self, key: Arc<str>, value: V) -> bool {
+        let index = self.iter().position(|(k, _)| k.as_ref().eq(key.as_ref()));
+        index.map(|i| self.remove(i).1);
+        self.push((key, value));
+        true
+    }
+
+    fn arc_get(&self, key: &str) -> Option<&V> {
+        self.iter()
+            .find_map(|(k, v)| if k.as_ref().eq(key) { Some(v) } else { None })
+    }
+
+    fn arc_rm(&mut self, key: &str) -> Option<V> {
+        let index = self.iter().position(|(k, _)| k.as_ref().eq(key));
+        index.map(|i| self.remove(i).1)
+    }
+}
+
+pub type AStrSet = Vec<Arc<str>>;
+
+pub trait AStrSetExt {
+    fn arc_set(&mut self, value: Arc<str>) -> bool;
+    fn arc_get(&self, value: &str) -> bool;
+    fn arc_rm(&mut self, value: &str) -> bool;
+}
+
+impl AStrSetExt for AStrSet {
+    fn arc_set(&mut self, value: Arc<str>) -> bool {
+        if self.iter().any(|v| v.as_ref().eq(value.as_ref())) {
+            return false;
+        }
+        self.push(value);
+        true
+    }
+
+    fn arc_get(&self, value: &str) -> bool {
+        self.iter().any(|v| v.as_ref().eq(value))
+    }
+
+    fn arc_rm(&mut self, value: &str) -> bool {
+        let index = self.iter().position(|v| v.as_ref().eq(value));
+        index.is_some_and(|i| {
+            self.remove(i);
+            true
+        })
+    }
+}
+
+pub type PwTokenMap = AStrMap<String>;
+
+pub const fn def_watch_pos() -> Vec3A {
+    vec3a(-0.03, -0.01, 0.125)
+}
+
+pub const fn def_watch_rot() -> Quat {
+    Quat::from_xyzw(-0.707_106_6, 0.000_796_361_8, 0.707_106_6, 0.0)
+}
+
+pub const fn def_left() -> LeftRight {
+    LeftRight::Left
+}
+
+pub const fn def_pw_tokens() -> PwTokenMap {
+    AStrMap::new()
+}
+
+const fn def_mouse_move_interval_ms() -> u32 {
+    10 // 100fps
+}
+
+const fn def_click_freeze_time_ms() -> u32 {
+    300
+}
+
+pub const fn def_true() -> bool {
+    true
+}
+
+const fn def_false() -> bool {
+    false
+}
+
+const fn def_one() -> f32 {
+    1.0
+}
+
+pub const fn def_half() -> f32 {
+    0.5
+}
+
+pub const fn def_point7() -> f32 {
+    0.7
+}
+
+pub const fn def_point3() -> f32 {
+    0.3
+}
+
+const fn def_osc_port() -> u16 {
+    9000
+}
+
+const fn def_empty_vec_string() -> Vec<String> {
+    Vec::new()
+}
+
+fn def_timezones() -> Vec<String> {
+    const EMEA: i32 = -60 * 60; // UTC-1
+    const APAC: i32 = 5 * 60 * 60; // UTC+5
+
+    let offset = chrono::Local::now().offset().fix();
+    match offset.local_minus_utc() {
+        i32::MIN..EMEA => vec!["Europe/Paris".into(), "Asia/Tokyo".into()],
+        EMEA..APAC => vec!["America/New_York".into(), "Asia/Tokyo".into()],
+        APAC..=i32::MAX => vec!["Europe/Paris".into(), "America/New_York".into()],
+    }
+}
+
+const fn def_screens() -> AStrSet {
+    AStrSet::new()
+}
+
+const fn def_curve_values() -> AStrMap<f32> {
+    AStrMap::new()
+}
+
+const fn def_transforms() -> AStrMap<Affine3A> {
+    AStrMap::new()
+}
+
+fn def_auto() -> Arc<str> {
+    "auto".into()
+}
+
+fn def_empty() -> Arc<str> {
+    "".into()
+}
+
+fn def_toast_topics() -> IdMap<ToastTopic, DisplayMethod> {
+    IdMap::new()
+}
+
+fn def_font() -> Arc<str> {
+    "LiberationSans:style=Bold".into()
+}
+
+const fn def_max_height() -> u16 {
+    1440
+}
+
+#[derive(Deserialize, Serialize)]
+pub struct GeneralConfig {
+    #[serde(default = "def_watch_pos")]
+    pub watch_pos: Vec3A,
+
+    #[serde(default = "def_watch_rot")]
+    pub watch_rot: Quat,
+
+    #[serde(default = "def_left")]
+    pub watch_hand: LeftRight,
+
+    #[serde(default = "def_click_freeze_time_ms")]
+    pub click_freeze_time_ms: u32,
+
+    #[serde(default = "def_mouse_move_interval_ms")]
+    pub mouse_move_interval_ms: u32,
+
+    #[serde(default = "def_true")]
+    pub notifications_enabled: bool,
+
+    #[serde(default = "def_true")]
+    pub notifications_sound_enabled: bool,
+
+    #[serde(default = "def_toast_topics")]
+    pub notification_topics: IdMap<ToastTopic, DisplayMethod>,
+
+    #[serde(default = "def_empty")]
+    pub notification_sound: Arc<str>,
+
+    #[serde(default = "def_true")]
+    pub keyboard_sound_enabled: bool,
+
+    #[serde(default = "def_one")]
+    pub keyboard_scale: f32,
+
+    #[serde(default = "def_one")]
+    pub desktop_view_scale: f32,
+
+    #[serde(default = "def_half")]
+    pub watch_view_angle_min: f32,
+
+    #[serde(default = "def_point7")]
+    pub watch_view_angle_max: f32,
+
+    #[serde(default = "def_one")]
+    pub long_press_duration: f32,
+
+    #[serde(default = "def_osc_port")]
+    pub osc_out_port: u16,
+
+    #[serde(default = "def_false")]
+    pub upright_screen_fix: bool,
+
+    #[serde(default = "def_false")]
+    pub double_cursor_fix: bool,
+
+    #[serde(default = "def_screens")]
+    pub show_screens: AStrSet,
+
+    #[serde(default = "def_curve_values")]
+    pub curve_values: AStrMap<f32>,
+
+    #[serde(default = "def_transforms")]
+    pub transform_values: AStrMap<Affine3A>,
+
+    #[serde(default = "def_auto")]
+    pub capture_method: Arc<str>,
+
+    #[serde(default = "def_point7")]
+    pub xr_grab_sensitivity: f32,
+
+    #[serde(default = "def_point7")]
+    pub xr_click_sensitivity: f32,
+
+    #[serde(default = "def_point7")]
+    pub xr_alt_click_sensitivity: f32,
+
+    #[serde(default = "def_half")]
+    pub xr_grab_sensitivity_release: f32,
+
+    #[serde(default = "def_half")]
+    pub xr_click_sensitivity_release: f32,
+
+    #[serde(default = "def_half")]
+    pub xr_alt_click_sensitivity_release: f32,
+
+    #[serde(default = "def_true")]
+    pub allow_sliding: bool,
+
+    #[serde(default = "def_true")]
+    pub realign_on_showhide: bool,
+
+    #[serde(default = "def_false")]
+    pub focus_follows_mouse_mode: bool,
+
+    #[serde(default = "def_false")]
+    pub block_game_input: bool,
+
+    #[serde(default = "def_true")]
+    pub block_game_input_ignore_watch: bool,
+
+    #[serde(default = "def_font")]
+    pub primary_font: Arc<str>,
+
+    #[serde(default = "def_one")]
+    pub space_drag_multiplier: f32,
+
+    #[serde(default = "def_empty")]
+    pub skybox_texture: Arc<str>,
+
+    #[serde(default = "def_true")]
+    pub use_skybox: bool,
+
+    #[serde(default = "def_true")]
+    pub use_passthrough: bool,
+
+    #[serde(default = "def_max_height")]
+    pub screen_max_height: u16,
+
+    #[serde(default = "def_false")]
+    pub screen_render_down: bool,
+
+    #[serde(default = "def_point3")]
+    pub pointer_lerp_factor: f32,
+
+    #[serde(default = "def_false")]
+    pub space_rotate_unlocked: bool,
+
+    #[serde(default = "def_empty_vec_string")]
+    pub alt_click_down: Vec<String>,
+
+    #[serde(default = "def_empty_vec_string")]
+    pub alt_click_up: Vec<String>,
+
+    #[serde(default = "def_timezones")]
+    pub timezones: Vec<String>,
+}
+
+impl GeneralConfig {
+    fn sanitize_range(name: &str, val: f32, from: f32, to: f32) {
+        assert!(
+            !(!val.is_normal() || val < from || val > to),
+            "GeneralConfig: {name} needs to be between {from} and {to}"
+        );
+    }
+
+    pub fn load_from_disk() -> Self {
+        let config = load_general();
+        config.post_load();
+        config
+    }
+
+    fn post_load(&self) {
+        Self::sanitize_range("keyboard_scale", self.keyboard_scale, 0.05, 5.0);
+        Self::sanitize_range("desktop_view_scale", self.desktop_view_scale, 0.05, 5.0);
+    }
+}
+
+const FALLBACKS: [&str; 5] = [
+    include_str!("res/keyboard.yaml"),
+    include_str!("res/watch.yaml"),
+    include_str!("res/settings.yaml"),
+    include_str!("res/anchor.yaml"),
+    include_str!("res/wayvr.yaml"),
+];
+
+const FILES: [&str; 5] = [
+    "keyboard.yaml",
+    "watch.yaml",
+    "settings.yaml",
+    "anchor.yaml",
+    "wayvr.yaml",
+];
+
+#[derive(Clone, Copy)]
+#[repr(usize)]
+pub enum ConfigType {
+    Keyboard,
+    Watch,
+    Settings,
+    Anchor,
+    #[allow(dead_code)]
+    WayVR,
+}
+
+pub fn load_known_yaml<T>(config_type: ConfigType) -> T
+where
+    T: for<'de> Deserialize<'de>,
+{
+    let fallback = FALLBACKS[config_type as usize];
+    let file_name = FILES[config_type as usize];
+    let maybe_override = config_io::load(file_name);
+
+    for yaml in [maybe_override.as_deref(), Some(fallback)].iter().flatten() {
+        match serde_yaml::from_str::<T>(yaml) {
+            Ok(d) => return d,
+            Err(e) => {
+                error!("Failed to parse {file_name}, falling back to defaults.");
+                error!("{e}");
+            }
+        }
+    }
+    // can only get here if internal fallback is broken
+    panic!("No usable config found.");
+}
+
+pub fn load_config_with_conf_d<ConfigData>(
+    root_config_filename: &str,
+    ctype: config_io::ConfigRoot,
+) -> ConfigData
+where
+    ConfigData: for<'de> Deserialize<'de>,
+{
+    let mut settings_builder = Config::builder();
+
+    // Add files from conf.d directory
+    let path_conf_d = ctype.get_conf_d_path();
+
+    for mut base_conf in [config_io::get_config_root(), path_conf_d.clone()] {
+        base_conf.push(root_config_filename);
+        if base_conf.exists() {
+            log::info!("Loading config file: {}", base_conf.to_string_lossy());
+            settings_builder = settings_builder.add_source(File::from(base_conf));
+        }
+    }
+
+    if let Ok(paths_unsorted) = std::fs::read_dir(path_conf_d) {
+        let mut paths: Vec<_> = paths_unsorted
+            .filter_map(|r| match r {
+                Ok(entry) => Some(entry),
+                Err(e) => {
+                    error!("Failed to read conf.d directory: {e}");
+                    None
+                }
+            })
+            .collect();
+        // Sort paths alphabetically
+        paths.sort_by_key(std::fs::DirEntry::path);
+        for path in paths {
+            log::info!("Loading config file: {}", path.path().to_string_lossy());
+            settings_builder = settings_builder.add_source(File::from(path.path()));
+        }
+    }
+
+    match settings_builder.build() {
+        Ok(settings) => match settings.try_deserialize::<ConfigData>() {
+            Ok(config) => config,
+            Err(e) => {
+                panic!("Failed to deserialize settings: {e}");
+            }
+        },
+        Err(e) => {
+            panic!("Failed to build settings: {e}");
+        }
+    }
+}
+
+pub fn load_general() -> GeneralConfig {
+    load_config_with_conf_d::<GeneralConfig>("config.yaml", config_io::ConfigRoot::Generic)
+}
+
+// Config that is saved from the settings panel
+
+#[derive(Serialize)]
+pub struct AutoSettings {
+    pub watch_pos: Vec3A,
+    pub watch_rot: Quat,
+    pub watch_hand: LeftRight,
+    pub watch_view_angle_min: f32,
+    pub watch_view_angle_max: f32,
+    pub notifications_enabled: bool,
+    pub notifications_sound_enabled: bool,
+    pub realign_on_showhide: bool,
+    pub allow_sliding: bool,
+    pub space_drag_multiplier: f32,
+}
+
+fn get_settings_path() -> PathBuf {
+    config_io::ConfigRoot::Generic
+        .get_conf_d_path()
+        .join("zz-saved-config.json5")
+}
+
+pub fn save_settings(config: &GeneralConfig) -> anyhow::Result<()> {
+    let conf = AutoSettings {
+        watch_pos: config.watch_pos,
+        watch_rot: config.watch_rot,
+        watch_hand: config.watch_hand,
+        watch_view_angle_min: config.watch_view_angle_min,
+        watch_view_angle_max: config.watch_view_angle_max,
+        notifications_enabled: config.notifications_enabled,
+        notifications_sound_enabled: config.notifications_sound_enabled,
+        realign_on_showhide: config.realign_on_showhide,
+        allow_sliding: config.allow_sliding,
+        space_drag_multiplier: config.space_drag_multiplier,
+    };
+
+    let json = serde_json::to_string_pretty(&conf).unwrap(); // want panic
+    std::fs::write(get_settings_path(), json)?;
+
+    Ok(())
+}
+
+// Config that is saved after manipulating overlays
+
+#[derive(Serialize)]
+pub struct AutoState {
+    pub show_screens: AStrSet,
+    pub curve_values: AStrMap<f32>,
+    pub transform_values: AStrMap<Affine3A>,
+}
+
+fn get_state_path() -> PathBuf {
+    config_io::ConfigRoot::Generic
+        .get_conf_d_path()
+        .join("zz-saved-state.json5")
+}
+
+pub fn save_layout(config: &GeneralConfig) -> anyhow::Result<()> {
+    let conf = AutoState {
+        show_screens: config.show_screens.clone(),
+        curve_values: config.curve_values.clone(),
+        transform_values: config.transform_values.clone(),
+    };
+
+    let json = serde_json::to_string_pretty(&conf).unwrap(); // want panic
+    std::fs::write(get_state_path(), json)?;
+
+    Ok(())
+}

wlx-overlay-s/src/config_io.rs

@@ -0,0 +1,54 @@
+use log::error;
+use std::{path::PathBuf, sync::LazyLock};
+
+pub enum ConfigRoot {
+    Generic,
+    #[allow(dead_code)]
+    WayVR,
+}
+
+const FALLBACK_CONFIG_PATH: &str = "/tmp/wlxoverlay";
+
+static CONFIG_ROOT_PATH: LazyLock<PathBuf> = LazyLock::new(|| {
+    if let Some(mut dir) = xdg::BaseDirectories::new().get_config_home() {
+        dir.push("wlxoverlay");
+        return dir;
+    }
+    //Return fallback config path
+    error!("Err: Failed to find config path, using {FALLBACK_CONFIG_PATH}");
+    PathBuf::from(FALLBACK_CONFIG_PATH)
+});
+
+pub fn get_config_root() -> PathBuf {
+    CONFIG_ROOT_PATH.clone()
+}
+
+impl ConfigRoot {
+    pub fn get_conf_d_path(&self) -> PathBuf {
+        get_config_root().join(match self {
+            Self::Generic => "conf.d",
+            Self::WayVR => "wayvr.conf.d",
+        })
+    }
+
+    // Make sure config directory is present and return root config path
+    pub fn ensure_dir(&self) -> PathBuf {
+        let path = get_config_root();
+        let _ = std::fs::create_dir(&path);
+
+        let path_conf_d = self.get_conf_d_path();
+        let _ = std::fs::create_dir(path_conf_d);
+        path
+    }
+}
+
+pub fn get_config_file_path(filename: &str) -> PathBuf {
+    get_config_root().join(filename)
+}
+
+pub fn load(filename: &str) -> Option<String> {
+    let path = get_config_file_path(filename);
+    log::info!("Loading config: {}", path.to_string_lossy());
+
+    std::fs::read_to_string(path).ok()
+}

wlx-overlay-s/src/config_wayvr.rs

@@ -0,0 +1,275 @@
+#[cfg(not(feature = "wayvr"))]
+compile_error!("WayVR feature is not enabled");
+
+use std::{
+    cell::RefCell,
+    collections::{BTreeMap, HashMap},
+    rc::Rc,
+    sync::Arc,
+};
+
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    backend::{
+        overlay::Positioning,
+        task::{TaskContainer, TaskType},
+        wayvr::{self, WayVRAction},
+    },
+    config::load_config_with_conf_d,
+    config_io,
+    overlays::wayvr::{executable_exists_in_path, WayVRData},
+};
+
+// Flat version of RelativeTo
+#[derive(Clone, Deserialize, Serialize)]
+pub enum AttachTo {
+    None,
+    HandLeft,
+    HandRight,
+    Head,
+    Stage,
+}
+
+impl AttachTo {
+    // TODO: adjustable lerp factor
+    pub const fn get_positioning(&self) -> Positioning {
+        match self {
+            Self::None => Positioning::Floating,
+            Self::HandLeft => Positioning::FollowHand { hand: 0, lerp: 1.0 },
+            Self::HandRight => Positioning::FollowHand { hand: 1, lerp: 1.0 },
+            Self::Stage => Positioning::Static,
+            Self::Head => Positioning::FollowHead { lerp: 1.0 },
+        }
+    }
+
+    pub const fn from_packet(input: &wayvr_ipc::packet_client::AttachTo) -> Self {
+        match input {
+            wayvr_ipc::packet_client::AttachTo::None => Self::None,
+            wayvr_ipc::packet_client::AttachTo::HandLeft => Self::HandLeft,
+            wayvr_ipc::packet_client::AttachTo::HandRight => Self::HandRight,
+            wayvr_ipc::packet_client::AttachTo::Head => Self::Head,
+            wayvr_ipc::packet_client::AttachTo::Stage => Self::Stage,
+        }
+    }
+}
+
+#[derive(Clone, Deserialize, Serialize)]
+pub struct Rotation {
+    pub axis: [f32; 3],
+    pub angle: f32,
+}
+
+#[derive(Clone, Deserialize, Serialize)]
+pub struct WayVRAppEntry {
+    pub name: String,
+    pub target_display: String,
+    pub exec: String,
+    pub args: Option<String>,
+    pub env: Option<Vec<String>>,
+    pub shown_at_start: Option<bool>,
+}
+
+#[derive(Clone, Deserialize, Serialize)]
+pub struct WayVRDisplay {
+    pub width: u16,
+    pub height: u16,
+    pub scale: Option<f32>,
+    pub rotation: Option<Rotation>,
+    pub pos: Option<[f32; 3]>,
+    pub attach_to: Option<AttachTo>,
+    pub primary: Option<bool>,
+}
+
+#[derive(Clone, Deserialize, Serialize)]
+pub struct WayVRCatalog {
+    pub apps: Vec<WayVRAppEntry>,
+}
+
+impl WayVRCatalog {
+    pub fn get_app(&self, name: &str) -> Option<&WayVRAppEntry> {
+        self.apps.iter().find(|&app| app.name.as_str() == name)
+    }
+}
+
+const fn def_false() -> bool {
+    false
+}
+
+const fn def_true() -> bool {
+    true
+}
+
+const fn def_autohide_delay() -> u32 {
+    750
+}
+
+const fn def_keyboard_repeat_delay() -> u32 {
+    200
+}
+
+const fn def_keyboard_repeat_rate() -> u32 {
+    50
+}
+
+fn def_blit_method() -> String {
+    String::from("dmabuf")
+}
+
+#[derive(Clone, Deserialize, Serialize)]
+pub struct WayVRDashboard {
+    pub exec: String,
+    pub working_dir: Option<String>,
+    pub args: Option<String>,
+    pub env: Option<Vec<String>>,
+}
+
+#[derive(Deserialize, Serialize)]
+pub struct WayVRConfig {
+    #[serde(default = "def_false")]
+    pub run_compositor_at_start: bool,
+
+    #[serde(default = "Default::default")]
+    pub catalogs: HashMap<String, WayVRCatalog>,
+
+    #[serde(default = "Default::default")]
+    pub displays: BTreeMap<String, WayVRDisplay>, // sorted alphabetically
+
+    #[serde(default = "Default::default")]
+    pub dashboard: Option<WayVRDashboard>,
+
+    #[serde(default = "def_true")]
+    pub auto_hide: bool,
+
+    #[serde(default = "def_autohide_delay")]
+    pub auto_hide_delay: u32,
+
+    #[serde(default = "def_keyboard_repeat_delay")]
+    pub keyboard_repeat_delay: u32,
+
+    #[serde(default = "def_keyboard_repeat_rate")]
+    pub keyboard_repeat_rate: u32,
+
+    #[serde(default = "def_blit_method")]
+    pub blit_method: String,
+}
+
+impl WayVRConfig {
+    pub fn get_catalog(&self, name: &str) -> Option<&WayVRCatalog> {
+        self.catalogs.get(name)
+    }
+
+    pub fn get_display(&self, name: &str) -> Option<&WayVRDisplay> {
+        self.displays.get(name)
+    }
+
+    pub fn get_default_display(&self) -> Option<(String, &WayVRDisplay)> {
+        for (disp_name, disp) in &self.displays {
+            if disp.primary.unwrap_or(false) {
+                return Some((disp_name.clone(), disp));
+            }
+        }
+        None
+    }
+
+    pub fn get_wayvr_config(
+        config_general: &crate::config::GeneralConfig,
+        config_wayvr: &Self,
+    ) -> anyhow::Result<wayvr::Config> {
+        Ok(wayvr::Config {
+            click_freeze_time_ms: config_general.click_freeze_time_ms,
+            keyboard_repeat_delay_ms: config_wayvr.keyboard_repeat_delay,
+            keyboard_repeat_rate: config_wayvr.keyboard_repeat_rate,
+            blit_method: wayvr::BlitMethod::from_string(&config_wayvr.blit_method)
+                .ok_or_else(|| anyhow::anyhow!("Unknown blit method"))?,
+            auto_hide_delay: if config_wayvr.auto_hide {
+                Some(config_wayvr.auto_hide_delay)
+            } else {
+                None
+            },
+        })
+    }
+
+    pub fn post_load(
+        &self,
+        config: &crate::config::GeneralConfig,
+        tasks: &mut TaskContainer,
+    ) -> anyhow::Result<Option<Rc<RefCell<WayVRData>>>> {
+        let primary_count = self
+            .displays
+            .iter()
+            .filter(|d| d.1.primary.unwrap_or(false))
+            .count();
+
+        if primary_count > 1 {
+            anyhow::bail!("Number of primary displays is more than 1")
+        } else if primary_count == 0 {
+            log::warn!(
+                "No primary display specified. External Wayland applications will not be attached."
+            );
+        }
+
+        for (catalog_name, catalog) in &self.catalogs {
+            for app in &catalog.apps {
+                if let Some(b) = app.shown_at_start {
+                    if b {
+                        tasks.enqueue(TaskType::WayVR(WayVRAction::AppClick {
+                            catalog_name: Arc::from(catalog_name.as_str()),
+                            app_name: Arc::from(app.name.as_str()),
+                        }));
+                    }
+                }
+            }
+        }
+
+        if self.run_compositor_at_start {
+            // Start Wayland server instantly
+            Ok(Some(Rc::new(RefCell::new(WayVRData::new(
+                Self::get_wayvr_config(config, self)?,
+            )?))))
+        } else {
+            // Lazy-init WayVR later if the user requested
+            Ok(None)
+        }
+    }
+}
+
+fn get_default_dashboard_exec() -> (
+    String,         /* exec path */
+    Option<String>, /* working directory */
+) {
+    if let Ok(appdir) = std::env::var("APPDIR") {
+        // Running in AppImage
+        let embedded_path = format!("{appdir}/usr/bin/wayvr-dashboard");
+        if executable_exists_in_path(&embedded_path) {
+            log::info!("Using WayVR Dashboard from AppDir: {embedded_path}");
+            return (embedded_path, Some(format!("{appdir}/usr")));
+        }
+    }
+    (String::from("wayvr-dashboard"), None)
+}
+
+pub fn load_wayvr() -> WayVRConfig {
+    let config_root_path = config_io::ConfigRoot::WayVR.ensure_dir();
+    log::info!("WayVR Config root path: {}", config_root_path.display());
+    log::info!(
+        "WayVR conf.d path: {}",
+        config_io::ConfigRoot::WayVR.get_conf_d_path().display()
+    );
+
+    let mut conf =
+        load_config_with_conf_d::<WayVRConfig>("wayvr.yaml", config_io::ConfigRoot::WayVR);
+
+    if conf.dashboard.is_none() {
+        let (exec, working_dir) = get_default_dashboard_exec();
+
+        conf.dashboard = Some(WayVRDashboard {
+            args: None,
+            env: None,
+            exec,
+            working_dir,
+        });
+    }
+
+    conf
+}

wlx-overlay-s/src/graphics/dds.rs

@@ -0,0 +1,101 @@
+use image_dds::{ImageFormat, Surface};
+use std::{io::Read, sync::Arc};
+use vulkano::{
+    buffer::{Buffer, BufferCreateInfo, BufferUsage, Subbuffer},
+    command_buffer::CopyBufferToImageInfo,
+    format::Format,
+    image::{Image, ImageCreateInfo, ImageType, ImageUsage},
+    memory::allocator::{AllocationCreateInfo, MemoryTypeFilter},
+    DeviceSize,
+};
+use wgui::gfx::cmd::XferCommandBuffer;
+
+pub trait WlxCommandBufferDds {
+    fn upload_image_dds<R>(&mut self, r: R) -> anyhow::Result<Arc<Image>>
+    where
+        R: Read;
+}
+
+impl WlxCommandBufferDds for XferCommandBuffer {
+    fn upload_image_dds<R>(&mut self, r: R) -> anyhow::Result<Arc<Image>>
+    where
+        R: Read,
+    {
+        let Ok(dds) = image_dds::ddsfile::Dds::read(r) else {
+            anyhow::bail!("Not a valid DDS file.\nSee: https://github.com/galister/wlx-overlay-s/wiki/Custom-Textures");
+        };
+
+        let surface = Surface::from_dds(&dds)?;
+
+        if surface.depth != 1 {
+            anyhow::bail!("Not a 2D texture.")
+        }
+
+        let image = Image::new(
+            self.graphics.memory_allocator.clone(),
+            ImageCreateInfo {
+                image_type: ImageType::Dim2d,
+                format: dds_to_vk(surface.image_format)?,
+                extent: [surface.width, surface.height, surface.depth],
+                usage: ImageUsage::TRANSFER_DST | ImageUsage::TRANSFER_SRC | ImageUsage::SAMPLED,
+                ..Default::default()
+            },
+            AllocationCreateInfo::default(),
+        )?;
+
+        let buffer: Subbuffer<[u8]> = Buffer::new_slice(
+            self.graphics.memory_allocator.clone(),
+            BufferCreateInfo {
+                usage: BufferUsage::TRANSFER_SRC,
+                ..Default::default()
+            },
+            AllocationCreateInfo {
+                memory_type_filter: MemoryTypeFilter::PREFER_HOST
+                    | MemoryTypeFilter::HOST_SEQUENTIAL_WRITE,
+                ..Default::default()
+            },
+            surface.data.len() as DeviceSize,
+        )?;
+
+        buffer.write()?.copy_from_slice(surface.data);
+
+        self.command_buffer
+            .copy_buffer_to_image(CopyBufferToImageInfo::buffer_image(buffer, image.clone()))?;
+
+        Ok(image)
+    }
+}
+
+pub fn dds_to_vk(dds_fmt: ImageFormat) -> anyhow::Result<Format> {
+    match dds_fmt {
+        ImageFormat::R8Unorm => Ok(Format::R8_UNORM),
+        ImageFormat::Rgba8Unorm => Ok(Format::R8G8B8A8_UNORM),
+        ImageFormat::Rgba8UnormSrgb => Ok(Format::R8G8B8A8_SRGB),
+        ImageFormat::Rgba16Float => Ok(Format::R16G16B16A16_SFLOAT),
+        ImageFormat::Rgba32Float => Ok(Format::R32G32B32A32_SFLOAT),
+        ImageFormat::Bgra8Unorm => Ok(Format::B8G8R8A8_UNORM),
+        ImageFormat::Bgra8UnormSrgb => Ok(Format::B8G8R8A8_SRGB),
+        // DXT1
+        ImageFormat::BC1RgbaUnorm => Ok(Format::BC1_RGBA_UNORM_BLOCK),
+        ImageFormat::BC1RgbaUnormSrgb => Ok(Format::BC1_RGBA_SRGB_BLOCK),
+        // DXT3
+        ImageFormat::BC2RgbaUnorm => Ok(Format::BC2_UNORM_BLOCK),
+        ImageFormat::BC2RgbaUnormSrgb => Ok(Format::BC2_SRGB_BLOCK),
+        // DXT5
+        ImageFormat::BC3RgbaUnorm => Ok(Format::BC3_UNORM_BLOCK),
+        ImageFormat::BC3RgbaUnormSrgb => Ok(Format::BC3_SRGB_BLOCK),
+        // RGTC1
+        ImageFormat::BC4RUnorm => Ok(Format::BC4_UNORM_BLOCK),
+        ImageFormat::BC4RSnorm => Ok(Format::BC4_SNORM_BLOCK),
+        // RGTC2
+        ImageFormat::BC5RgUnorm => Ok(Format::BC5_UNORM_BLOCK),
+        ImageFormat::BC5RgSnorm => Ok(Format::BC5_SNORM_BLOCK),
+        // BPTC
+        ImageFormat::BC6hRgbUfloat => Ok(Format::BC6H_UFLOAT_BLOCK),
+        ImageFormat::BC6hRgbSfloat => Ok(Format::BC6H_SFLOAT_BLOCK),
+        // BPTC
+        ImageFormat::BC7RgbaUnorm => Ok(Format::BC7_UNORM_BLOCK),
+        ImageFormat::BC7RgbaUnormSrgb => Ok(Format::BC7_SRGB_BLOCK),
+        _ => anyhow::bail!("Unsupported format {:?}", dds_fmt),
+    }
+}

wlx-overlay-s/src/graphics/dmabuf.rs

@@ -0,0 +1,350 @@
+use std::{
+    mem::MaybeUninit,
+    os::fd::{FromRawFd, IntoRawFd},
+    sync::Arc,
+};
+
+use smallvec::SmallVec;
+use vulkano::{
+    device::Device,
+    format::Format,
+    image::{sys::RawImage, Image, ImageCreateInfo, ImageTiling, ImageUsage, SubresourceLayout},
+    memory::{
+        allocator::{MemoryAllocator, MemoryTypeFilter},
+        DedicatedAllocation, DeviceMemory, ExternalMemoryHandleType, ExternalMemoryHandleTypes,
+        MemoryAllocateInfo, MemoryImportInfo, MemoryPropertyFlags, ResourceMemory,
+    },
+    sync::Sharing,
+    VulkanError, VulkanObject,
+};
+use wgui::gfx::WGfx;
+use wlx_capture::frame::{
+    DmabufFrame, DrmFormat, FourCC, DRM_FORMAT_ABGR2101010, DRM_FORMAT_ABGR8888,
+    DRM_FORMAT_ARGB8888, DRM_FORMAT_XBGR2101010, DRM_FORMAT_XBGR8888, DRM_FORMAT_XRGB8888,
+};
+
+pub const DRM_FORMAT_MOD_INVALID: u64 = 0xff_ffff_ffff_ffff;
+
+pub trait WGfxDmabuf {
+    fn dmabuf_texture_ex(
+        &self,
+        frame: DmabufFrame,
+        tiling: ImageTiling,
+        layouts: Vec<SubresourceLayout>,
+        modifiers: &[u64],
+    ) -> anyhow::Result<Arc<Image>>;
+
+    fn dmabuf_texture(&self, frame: DmabufFrame) -> anyhow::Result<Arc<Image>>;
+}
+
+impl WGfxDmabuf for WGfx {
+    fn dmabuf_texture_ex(
+        &self,
+        frame: DmabufFrame,
+        tiling: ImageTiling,
+        layouts: Vec<SubresourceLayout>,
+        modifiers: &[u64],
+    ) -> anyhow::Result<Arc<Image>> {
+        let extent = [frame.format.width, frame.format.height, 1];
+        let format = fourcc_to_vk(frame.format.fourcc)?;
+
+        let image = unsafe {
+            create_dmabuf_image(
+                self.device.clone(),
+                ImageCreateInfo {
+                    format,
+                    extent,
+                    usage: ImageUsage::SAMPLED,
+                    external_memory_handle_types: ExternalMemoryHandleTypes::DMA_BUF,
+                    tiling,
+                    drm_format_modifiers: modifiers.to_owned(),
+                    drm_format_modifier_plane_layouts: layouts,
+                    ..Default::default()
+                },
+            )?
+        };
+
+        let requirements = image.memory_requirements()[0];
+        let memory_type_index = self
+            .memory_allocator
+            .find_memory_type_index(
+                requirements.memory_type_bits,
+                MemoryTypeFilter {
+                    required_flags: MemoryPropertyFlags::DEVICE_LOCAL,
+                    ..Default::default()
+                },
+            )
+            .ok_or_else(|| anyhow::anyhow!("failed to get memory type index"))?;
+
+        debug_assert!(self.device.enabled_extensions().khr_external_memory_fd);
+        debug_assert!(self.device.enabled_extensions().khr_external_memory);
+        debug_assert!(self.device.enabled_extensions().ext_external_memory_dma_buf);
+
+        // only do the 1st
+        unsafe {
+            let Some(fd) = frame.planes[0].fd else {
+                anyhow::bail!("DMA-buf plane has no FD");
+            };
+
+            let file = std::fs::File::from_raw_fd(fd);
+            let new_file = file.try_clone()?;
+            let _ = file.into_raw_fd();
+
+            let memory = DeviceMemory::allocate_unchecked(
+                self.device.clone(),
+                MemoryAllocateInfo {
+                    allocation_size: requirements.layout.size(),
+                    memory_type_index,
+                    dedicated_allocation: Some(DedicatedAllocation::Image(&image)),
+                    ..Default::default()
+                },
+                Some(MemoryImportInfo::Fd {
+                    file: new_file,
+                    handle_type: ExternalMemoryHandleType::DmaBuf,
+                }),
+            )?;
+
+            let mem_alloc = ResourceMemory::new_dedicated(memory);
+            match image.bind_memory_unchecked([mem_alloc]) {
+                Ok(image) => Ok(Arc::new(image)),
+                Err(e) => {
+                    anyhow::bail!("Failed to bind memory to image: {}", e.0);
+                }
+            }
+        }
+    }
+
+    fn dmabuf_texture(&self, frame: DmabufFrame) -> anyhow::Result<Arc<Image>> {
+        let mut modifiers: Vec<u64> = vec![];
+        let mut tiling: ImageTiling = ImageTiling::Optimal;
+        let mut layouts: Vec<SubresourceLayout> = vec![];
+
+        if frame.format.modifier != DRM_FORMAT_MOD_INVALID {
+            (0..frame.num_planes).for_each(|i| {
+                let plane = &frame.planes[i];
+                layouts.push(SubresourceLayout {
+                    offset: plane.offset.into(),
+                    size: 0,
+                    row_pitch: plane.stride as _,
+                    array_pitch: None,
+                    depth_pitch: None,
+                });
+                modifiers.push(frame.format.modifier);
+            });
+            tiling = ImageTiling::DrmFormatModifier;
+        }
+
+        self.dmabuf_texture_ex(frame, tiling, layouts, &modifiers)
+    }
+}
+
+#[allow(clippy::all, clippy::pedantic)]
+pub(super) unsafe fn create_dmabuf_image(
+    device: Arc<Device>,
+    create_info: ImageCreateInfo,
+) -> Result<RawImage, VulkanError> {
+    let &ImageCreateInfo {
+        flags,
+        image_type,
+        format,
+        ref view_formats,
+        extent,
+        array_layers,
+        mip_levels,
+        samples,
+        tiling,
+        usage,
+        stencil_usage,
+        ref sharing,
+        initial_layout,
+        ref drm_format_modifiers,
+        ref drm_format_modifier_plane_layouts,
+        external_memory_handle_types,
+        _ne: _,
+    } = &create_info;
+
+    let (sharing_mode, queue_family_index_count, p_queue_family_indices) = match sharing {
+        Sharing::Exclusive => (ash::vk::SharingMode::EXCLUSIVE, 0, &[] as _),
+        Sharing::Concurrent(queue_family_indices) => (
+            ash::vk::SharingMode::CONCURRENT,
+            queue_family_indices.len() as u32,
+            queue_family_indices.as_ptr(),
+        ),
+    };
+
+    let mut create_info_vk = ash::vk::ImageCreateInfo {
+        flags: flags.into(),
+        image_type: image_type.into(),
+        format: format.into(),
+        extent: ash::vk::Extent3D {
+            width: extent[0],
+            height: extent[1],
+            depth: extent[2],
+        },
+        mip_levels,
+        array_layers,
+        samples: samples.into(),
+        tiling: tiling.into(),
+        usage: usage.into(),
+        sharing_mode,
+        queue_family_index_count,
+        p_queue_family_indices,
+        initial_layout: initial_layout.into(),
+        ..Default::default()
+    };
+
+    let mut drm_format_modifier_explicit_info_vk = None;
+    let drm_format_modifier_plane_layouts_vk: SmallVec<[_; 4]>;
+    let mut drm_format_modifier_list_info_vk = None;
+    let mut external_memory_info_vk = None;
+    let mut format_list_info_vk = None;
+    let format_list_view_formats_vk: Vec<_>;
+    let mut stencil_usage_info_vk = None;
+
+    if drm_format_modifiers.len() == 1 {
+        drm_format_modifier_plane_layouts_vk = drm_format_modifier_plane_layouts
+            .iter()
+            .map(|subresource_layout| {
+                let &SubresourceLayout {
+                    offset,
+                    size,
+                    row_pitch,
+                    array_pitch,
+                    depth_pitch,
+                } = subresource_layout;
+
+                ash::vk::SubresourceLayout {
+                    offset,
+                    size,
+                    row_pitch,
+                    array_pitch: array_pitch.unwrap_or(0),
+                    depth_pitch: depth_pitch.unwrap_or(0),
+                }
+            })
+            .collect();
+
+        let next = drm_format_modifier_explicit_info_vk.insert(
+            ash::vk::ImageDrmFormatModifierExplicitCreateInfoEXT {
+                drm_format_modifier: drm_format_modifiers[0],
+                drm_format_modifier_plane_count: drm_format_modifier_plane_layouts_vk.len() as u32,
+                p_plane_layouts: drm_format_modifier_plane_layouts_vk.as_ptr(),
+                ..Default::default()
+            },
+        );
+
+        next.p_next = create_info_vk.p_next;
+        create_info_vk.p_next = next as *const _ as *const _;
+    } else if drm_format_modifiers.len() > 1 {
+        let next = drm_format_modifier_list_info_vk.insert(
+            ash::vk::ImageDrmFormatModifierListCreateInfoEXT {
+                drm_format_modifier_count: drm_format_modifiers.len() as u32,
+                p_drm_format_modifiers: drm_format_modifiers.as_ptr(),
+                ..Default::default()
+            },
+        );
+
+        next.p_next = create_info_vk.p_next;
+        create_info_vk.p_next = next as *const _ as *const _;
+    }
+
+    if !external_memory_handle_types.is_empty() {
+        let next = external_memory_info_vk.insert(ash::vk::ExternalMemoryImageCreateInfo {
+            handle_types: external_memory_handle_types.into(),
+            ..Default::default()
+        });
+
+        next.p_next = create_info_vk.p_next;
+        create_info_vk.p_next = next as *const _ as *const _;
+    }
+
+    if !view_formats.is_empty() {
+        format_list_view_formats_vk = view_formats
+            .iter()
+            .copied()
+            .map(ash::vk::Format::from)
+            .collect();
+
+        let next = format_list_info_vk.insert(ash::vk::ImageFormatListCreateInfo {
+            view_format_count: format_list_view_formats_vk.len() as u32,
+            p_view_formats: format_list_view_formats_vk.as_ptr(),
+            ..Default::default()
+        });
+
+        next.p_next = create_info_vk.p_next;
+        create_info_vk.p_next = next as *const _ as *const _;
+    }
+
+    if let Some(stencil_usage) = stencil_usage {
+        let next = stencil_usage_info_vk.insert(ash::vk::ImageStencilUsageCreateInfo {
+            stencil_usage: stencil_usage.into(),
+            ..Default::default()
+        });
+
+        next.p_next = create_info_vk.p_next;
+        create_info_vk.p_next = next as *const _ as *const _;
+    }
+
+    let handle = {
+        let fns = device.fns();
+        let mut output = MaybeUninit::uninit();
+        (fns.v1_0.create_image)(
+            device.handle(),
+            &create_info_vk,
+            std::ptr::null(),
+            output.as_mut_ptr(),
+        )
+        .result()
+        .map_err(VulkanError::from)?;
+        output.assume_init()
+    };
+
+    RawImage::from_handle(device, handle, create_info)
+}
+
+pub fn get_drm_formats(device: Arc<Device>) -> Vec<DrmFormat> {
+    let possible_formats = [
+        DRM_FORMAT_ABGR8888.into(),
+        DRM_FORMAT_XBGR8888.into(),
+        DRM_FORMAT_ARGB8888.into(),
+        DRM_FORMAT_XRGB8888.into(),
+        DRM_FORMAT_ABGR2101010.into(),
+        DRM_FORMAT_XBGR2101010.into(),
+    ];
+
+    let mut final_formats = vec![];
+
+    for &f in &possible_formats {
+        let Ok(vk_fmt) = fourcc_to_vk(f) else {
+            continue;
+        };
+        let Ok(props) = device.physical_device().format_properties(vk_fmt) else {
+            continue;
+        };
+        let mut fmt = DrmFormat {
+            fourcc: f,
+            modifiers: props
+                .drm_format_modifier_properties
+                .iter()
+                // important bit: only allow single-plane
+                .filter(|m| m.drm_format_modifier_plane_count == 1)
+                .map(|m| m.drm_format_modifier)
+                .collect(),
+        };
+        fmt.modifiers.push(DRM_FORMAT_MOD_INVALID); // implicit modifiers support
+        final_formats.push(fmt);
+    }
+    log::debug!("Supported DRM formats:");
+    for f in &final_formats {
+        log::debug!("  {} {:?}", f.fourcc, f.modifiers);
+    }
+    final_formats
+}
+
+pub fn fourcc_to_vk(fourcc: FourCC) -> anyhow::Result<Format> {
+    match fourcc.value {
+        DRM_FORMAT_ABGR8888 | DRM_FORMAT_XBGR8888 => Ok(Format::R8G8B8A8_UNORM),
+        DRM_FORMAT_ARGB8888 | DRM_FORMAT_XRGB8888 => Ok(Format::B8G8R8A8_UNORM),
+        DRM_FORMAT_ABGR2101010 | DRM_FORMAT_XBGR2101010 => Ok(Format::A2B10G10R10_UNORM_PACK32),
+        _ => anyhow::bail!("Unsupported format {}", fourcc),
+    }
+}

wlx-overlay-s/src/graphics/mod.rs

@@ -0,0 +1,666 @@
+pub mod dds;
+pub mod dmabuf;
+
+use std::{
+    collections::HashMap,
+    sync::{Arc, OnceLock},
+};
+
+use glam::{vec2, Vec2};
+use vulkano::{
+    buffer::{BufferCreateInfo, BufferUsage},
+    command_buffer::{PrimaryAutoCommandBuffer, PrimaryCommandBufferAbstract},
+    image::view::ImageView,
+    memory::allocator::{AllocationCreateInfo, MemoryTypeFilter},
+    sync::GpuFuture,
+};
+use wgui::gfx::WGfx;
+
+#[cfg(feature = "openvr")]
+use vulkano::instance::InstanceCreateFlags;
+use wlx_capture::frame::DrmFormat;
+
+use crate::shaders::{frag_color, frag_grid, frag_screen, frag_srgb, vert_quad};
+
+#[cfg(feature = "openxr")]
+use {ash::vk, std::os::raw::c_void};
+
+use vulkano::{
+    self,
+    buffer::{Buffer, BufferContents, IndexBuffer, Subbuffer},
+    device::{
+        physical::{PhysicalDevice, PhysicalDeviceType},
+        Device, DeviceCreateInfo, DeviceExtensions, DeviceFeatures, Queue, QueueCreateInfo,
+        QueueFlags,
+    },
+    format::Format,
+    instance::{Instance, InstanceCreateInfo, InstanceExtensions},
+    pipeline::graphics::{
+        color_blend::{AttachmentBlend, BlendFactor, BlendOp},
+        vertex_input::Vertex,
+    },
+    shader::ShaderModule,
+    VulkanObject,
+};
+
+use dmabuf::get_drm_formats;
+
+pub type Vert2Buf = Subbuffer<[Vert2Uv]>;
+pub type IndexBuf = IndexBuffer;
+
+#[repr(C)]
+#[derive(BufferContents, Vertex, Copy, Clone, Debug)]
+pub struct Vert2Uv {
+    #[format(R32G32_SFLOAT)]
+    pub in_pos: [f32; 2],
+    #[format(R32G32_SFLOAT)]
+    pub in_uv: [f32; 2],
+}
+
+pub const INDICES: [u16; 6] = [2, 1, 0, 1, 2, 3];
+
+pub const BLEND_ALPHA: AttachmentBlend = AttachmentBlend {
+    src_color_blend_factor: BlendFactor::SrcAlpha,
+    dst_color_blend_factor: BlendFactor::OneMinusSrcAlpha,
+    color_blend_op: BlendOp::Add,
+    src_alpha_blend_factor: BlendFactor::One,
+    dst_alpha_blend_factor: BlendFactor::One,
+    alpha_blend_op: BlendOp::Max,
+};
+
+pub struct WGfxExtras {
+    pub shaders: HashMap<&'static str, Arc<ShaderModule>>,
+    pub drm_formats: Vec<DrmFormat>,
+    pub queue_capture: Option<Arc<Queue>>,
+    pub quad_verts: Vert2Buf,
+}
+
+impl WGfxExtras {
+    pub fn new(gfx: &WGfx, queue_capture: Option<Arc<Queue>>) -> anyhow::Result<Self> {
+        let mut shaders = HashMap::new();
+
+        let shader = vert_quad::load(gfx.device.clone())?;
+        shaders.insert("vert_quad", shader);
+
+        let shader = frag_color::load(gfx.device.clone())?;
+        shaders.insert("frag_color", shader);
+
+        let shader = frag_srgb::load(gfx.device.clone())?;
+        shaders.insert("frag_srgb", shader);
+
+        let shader = frag_grid::load(gfx.device.clone())?;
+        shaders.insert("frag_grid", shader);
+
+        let shader = frag_screen::load(gfx.device.clone())?;
+        shaders.insert("frag_screen", shader);
+
+        let drm_formats = get_drm_formats(gfx.device.clone());
+
+        let vertices = [
+            Vert2Uv {
+                in_pos: [0., 0.],
+                in_uv: [0., 0.],
+            },
+            Vert2Uv {
+                in_pos: [1., 0.],
+                in_uv: [1., 0.],
+            },
+            Vert2Uv {
+                in_pos: [0., 1.],
+                in_uv: [0., 1.],
+            },
+            Vert2Uv {
+                in_pos: [1., 1.],
+                in_uv: [1., 1.],
+            },
+        ];
+        let quad_verts = Buffer::from_iter(
+            gfx.memory_allocator.clone(),
+            BufferCreateInfo {
+                usage: BufferUsage::VERTEX_BUFFER,
+                ..Default::default()
+            },
+            AllocationCreateInfo {
+                memory_type_filter: MemoryTypeFilter::PREFER_DEVICE
+                    | MemoryTypeFilter::HOST_SEQUENTIAL_WRITE,
+                ..Default::default()
+            },
+            vertices.into_iter(),
+        )?;
+
+        Ok(Self {
+            shaders,
+            drm_formats,
+            queue_capture,
+            quad_verts,
+        })
+    }
+}
+
+const fn get_dmabuf_extensions() -> DeviceExtensions {
+    DeviceExtensions {
+        khr_external_memory: true,
+        khr_external_memory_fd: true,
+        ext_external_memory_dma_buf: true,
+        ..DeviceExtensions::empty()
+    }
+}
+
+static VULKAN_LIBRARY: OnceLock<Arc<vulkano::VulkanLibrary>> = OnceLock::new();
+fn get_vulkan_library() -> &'static Arc<vulkano::VulkanLibrary> {
+    VULKAN_LIBRARY.get_or_init(|| vulkano::VulkanLibrary::new().unwrap()) // want panic
+}
+
+#[cfg(feature = "openxr")]
+unsafe extern "system" fn get_instance_proc_addr(
+    instance: openxr::sys::platform::VkInstance,
+    name: *const std::ffi::c_char,
+) -> Option<unsafe extern "system" fn()> {
+    use vulkano::Handle;
+    let instance = ash::vk::Instance::from_raw(instance as _);
+    let library = get_vulkan_library();
+    library.get_instance_proc_addr(instance, name)
+}
+
+#[cfg(feature = "openxr")]
+#[allow(clippy::too_many_lines)]
+pub fn init_openxr_graphics(
+    xr_instance: openxr::Instance,
+    system: openxr::SystemId,
+) -> anyhow::Result<(Arc<WGfx>, WGfxExtras)> {
+    use std::ffi::{self, CString};
+
+    use vulkano::{Handle, Version};
+
+    let instance_extensions = InstanceExtensions {
+        khr_get_physical_device_properties2: true,
+        ..InstanceExtensions::empty()
+    };
+
+    let instance_extensions_raw = instance_extensions
+        .into_iter()
+        .filter_map(|(name, enabled)| {
+            if enabled {
+                Some(ffi::CString::new(name).unwrap().into_raw().cast_const())
+            // want panic
+            } else {
+                None
+            }
+        })
+        .collect::<Vec<_>>();
+
+    let vk_target_version = vk::make_api_version(0, 1, 3, 0);
+    let target_version = vulkano::Version::V1_3;
+    let library = get_vulkan_library();
+
+    let vk_app_info_raw = vk::ApplicationInfo::default()
+        .application_version(0)
+        .engine_version(0)
+        .api_version(vk_target_version);
+
+    let instance = unsafe {
+        let vk_instance = xr_instance
+            .create_vulkan_instance(
+                system,
+                get_instance_proc_addr,
+                std::ptr::from_ref(
+                    &vk::InstanceCreateInfo::default()
+                        .application_info(&vk_app_info_raw)
+                        .enabled_extension_names(&instance_extensions_raw),
+                )
+                .cast(),
+            )
+            .expect("XR error creating Vulkan instance")
+            .map_err(vk::Result::from_raw)
+            .expect("Vulkan error creating Vulkan instance");
+
+        Instance::from_handle(
+            library.clone(),
+            ash::vk::Instance::from_raw(vk_instance as _),
+            InstanceCreateInfo {
+                application_version: Version::major_minor(0, 0),
+                engine_version: Version::major_minor(0, 0),
+                max_api_version: Some(Version::V1_3),
+                enabled_extensions: instance_extensions,
+                ..Default::default()
+            },
+        )
+    };
+
+    let physical_device = unsafe {
+        PhysicalDevice::from_handle(
+            instance.clone(),
+            vk::PhysicalDevice::from_raw(
+                xr_instance.vulkan_graphics_device(system, instance.handle().as_raw() as _)? as _,
+            ),
+        )
+    }?;
+
+    let vk_device_properties = physical_device.properties();
+    assert!(
+        (vk_device_properties.api_version >= target_version),
+        "Vulkan physical device doesn't support Vulkan {target_version}"
+    );
+
+    log::info!(
+        "Using vkPhysicalDevice: {}",
+        physical_device.properties().device_name,
+    );
+
+    let queue_families = try_all_queue_families(physical_device.as_ref())
+        .expect("vkPhysicalDevice does not have a GRAPHICS / TRANSFER queue.");
+
+    let mut device_extensions = DeviceExtensions::empty();
+    let dmabuf_extensions = get_dmabuf_extensions();
+
+    if physical_device
+        .supported_extensions()
+        .contains(&dmabuf_extensions)
+    {
+        device_extensions = device_extensions.union(&dmabuf_extensions);
+        device_extensions.ext_image_drm_format_modifier = physical_device
+            .supported_extensions()
+            .ext_image_drm_format_modifier;
+    }
+
+    let device_extensions_raw = device_extensions
+        .into_iter()
+        .filter_map(|(name, enabled)| {
+            if enabled {
+                Some(ffi::CString::new(name).unwrap().into_raw().cast_const())
+            // want panic
+            } else {
+                None
+            }
+        })
+        .collect::<Vec<_>>();
+
+    let features = DeviceFeatures {
+        dynamic_rendering: true,
+        ..Default::default()
+    };
+
+    let queue_create_infos = queue_families
+        .iter()
+        .map(|fam| {
+            vk::DeviceQueueCreateInfo::default()
+                .queue_family_index(fam.queue_family_index)
+                .queue_priorities(&fam.priorities)
+        })
+        .collect::<Vec<_>>();
+
+    let mut device_create_info = vk::DeviceCreateInfo::default()
+        .queue_create_infos(&queue_create_infos)
+        .enabled_extension_names(&device_extensions_raw);
+
+    let mut dynamic_rendering =
+        vk::PhysicalDeviceDynamicRenderingFeatures::default().dynamic_rendering(true);
+
+    dynamic_rendering.p_next = device_create_info.p_next.cast_mut();
+    device_create_info.p_next = &raw mut dynamic_rendering as *const c_void;
+
+    let (device, queues) = unsafe {
+        let vk_device = xr_instance
+            .create_vulkan_device(
+                system,
+                get_instance_proc_addr,
+                physical_device.handle().as_raw() as _,
+                (&raw const device_create_info).cast(),
+            )
+            .expect("XR error creating Vulkan device")
+            .map_err(vk::Result::from_raw)
+            .expect("Vulkan error creating Vulkan device");
+
+        vulkano::device::Device::from_handle(
+            physical_device,
+            vk::Device::from_raw(vk_device as _),
+            DeviceCreateInfo {
+                queue_create_infos: queue_families
+                    .iter()
+                    .map(|fam| QueueCreateInfo {
+                        queue_family_index: fam.queue_family_index,
+                        queues: fam.priorities.clone(),
+                        ..Default::default()
+                    })
+                    .collect::<Vec<_>>(),
+                enabled_extensions: device_extensions,
+                enabled_features: features,
+                ..Default::default()
+            },
+        )
+    };
+
+    log::debug!(
+        "  DMA-buf supported: {}",
+        device.enabled_extensions().ext_external_memory_dma_buf
+    );
+    log::debug!(
+        "  DRM format modifiers supported: {}",
+        device.enabled_extensions().ext_image_drm_format_modifier
+    );
+
+    // Drop the CStrings
+    device_extensions_raw
+        .into_iter()
+        .for_each(|c_string| unsafe {
+            let _ = CString::from_raw(c_string.cast_mut());
+        });
+
+    let (queue_gfx, queue_xfer, queue_capture) = unwrap_queues(queues.collect());
+
+    let gfx = WGfx::new_from_raw(
+        instance,
+        device,
+        queue_gfx,
+        queue_xfer,
+        Format::R8G8B8A8_SRGB,
+    );
+    let extras = WGfxExtras::new(&gfx, queue_capture)?;
+
+    Ok((gfx, extras))
+}
+
+#[allow(clippy::too_many_lines)]
+#[cfg(feature = "openvr")]
+pub fn init_openvr_graphics(
+    mut vk_instance_extensions: InstanceExtensions,
+    mut vk_device_extensions_fn: impl FnMut(&PhysicalDevice) -> DeviceExtensions,
+) -> anyhow::Result<(Arc<WGfx>, WGfxExtras)> {
+    //#[cfg(debug_assertions)]
+    //let layers = vec!["VK_LAYER_KHRONOS_validation".to_owned()];
+    //#[cfg(not(debug_assertions))]
+
+    let layers = vec![];
+
+    log::debug!("Instance exts for runtime: {:?}", &vk_instance_extensions);
+
+    vk_instance_extensions.khr_get_physical_device_properties2 = true;
+
+    let instance = Instance::new(
+        get_vulkan_library().clone(),
+        InstanceCreateInfo {
+            flags: InstanceCreateFlags::ENUMERATE_PORTABILITY,
+            enabled_extensions: vk_instance_extensions,
+            enabled_layers: layers,
+            ..Default::default()
+        },
+    )?;
+
+    let dmabuf_extensions = get_dmabuf_extensions();
+
+    let (physical_device, my_extensions, queue_families) = instance
+        .enumerate_physical_devices()?
+        .filter_map(|p| {
+            let mut my_extensions = vk_device_extensions_fn(&p);
+
+            if !p.supported_extensions().contains(&my_extensions) {
+                log::debug!(
+                    "Not using {} due to missing extensions:",
+                    p.properties().device_name,
+                );
+                for (ext, missing) in p.supported_extensions().difference(&my_extensions) {
+                    if missing {
+                        log::debug!("  {ext}");
+                    }
+                }
+                return None;
+            }
+
+            if p.supported_extensions().contains(&dmabuf_extensions) {
+                my_extensions = my_extensions.union(&dmabuf_extensions);
+                my_extensions.ext_image_drm_format_modifier =
+                    p.supported_extensions().ext_image_drm_format_modifier;
+            }
+
+            if p.supported_extensions().ext_filter_cubic {
+                my_extensions.ext_filter_cubic = true;
+            }
+
+            log::debug!(
+                "Device exts for {}: {:?}",
+                p.properties().device_name,
+                &my_extensions
+            );
+            Some((p, my_extensions))
+        })
+        .filter_map(|(p, my_extensions)| {
+            try_all_queue_families(p.as_ref()).map(|families| (p, my_extensions, families))
+        })
+        .min_by_key(|(p, _, families)| prio_from_device_type(p) * 10 + prio_from_families(families))
+        .expect("no suitable physical device found");
+
+    log::info!(
+        "Using vkPhysicalDevice: {}",
+        physical_device.properties().device_name,
+    );
+
+    let (device, queues) = Device::new(
+        physical_device,
+        DeviceCreateInfo {
+            enabled_extensions: my_extensions,
+            enabled_features: DeviceFeatures {
+                dynamic_rendering: true,
+                ..DeviceFeatures::empty()
+            },
+            queue_create_infos: queue_families
+                .iter()
+                .map(|fam| QueueCreateInfo {
+                    queue_family_index: fam.queue_family_index,
+                    queues: fam.priorities.clone(),
+                    ..Default::default()
+                })
+                .collect::<Vec<_>>(),
+            ..Default::default()
+        },
+    )?;
+
+    log::debug!(
+        "  DMA-buf supported: {}",
+        device.enabled_extensions().ext_external_memory_dma_buf
+    );
+    log::debug!(
+        "  DRM format modifiers supported: {}",
+        device.enabled_extensions().ext_image_drm_format_modifier
+    );
+
+    let (queue_gfx, queue_xfer, queue_capture) = unwrap_queues(queues.collect());
+
+    let gfx = WGfx::new_from_raw(
+        instance,
+        device,
+        queue_gfx,
+        queue_xfer,
+        Format::R8G8B8A8_SRGB,
+    );
+    let extras = WGfxExtras::new(&gfx, queue_capture)?;
+
+    Ok((gfx, extras))
+}
+
+pub fn upload_quad_vertices(
+    buf: &mut Subbuffer<[Vert2Uv]>,
+    width: f32,
+    height: f32,
+    x: f32,
+    y: f32,
+    w: f32,
+    h: f32,
+) -> anyhow::Result<()> {
+    let rw = width;
+    let rh = height;
+
+    let x0 = x / rw;
+    let y0 = y / rh;
+
+    let x1 = w / rw + x0;
+    let y1 = h / rh + y0;
+
+    let data = [
+        Vert2Uv {
+            in_pos: [x0, y0],
+            in_uv: [0.0, 0.0],
+        },
+        Vert2Uv {
+            in_pos: [x0, y1],
+            in_uv: [0.0, 1.0],
+        },
+        Vert2Uv {
+            in_pos: [x1, y0],
+            in_uv: [1.0, 0.0],
+        },
+        Vert2Uv {
+            in_pos: [x1, y1],
+            in_uv: [1.0, 1.0],
+        },
+    ];
+
+    buf.write()?[0..4].copy_from_slice(&data);
+    Ok(())
+}
+
+#[derive(Debug)]
+struct QueueFamilyLayout {
+    queue_family_index: u32,
+    priorities: Vec<f32>,
+}
+
+fn prio_from_device_type(physical_device: &PhysicalDevice) -> u32 {
+    match physical_device.properties().device_type {
+        PhysicalDeviceType::DiscreteGpu => 0,
+        PhysicalDeviceType::IntegratedGpu => 1,
+        PhysicalDeviceType::VirtualGpu => 2,
+        PhysicalDeviceType::Cpu => 3,
+        _ => 4,
+    }
+}
+
+const fn prio_from_families(families: &[QueueFamilyLayout]) -> u32 {
+    match families.len() {
+        2 | 3 => 0,
+        _ => 1,
+    }
+}
+
+fn unwrap_queues(queues: Vec<Arc<Queue>>) -> (Arc<Queue>, Arc<Queue>, Option<Arc<Queue>>) {
+    match queues[..] {
+        [ref g, ref t, ref c] => (g.clone(), t.clone(), Some(c.clone())),
+        [ref gt, ref c] => (gt.clone(), gt.clone(), Some(c.clone())),
+        [ref gt] => (gt.clone(), gt.clone(), None),
+        _ => unreachable!(),
+    }
+}
+
+fn try_all_queue_families(physical_device: &PhysicalDevice) -> Option<Vec<QueueFamilyLayout>> {
+    queue_families_priorities(
+        physical_device,
+        vec![
+            // main-thread graphics + uploads
+            QueueFlags::GRAPHICS | QueueFlags::TRANSFER,
+            // capture-thread uploads
+            QueueFlags::TRANSFER,
+        ],
+    )
+    .or_else(|| {
+        queue_families_priorities(
+            physical_device,
+            vec![
+                // main thread graphics
+                QueueFlags::GRAPHICS,
+                // main thread uploads
+                QueueFlags::TRANSFER,
+                // capture thread uploads
+                QueueFlags::TRANSFER,
+            ],
+        )
+    })
+    .or_else(|| {
+        queue_families_priorities(
+            physical_device,
+            // main thread-only. software capture not supported.
+            vec![QueueFlags::GRAPHICS | QueueFlags::TRANSFER],
+        )
+    })
+}
+
+fn queue_families_priorities(
+    physical_device: &PhysicalDevice,
+    mut requested_queues: Vec<QueueFlags>,
+) -> Option<Vec<QueueFamilyLayout>> {
+    let mut result = Vec::with_capacity(3);
+
+    for (idx, props) in physical_device.queue_family_properties().iter().enumerate() {
+        let mut remaining = props.queue_count;
+        let mut want = 0usize;
+
+        requested_queues.retain(|requested| {
+            if props.queue_flags.intersects(*requested) && remaining > 0 {
+                remaining -= 1;
+                want += 1;
+                false
+            } else {
+                true
+            }
+        });
+
+        if want > 0 {
+            result.push(QueueFamilyLayout {
+                queue_family_index: idx as u32,
+                priorities: std::iter::repeat_n(1.0, want).collect(),
+            });
+        }
+    }
+
+    if requested_queues.is_empty() {
+        log::debug!("Selected GPU queue families: {result:?}");
+        Some(result)
+    } else {
+        None
+    }
+}
+
+#[derive(Default)]
+pub struct CommandBuffers {
+    inner: Vec<Arc<PrimaryAutoCommandBuffer>>,
+}
+
+impl CommandBuffers {
+    pub fn push(&mut self, buffer: Arc<PrimaryAutoCommandBuffer>) {
+        self.inner.push(buffer);
+    }
+    pub fn execute_now(self, queue: Arc<Queue>) -> anyhow::Result<Option<Box<dyn GpuFuture>>> {
+        let mut buffers = self.inner.into_iter();
+        let Some(first) = buffers.next() else {
+            return Ok(None);
+        };
+
+        let future = first.execute(queue)?;
+        let mut future: Box<dyn GpuFuture> = Box::new(future);
+
+        for buf in buffers {
+            future = Box::new(future.then_execute_same_queue(buf)?);
+        }
+
+        Ok(Some(future))
+    }
+}
+
+pub trait ExtentExt {
+    fn extent_f32(&self) -> [f32; 2];
+    fn extent_vec2(&self) -> Vec2;
+    fn extent_u32arr(&self) -> [u32; 2];
+}
+
+impl ExtentExt for Arc<ImageView> {
+    fn extent_f32(&self) -> [f32; 2] {
+        let [w, h, _] = self.image().extent();
+        [w as _, h as _]
+    }
+    fn extent_vec2(&self) -> Vec2 {
+        let [w, h, _] = self.image().extent();
+        vec2(w as _, h as _)
+    }
+    fn extent_u32arr(&self) -> [u32; 2] {
+        let [w, h, _] = self.image().extent();
+        [w, h]
+    }
+}

wlx-overlay-s/src/gui/asset.rs

@@ -0,0 +1,12 @@
+#[derive(rust_embed::Embed)]
+#[folder = "src/gui/assets/"]
+pub struct GuiAsset;
+
+impl wgui::assets::AssetProvider for GuiAsset {
+    fn load_from_path(&mut self, path: &str) -> anyhow::Result<Vec<u8>> {
+        match GuiAsset::get(path) {
+            Some(data) => Ok(data.data.to_vec()),
+            None => anyhow::bail!("embedded file {} not found", path),
+        }
+    }
+}

wlx-overlay-s/src/gui/mod.rs

@@ -0,0 +1,3 @@
+mod asset;
+pub mod panel;
+mod timestep;

wlx-overlay-s/src/gui/panel.rs

@@ -0,0 +1,161 @@
+use std::sync::Arc;
+
+use glam::vec2;
+use vulkano::{command_buffer::CommandBufferUsage, image::view::ImageView};
+use wgui::{
+    event::{Event as WguiEvent, MouseDownEvent, MouseMotionEvent, MouseUpEvent, MouseWheelEvent},
+    layout::Layout,
+    renderer_vk::context::Context as WguiContext,
+};
+
+use crate::{
+    backend::{
+        input::{Haptics, InteractionHandler, PointerHit},
+        overlay::{FrameMeta, OverlayBackend, OverlayRenderer, ShouldRender},
+    },
+    graphics::{CommandBuffers, ExtentExt},
+    state::AppState,
+};
+
+use super::{asset::GuiAsset, timestep::Timestep};
+
+pub struct GuiPanel {
+    pub layout: Layout,
+    context: WguiContext,
+    timestep: Timestep,
+    pub width: u32,
+    pub height: u32,
+}
+
+impl GuiPanel {
+    pub fn new_from_template(
+        app: &AppState,
+        width: u32,
+        height: u32,
+        path: &str,
+    ) -> anyhow::Result<Self> {
+        let mut me = Self::new_blank(app, width, height)?;
+
+        let parent = me.layout.root_widget;
+        let _res = wgui::parser::parse_from_assets(&mut me.layout, parent, path)?;
+
+        Ok(me)
+    }
+
+    pub fn new_blank(app: &AppState, width: u32, height: u32) -> anyhow::Result<Self> {
+        let layout = Layout::new(Box::new(GuiAsset {}))?;
+        let context = WguiContext::new(app.gfx.clone(), app.gfx.surface_format, 1.0)?;
+        let mut timestep = Timestep::new();
+        timestep.set_tps(60.0);
+
+        Ok(Self {
+            layout,
+            context,
+            timestep,
+            width,
+            height,
+        })
+    }
+}
+
+impl OverlayBackend for GuiPanel {
+    fn set_renderer(&mut self, _: Box<dyn OverlayRenderer>) {
+        log::debug!("Attempted to replace renderer on GuiPanel!");
+    }
+    fn set_interaction(&mut self, _: Box<dyn InteractionHandler>) {
+        log::debug!("Attempted to replace interaction layer on GuiPanel!");
+    }
+}
+
+impl InteractionHandler for GuiPanel {
+    fn on_scroll(&mut self, _app: &mut AppState, hit: &PointerHit, delta_y: f32, delta_x: f32) {
+        self.layout
+            .push_event(&WguiEvent::MouseWheel(MouseWheelEvent {
+                shift: vec2(delta_x, delta_y),
+                pos: hit.uv,
+            }))
+            .unwrap()
+    }
+
+    fn on_hover(&mut self, _app: &mut AppState, hit: &PointerHit) -> Option<Haptics> {
+        self.layout
+            .push_event(&WguiEvent::MouseMotion(MouseMotionEvent { pos: hit.uv }))
+            .unwrap();
+
+        None
+    }
+
+    fn on_left(&mut self, _app: &mut AppState, _pointer: usize) {
+        //TODO: is this needed?
+    }
+
+    fn on_pointer(&mut self, _app: &mut AppState, hit: &PointerHit, pressed: bool) {
+        if pressed {
+            self.layout
+                .push_event(&WguiEvent::MouseDown(MouseDownEvent { pos: hit.uv }))
+                .unwrap();
+        } else {
+            self.layout
+                .push_event(&WguiEvent::MouseUp(MouseUpEvent { pos: hit.uv }))
+                .unwrap();
+        }
+    }
+}
+
+impl OverlayRenderer for GuiPanel {
+    fn init(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+
+    fn pause(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+
+    fn resume(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        self.timestep.reset();
+        Ok(())
+    }
+
+    fn should_render(&mut self, _app: &mut AppState) -> anyhow::Result<ShouldRender> {
+        while self.timestep.on_tick() {
+            self.layout.tick()?;
+        }
+
+        Ok(if self.layout.check_toggle_needs_redraw() {
+            ShouldRender::Should
+        } else {
+            ShouldRender::Can
+        })
+    }
+
+    fn render(
+        &mut self,
+        app: &mut AppState,
+        tgt: Arc<ImageView>,
+        buf: &mut CommandBuffers,
+        _alpha: f32,
+    ) -> anyhow::Result<bool> {
+        self.context.update_viewport(tgt.extent_u32arr(), 1.0)?;
+        self.layout.update(tgt.extent_vec2(), self.timestep.alpha);
+
+        let mut cmd_buf = app
+            .gfx
+            .create_gfx_command_buffer(CommandBufferUsage::OneTimeSubmit)
+            .unwrap();
+
+        cmd_buf.begin_rendering(tgt)?;
+        let primitives = wgui::drawing::draw(&self.layout)?;
+        self.context.draw(&app.gfx, &mut cmd_buf, &primitives)?;
+        cmd_buf.end_rendering()?;
+        buf.push(cmd_buf.build()?);
+
+        Ok(true)
+    }
+
+    fn frame_meta(&mut self) -> Option<FrameMeta> {
+        Some(FrameMeta {
+            extent: [self.width, self.height, 1],
+            ..Default::default()
+        })
+    }
+}

wlx-overlay-s/src/gui/timestep.rs

@@ -0,0 +1,70 @@
+use std::{sync::LazyLock, time::Instant};
+static TIME_START: LazyLock<Instant> = LazyLock::new(Instant::now);
+
+pub fn get_micros() -> u64 {
+    TIME_START.elapsed().as_micros() as u64
+}
+
+#[derive(Default)]
+pub struct Timestep {
+    current_time_us: u64,
+    accumulator: f32,
+    time_micros: u64,
+    ticks: u32,
+    speed: f32,
+    pub alpha: f32,
+    delta: f32,
+    loopnum: u8,
+}
+
+impl Timestep {
+    pub fn new() -> Timestep {
+        let mut timestep = Timestep {
+            speed: 1.0,
+            ..Default::default()
+        };
+
+        timestep.reset();
+        timestep
+    }
+
+    fn calculate_alpha(&mut self) {
+        self.alpha = (self.accumulator / self.delta).clamp(0.0, 1.0);
+    }
+
+    pub fn set_tps(&mut self, tps: f32) {
+        self.delta = 1000.0 / tps;
+    }
+
+    pub fn reset(&mut self) {
+        self.current_time_us = get_micros();
+        self.accumulator = 0.0;
+    }
+
+    pub fn on_tick(&mut self) -> bool {
+        let newtime = get_micros();
+        let frametime = newtime - self.current_time_us;
+        self.time_micros += frametime;
+        self.current_time_us = newtime;
+        self.accumulator += frametime as f32 * self.speed / 1000.0;
+        self.calculate_alpha();
+
+        if self.accumulator >= self.delta {
+            self.accumulator -= self.delta;
+            self.loopnum += 1;
+            self.ticks += 1;
+
+            if self.loopnum > 5 {
+                // cannot keep up!
+                self.loopnum = 0;
+                self.accumulator = 0.0;
+                return false;
+            }
+
+            true
+        } else {
+            self.loopnum = 0;
+            false
+        }
+    }
+}

wlx-overlay-s/src/hid/mod.rs

@@ -0,0 +1,604 @@
+use glam::{IVec2, Vec2};
+use idmap::{idmap, IdMap};
+use idmap_derive::IntegerId;
+use input_linux::{
+    AbsoluteAxis, AbsoluteInfo, AbsoluteInfoSetup, EventKind, InputId, Key, RelativeAxis,
+    UInputHandle,
+};
+use libc::{input_event, timeval};
+use serde::Deserialize;
+use std::mem::transmute;
+use std::sync::LazyLock;
+use std::{fs::File, sync::atomic::AtomicBool};
+use strum::{EnumIter, EnumString, IntoEnumIterator};
+use xkbcommon::xkb;
+
+#[cfg(feature = "wayland")]
+mod wayland;
+
+#[cfg(feature = "x11")]
+mod x11;
+
+pub static USE_UINPUT: AtomicBool = AtomicBool::new(true);
+
+pub fn initialize() -> Box<dyn HidProvider> {
+    if !USE_UINPUT.load(std::sync::atomic::Ordering::Relaxed) {
+        log::info!("Uinput disabled by user.");
+        return Box::new(DummyProvider {});
+    }
+
+    if let Some(uinput) = UInputProvider::try_new() {
+        log::info!("Initialized uinput.");
+        return Box::new(uinput);
+    }
+    log::error!("Could not create uinput provider. Keyboard/Mouse input will not work!");
+    log::error!("To check if you're in input group, run: id -nG");
+    if let Ok(user) = std::env::var("USER") {
+        log::error!("To add yourself to the input group, run: sudo usermod -aG input {user}");
+        log::error!("After adding yourself to the input group, you will need to reboot.");
+    }
+    Box::new(DummyProvider {})
+}
+
+pub trait HidProvider {
+    fn mouse_move(&mut self, pos: Vec2);
+    fn send_button(&mut self, button: u16, down: bool);
+    fn wheel(&mut self, delta_y: i32, delta_x: i32);
+    fn set_modifiers(&mut self, mods: u8);
+    fn send_key(&self, key: VirtualKey, down: bool);
+    fn set_desktop_extent(&mut self, extent: Vec2);
+    fn set_desktop_origin(&mut self, origin: Vec2);
+    fn commit(&mut self);
+}
+
+struct MouseButtonAction {
+    button: u16,
+    down: bool,
+}
+
+#[derive(Default)]
+struct MouseAction {
+    last_requested_pos: Option<Vec2>,
+    pos: Option<Vec2>,
+    button: Option<MouseButtonAction>,
+    scroll: Option<IVec2>,
+}
+
+pub struct UInputProvider {
+    keyboard_handle: UInputHandle<File>,
+    mouse_handle: UInputHandle<File>,
+    desktop_extent: Vec2,
+    desktop_origin: Vec2,
+    cur_modifiers: u8,
+    current_action: MouseAction,
+}
+
+pub struct DummyProvider;
+
+pub const MOUSE_LEFT: u16 = 0x110;
+pub const MOUSE_RIGHT: u16 = 0x111;
+pub const MOUSE_MIDDLE: u16 = 0x112;
+
+const MOUSE_EXTENT: f32 = 32768.;
+
+const EV_SYN: u16 = 0x0;
+const EV_KEY: u16 = 0x1;
+const EV_REL: u16 = 0x2;
+const EV_ABS: u16 = 0x3;
+
+impl UInputProvider {
+    fn try_new() -> Option<Self> {
+        let keyboard_file = File::create("/dev/uinput").ok()?;
+        let keyboard_handle = UInputHandle::new(keyboard_file);
+
+        let mouse_file = File::create("/dev/uinput").ok()?;
+        let mouse_handle = UInputHandle::new(mouse_file);
+
+        let kbd_id = InputId {
+            bustype: 0x03,
+            vendor: 0x4711,
+            product: 0x0829,
+            version: 5,
+        };
+        let mouse_id = InputId {
+            bustype: 0x03,
+            vendor: 0x4711,
+            product: 0x0830,
+            version: 5,
+        };
+        let kbd_name = b"WlxOverlay-S Keyboard\0";
+        let mouse_name = b"WlxOverlay-S Mouse\0";
+
+        let abs_info = vec![
+            AbsoluteInfoSetup {
+                axis: input_linux::AbsoluteAxis::X,
+                info: AbsoluteInfo {
+                    value: 0,
+                    minimum: 0,
+                    maximum: MOUSE_EXTENT as _,
+                    fuzz: 0,
+                    flat: 0,
+                    resolution: 10,
+                },
+            },
+            AbsoluteInfoSetup {
+                axis: input_linux::AbsoluteAxis::Y,
+                info: AbsoluteInfo {
+                    value: 0,
+                    minimum: 0,
+                    maximum: MOUSE_EXTENT as _,
+                    fuzz: 0,
+                    flat: 0,
+                    resolution: 10,
+                },
+            },
+        ];
+
+        keyboard_handle.set_evbit(EventKind::Key).ok()?;
+        for key in VirtualKey::iter() {
+            let mapped_key: Key = unsafe { std::mem::transmute((key as u16) - 8) };
+            keyboard_handle.set_keybit(mapped_key).ok()?;
+        }
+
+        keyboard_handle.create(&kbd_id, kbd_name, 0, &[]).ok()?;
+
+        mouse_handle.set_evbit(EventKind::Absolute).ok()?;
+        mouse_handle.set_evbit(EventKind::Relative).ok()?;
+        mouse_handle.set_absbit(AbsoluteAxis::X).ok()?;
+        mouse_handle.set_absbit(AbsoluteAxis::Y).ok()?;
+        mouse_handle.set_relbit(RelativeAxis::WheelHiRes).ok()?;
+        mouse_handle
+            .set_relbit(RelativeAxis::HorizontalWheelHiRes)
+            .ok()?;
+        mouse_handle.set_evbit(EventKind::Key).ok()?;
+
+        for btn in MOUSE_LEFT..=MOUSE_MIDDLE {
+            let mouse_btn: Key = unsafe { transmute(btn) };
+            mouse_handle.set_keybit(mouse_btn).ok()?;
+        }
+        mouse_handle
+            .create(&mouse_id, mouse_name, 0, &abs_info)
+            .ok()?;
+
+        Some(Self {
+            keyboard_handle,
+            mouse_handle,
+            desktop_extent: Vec2::ZERO,
+            desktop_origin: Vec2::ZERO,
+            current_action: MouseAction::default(),
+            cur_modifiers: 0,
+        })
+    }
+    fn send_button_internal(&self, button: u16, down: bool) {
+        let time = get_time();
+        let events = [
+            new_event(time, EV_KEY, button, down.into()),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.mouse_handle.write(&events) {
+            log::error!("send_button: {res}");
+        }
+    }
+    fn mouse_move_internal(&mut self, pos: Vec2) {
+        #[cfg(debug_assertions)]
+        log::trace!("Mouse move: {pos:?}");
+
+        let pos = (pos - self.desktop_origin) * (MOUSE_EXTENT / self.desktop_extent);
+
+        let time = get_time();
+        let events = [
+            new_event(time, EV_ABS, AbsoluteAxis::X as _, pos.x as i32),
+            new_event(time, EV_ABS, AbsoluteAxis::Y as _, pos.y as i32),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.mouse_handle.write(&events) {
+            log::error!("{res}");
+        }
+    }
+    fn wheel_internal(&self, delta_y: i32, delta_x: i32) {
+        let time = get_time();
+        let events = [
+            new_event(time, EV_REL, RelativeAxis::WheelHiRes as _, delta_y),
+            new_event(
+                time,
+                EV_REL,
+                RelativeAxis::HorizontalWheelHiRes as _,
+                delta_x,
+            ),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.mouse_handle.write(&events) {
+            log::error!("wheel: {res}");
+        }
+    }
+}
+
+impl HidProvider for UInputProvider {
+    fn set_modifiers(&mut self, modifiers: u8) {
+        let changed = self.cur_modifiers ^ modifiers;
+        for i in 0..8 {
+            let m = 1 << i;
+            if changed & m != 0 {
+                if let Some(vk) = MODS_TO_KEYS.get(m).into_iter().flatten().next() {
+                    self.send_key(*vk, modifiers & m != 0);
+                }
+            }
+        }
+        self.cur_modifiers = modifiers;
+    }
+    fn send_key(&self, key: VirtualKey, down: bool) {
+        #[cfg(debug_assertions)]
+        log::trace!("send_key: {key:?} {down}");
+
+        let time = get_time();
+        let events = [
+            new_event(time, EV_KEY, (key as u16) - 8, down.into()),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.keyboard_handle.write(&events) {
+            log::error!("send_key: {res}");
+        }
+    }
+    fn set_desktop_extent(&mut self, extent: Vec2) {
+        self.desktop_extent = extent;
+    }
+    fn set_desktop_origin(&mut self, origin: Vec2) {
+        self.desktop_origin = origin;
+    }
+    fn mouse_move(&mut self, pos: Vec2) {
+        if self.current_action.pos.is_none() && self.current_action.scroll.is_none() {
+            self.current_action.pos = Some(pos);
+        }
+        self.current_action.last_requested_pos = Some(pos);
+    }
+    fn send_button(&mut self, button: u16, down: bool) {
+        if self.current_action.button.is_none() {
+            self.current_action.button = Some(MouseButtonAction { button, down });
+            self.current_action.pos = self.current_action.last_requested_pos;
+        }
+    }
+    fn wheel(&mut self, delta_y: i32, delta_x: i32) {
+        if self.current_action.scroll.is_none() {
+            self.current_action.scroll = Some(IVec2::new(delta_x, delta_y));
+            // Pass mouse motion events only if not scrolling
+            // (allows scrolling on all Chromium-based applications)
+            self.current_action.pos = None;
+        }
+    }
+    fn commit(&mut self) {
+        if let Some(pos) = self.current_action.pos.take() {
+            self.mouse_move_internal(pos);
+        }
+        if let Some(button) = self.current_action.button.take() {
+            self.send_button_internal(button.button, button.down);
+        }
+        if let Some(scroll) = self.current_action.scroll.take() {
+            self.wheel_internal(scroll.y, scroll.x);
+        }
+    }
+}
+
+impl HidProvider for DummyProvider {
+    fn mouse_move(&mut self, _pos: Vec2) {}
+    fn send_button(&mut self, _button: u16, _down: bool) {}
+    fn wheel(&mut self, _delta_y: i32, _delta_x: i32) {}
+    fn set_modifiers(&mut self, _modifiers: u8) {}
+    fn send_key(&self, _key: VirtualKey, _down: bool) {}
+    fn set_desktop_extent(&mut self, _extent: Vec2) {}
+    fn set_desktop_origin(&mut self, _origin: Vec2) {}
+    fn commit(&mut self) {}
+}
+
+#[inline]
+fn get_time() -> timeval {
+    let mut time = timeval {
+        tv_sec: 0,
+        tv_usec: 0,
+    };
+    unsafe { libc::gettimeofday(&mut time, std::ptr::null_mut()) };
+    time
+}
+
+#[inline]
+const fn new_event(time: timeval, type_: u16, code: u16, value: i32) -> input_event {
+    input_event {
+        time,
+        type_,
+        code,
+        value,
+    }
+}
+
+pub type KeyModifier = u8;
+pub const SHIFT: KeyModifier = 0x01;
+pub const CAPS_LOCK: KeyModifier = 0x02;
+pub const CTRL: KeyModifier = 0x04;
+pub const ALT: KeyModifier = 0x08;
+pub const NUM_LOCK: KeyModifier = 0x10;
+pub const SUPER: KeyModifier = 0x40;
+pub const META: KeyModifier = 0x80;
+
+#[allow(non_camel_case_types)]
+#[repr(u16)]
+#[derive(Debug, Deserialize, PartialEq, Eq, Clone, Copy, IntegerId, EnumString, EnumIter)]
+pub enum VirtualKey {
+    Escape = 9,
+    N1, // number row
+    N2,
+    N3,
+    N4,
+    N5,
+    N6,
+    N7,
+    N8,
+    N9,
+    N0,
+    Minus,
+    Plus,
+    BackSpace,
+    Tab,
+    Q,
+    W,
+    E,
+    R,
+    T,
+    Y,
+    U,
+    I,
+    O,
+    P,
+    Oem4, // [ {
+    Oem6, // ] }
+    Return,
+    LCtrl,
+    A,
+    S,
+    D,
+    F,
+    G,
+    H,
+    J,
+    K,
+    L,
+    Oem1, // ; :
+    Oem7, // ' "
+    Oem3, // ` ~
+    LShift,
+    Oem5, // \ |
+    Z,
+    X,
+    C,
+    V,
+    B,
+    N,
+    M,
+    Comma,  // , <
+    Period, // . >
+    Oem2,   // / ?
+    RShift,
+    KP_Multiply,
+    LAlt,
+    Space,
+    Caps,
+    F1,
+    F2,
+    F3,
+    F4,
+    F5,
+    F6,
+    F7,
+    F8,
+    F9,
+    F10,
+    NumLock,
+    Scroll,
+    KP_7, // KeyPad
+    KP_8,
+    KP_9,
+    KP_Subtract,
+    KP_4,
+    KP_5,
+    KP_6,
+    KP_Add,
+    KP_1,
+    KP_2,
+    KP_3,
+    KP_0,
+    KP_Decimal,
+    Oem102 = 94, // Optional key usually between LShift and Z
+    F11,
+    F12,
+    AbntC1,
+    Katakana,
+    Hiragana,
+    Henkan,
+    Kana,
+    Muhenkan,
+    KP_Enter = 104,
+    RCtrl,
+    KP_Divide,
+    Print,
+    Meta, // Right Alt aka AltGr
+    Home = 110,
+    Up,
+    Prior,
+    Left,
+    Right,
+    End,
+    Down,
+    Next,
+    Insert,
+    Delete,
+    XF86AudioMute = 121,
+    XF86AudioLowerVolume,
+    XF86AudioRaiseVolume,
+    Pause = 127,
+    AbntC2 = 129,
+    Hangul,
+    Hanja,
+    LSuper = 133,
+    RSuper,
+    Menu,
+    Help = 146,
+    XF86MenuKB,
+    XF86Sleep = 150,
+    XF86Xfer = 155,
+    XF86Launch1,
+    XF86Launch2,
+    XF86WWW,
+    XF86Mail = 163,
+    XF86Favorites,
+    XF86MyComputer,
+    XF86Back,
+    XF86Forward,
+    XF86AudioNext = 171,
+    XF86AudioPlay,
+    XF86AudioPrev,
+    XF86AudioStop,
+    XF86HomePage = 180,
+    XF86Reload,
+    F13 = 191,
+    F14,
+    F15,
+    F16,
+    F17,
+    F18,
+    F19,
+    F20,
+    F21,
+    F22,
+    F23,
+    F24,
+    Hyper = 207,
+    XF86Launch3,
+    XF86Launch4,
+    XF86LaunchB,
+    XF86Search = 225,
+}
+
+pub static KEYS_TO_MODS: LazyLock<IdMap<VirtualKey, KeyModifier>> = LazyLock::new(|| {
+    idmap! {
+        VirtualKey::LShift => SHIFT,
+        VirtualKey::RShift => SHIFT,
+        VirtualKey::Caps => CAPS_LOCK,
+        VirtualKey::LCtrl => CTRL,
+        VirtualKey::RCtrl => CTRL,
+        VirtualKey::LAlt => ALT,
+        VirtualKey::NumLock => NUM_LOCK,
+        VirtualKey::LSuper => SUPER,
+        VirtualKey::RSuper => SUPER,
+        VirtualKey::Meta => META,
+    }
+});
+
+pub static MODS_TO_KEYS: LazyLock<IdMap<KeyModifier, Vec<VirtualKey>>> = LazyLock::new(|| {
+    idmap! {
+        SHIFT => vec![VirtualKey::LShift, VirtualKey::RShift],
+        CAPS_LOCK => vec![VirtualKey::Caps],
+        CTRL => vec![VirtualKey::LCtrl, VirtualKey::RCtrl],
+        ALT => vec![VirtualKey::LAlt],
+        NUM_LOCK => vec![VirtualKey::NumLock],
+        SUPER => vec![VirtualKey::LSuper, VirtualKey::RSuper],
+        META => vec![VirtualKey::Meta],
+    }
+});
+
+pub enum KeyType {
+    Symbol,
+    NumPad,
+    Other,
+}
+
+macro_rules! key_between {
+    ($key:expr, $start:expr, $end:expr) => {
+        $key as u32 >= $start as u32 && $key as u32 <= $end as u32
+    };
+}
+
+macro_rules! key_is {
+    ($key:expr, $val:expr) => {
+        $key as u32 == $val as u32
+    };
+}
+
+pub const fn get_key_type(key: VirtualKey) -> KeyType {
+    if key_between!(key, VirtualKey::N1, VirtualKey::Plus)
+        || key_between!(key, VirtualKey::Q, VirtualKey::Oem6)
+        || key_between!(key, VirtualKey::A, VirtualKey::Oem3)
+        || key_between!(key, VirtualKey::Oem5, VirtualKey::Oem2)
+        || key_is!(key, VirtualKey::Oem102)
+    {
+        KeyType::Symbol
+    } else if key_between!(key, VirtualKey::KP_7, VirtualKey::KP_0)
+        && !key_is!(key, VirtualKey::KP_Subtract)
+        && !key_is!(key, VirtualKey::KP_Add)
+    {
+        KeyType::NumPad
+    } else {
+        KeyType::Other
+    }
+}
+
+pub struct XkbKeymap {
+    pub keymap: xkb::Keymap,
+}
+
+impl XkbKeymap {
+    pub fn label_for_key(&self, key: VirtualKey, modifier: KeyModifier) -> String {
+        let mut state = xkb::State::new(&self.keymap);
+        if modifier > 0 {
+            if let Some(mod_key) = MODS_TO_KEYS.get(modifier) {
+                state.update_key(
+                    xkb::Keycode::from(mod_key[0] as u32),
+                    xkb::KeyDirection::Down,
+                );
+            }
+        }
+        state.key_get_utf8(xkb::Keycode::from(key as u32))
+    }
+
+    pub fn has_altgr(&self) -> bool {
+        let state0 = xkb::State::new(&self.keymap);
+        let mut state1 = xkb::State::new(&self.keymap);
+        state1.update_key(
+            xkb::Keycode::from(VirtualKey::Meta as u32),
+            xkb::KeyDirection::Down,
+        );
+
+        for key in [
+            VirtualKey::N0,
+            VirtualKey::N1,
+            VirtualKey::N2,
+            VirtualKey::N3,
+            VirtualKey::N4,
+            VirtualKey::N5,
+            VirtualKey::N6,
+            VirtualKey::N7,
+            VirtualKey::N8,
+            VirtualKey::N9,
+        ] {
+            let sym0 = state0.key_get_one_sym(xkb::Keycode::from(key as u32));
+            let sym1 = state1.key_get_one_sym(xkb::Keycode::from(key as u32));
+            if sym0 != sym1 {
+                return true;
+            }
+        }
+        false
+    }
+}
+
+#[cfg(feature = "wayland")]
+pub use wayland::get_keymap_wl;
+
+#[cfg(not(feature = "wayland"))]
+pub fn get_keymap_wl() -> anyhow::Result<XkbKeymap> {
+    anyhow::bail!("Wayland support not enabled.")
+}
+
+#[cfg(feature = "x11")]
+pub use x11::get_keymap_x11;
+
+#[cfg(not(feature = "x11"))]
+pub fn get_keymap_x11() -> anyhow::Result<XkbKeymap> {
+    anyhow::bail!("X11 support not enabled.")
+}

wlx-overlay-s/src/hid/wayland.rs

@@ -0,0 +1,140 @@
+use wlx_capture::wayland::wayland_client::{
+    globals::{registry_queue_init, GlobalListContents},
+    protocol::{
+        wl_keyboard::{self, WlKeyboard},
+        wl_registry::WlRegistry,
+        wl_seat::{self, Capability, WlSeat},
+    },
+    Connection, Dispatch, Proxy, QueueHandle,
+};
+use xkbcommon::xkb;
+
+use super::XkbKeymap;
+
+struct WlKeymapHandler {
+    seat: WlSeat,
+    keyboard: Option<WlKeyboard>,
+    keymap: Option<XkbKeymap>,
+}
+
+impl Drop for WlKeymapHandler {
+    fn drop(&mut self) {
+        if let Some(keyboard) = &self.keyboard {
+            keyboard.release();
+        }
+        self.seat.release();
+    }
+}
+
+pub fn get_keymap_wl() -> anyhow::Result<XkbKeymap> {
+    let connection = Connection::connect_to_env()?;
+    let (globals, mut queue) = registry_queue_init::<WlKeymapHandler>(&connection)?;
+    let qh = queue.handle();
+    let seat: WlSeat = globals
+        .bind(&qh, 4..=9, ())
+        .expect(WlSeat::interface().name);
+
+    let mut me = WlKeymapHandler {
+        seat,
+        keyboard: None,
+        keymap: None,
+    };
+
+    // this gets us the wl_seat
+    let _ = queue.blocking_dispatch(&mut me);
+
+    // this gets us the wl_keyboard
+    let _ = queue.blocking_dispatch(&mut me);
+
+    me.keymap
+        .take()
+        .ok_or_else(|| anyhow::anyhow!("Could not load keymap"))
+}
+
+impl Dispatch<WlRegistry, GlobalListContents> for WlKeymapHandler {
+    fn event(
+        _state: &mut Self,
+        _proxy: &WlRegistry,
+        _event: <WlRegistry as Proxy>::Event,
+        _data: &GlobalListContents,
+        _conn: &Connection,
+        _qhandle: &QueueHandle<Self>,
+    ) {
+    }
+}
+
+impl Dispatch<WlSeat, ()> for WlKeymapHandler {
+    fn event(
+        state: &mut Self,
+        proxy: &WlSeat,
+        event: <WlSeat as Proxy>::Event,
+        _data: &(),
+        _conn: &Connection,
+        qhandle: &QueueHandle<Self>,
+    ) {
+        match event {
+            wl_seat::Event::Capabilities { capabilities } => {
+                let capability = capabilities
+                    .into_result()
+                    .unwrap_or(wl_seat::Capability::empty());
+                if capability.contains(Capability::Keyboard) {
+                    state.keyboard = Some(proxy.get_keyboard(qhandle, ()));
+                }
+            }
+            wl_seat::Event::Name { name } => {
+                log::debug!("Using WlSeat: {name}");
+            }
+            _ => {}
+        }
+    }
+}
+
+impl Dispatch<WlKeyboard, ()> for WlKeymapHandler {
+    fn event(
+        state: &mut Self,
+        _proxy: &WlKeyboard,
+        event: <WlKeyboard as Proxy>::Event,
+        _data: &(),
+        _conn: &Connection,
+        _qhandle: &QueueHandle<Self>,
+    ) {
+        match event {
+            wl_keyboard::Event::Keymap { format, fd, size } => {
+                let format = format
+                    .into_result()
+                    .unwrap_or(wl_keyboard::KeymapFormat::NoKeymap);
+
+                if matches!(format, wl_keyboard::KeymapFormat::XkbV1) {
+                    let context = xkb::Context::new(xkb::CONTEXT_NO_DEFAULT_INCLUDES);
+                    let maybe_keymap = unsafe {
+                        xkb::Keymap::new_from_fd(
+                            &context,
+                            fd,
+                            size as _,
+                            xkb::KEYMAP_FORMAT_TEXT_V1,
+                            xkb::KEYMAP_COMPILE_NO_FLAGS,
+                        )
+                    };
+
+                    match maybe_keymap {
+                        Ok(Some(keymap)) => {
+                            state.keymap = Some(XkbKeymap { keymap });
+                        }
+                        Ok(None) => {
+                            log::error!("Could not load keymap: no keymap");
+                            log::error!("Default layout will be used.");
+                        }
+                        Err(err) => {
+                            log::error!("Could not load keymap: {err}");
+                            log::error!("Default layout will be used.");
+                        }
+                    }
+                }
+            }
+            wl_keyboard::Event::RepeatInfo { rate, delay } => {
+                log::debug!("WlKeyboard RepeatInfo rate: {rate}, delay: {delay}");
+            }
+            _ => {}
+        }
+    }
+}

wlx-overlay-s/src/hid/x11.rs

@@ -0,0 +1,35 @@
+use xkbcommon::xkb::{
+    self,
+    x11::{
+        get_core_keyboard_device_id, keymap_new_from_device, setup_xkb_extension,
+        SetupXkbExtensionFlags, MIN_MAJOR_XKB_VERSION, MIN_MINOR_XKB_VERSION,
+    },
+};
+
+use super::XkbKeymap;
+
+pub fn get_keymap_x11() -> anyhow::Result<XkbKeymap> {
+    let context = xkb::Context::new(xkb::CONTEXT_NO_FLAGS);
+
+    let (conn, _) = xcb::Connection::connect(None)?;
+    setup_xkb_extension(
+        &conn,
+        MIN_MAJOR_XKB_VERSION,
+        MIN_MINOR_XKB_VERSION,
+        SetupXkbExtensionFlags::NoFlags,
+        &mut 0,
+        &mut 0,
+        &mut 0,
+        &mut 0,
+    );
+
+    let device_id = get_core_keyboard_device_id(&conn);
+    if device_id == -1 {
+        return Err(anyhow::anyhow!(
+            "get_core_keyboard_device_id returned -1. Check your XKB installation."
+        ));
+    }
+    let keymap = keymap_new_from_device(&context, &conn, device_id, xkb::KEYMAP_COMPILE_NO_FLAGS);
+
+    Ok(XkbKeymap { keymap })
+}

wlx-overlay-s/src/main.rs

@@ -0,0 +1,293 @@
+#![warn(clippy::all, clippy::pedantic, clippy::nursery, clippy::cargo)]
+#![allow(
+    dead_code,
+    clippy::cast_precision_loss,
+    clippy::cast_possible_truncation,
+    clippy::cast_sign_loss,
+    clippy::cast_possible_wrap,
+    clippy::cast_lossless,
+    clippy::match_wildcard_for_single_variants,
+    clippy::doc_markdown,
+    clippy::struct_excessive_bools,
+    clippy::needless_pass_by_value,
+    clippy::needless_pass_by_ref_mut,
+    clippy::multiple_crate_versions
+)]
+mod backend;
+mod config;
+mod config_io;
+mod graphics;
+mod gui;
+mod hid;
+mod overlays;
+mod shaders;
+mod state;
+
+#[cfg(feature = "wayvr")]
+mod config_wayvr;
+
+use std::{
+    path::PathBuf,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+};
+
+use backend::notifications::DbusNotificationSender;
+use clap::Parser;
+use sysinfo::Pid;
+use tracing::level_filters::LevelFilter;
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
+
+/// The lightweight desktop overlay for OpenVR and OpenXR
+#[derive(Default, Parser, Debug)]
+#[command(version, about, long_about = None)]
+struct Args {
+    #[cfg(feature = "openvr")]
+    /// Force OpenVR backend
+    #[arg(long)]
+    openvr: bool,
+
+    #[cfg(feature = "openxr")]
+    /// Force OpenXR backend
+    #[arg(long)]
+    openxr: bool,
+
+    /// Show the working set of overlay on startup
+    #[arg(long)]
+    show: bool,
+
+    /// Uninstall OpenVR manifest and exit
+    #[arg(long)]
+    uninstall: bool,
+
+    /// Replace running WlxOverlay-S instance
+    #[arg(long)]
+    replace: bool,
+
+    /// Allow multiple running instances of WlxOverlay-S (things may break!)
+    #[arg(long)]
+    multi: bool,
+
+    /// Disable desktop access altogether.
+    #[arg(long)]
+    headless: bool,
+
+    /// Path to write logs to
+    #[arg(short, long, value_name = "FILE_PATH")]
+    log_to: Option<String>,
+
+    #[cfg(feature = "uidev")]
+    /// Show a desktop window of a UI panel for development
+    #[arg(short, long, value_name = "UI_NAME")]
+    uidev: Option<String>,
+}
+
+#[allow(clippy::unnecessary_wraps)]
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let mut args = if std::env::args().skip(1).any(|a| !a.is_empty()) {
+        Args::parse()
+    } else {
+        Args::default()
+    };
+
+    if !args.multi && !ensure_single_instance(args.replace) {
+        println!("Looks like WlxOverlay-S is already running.");
+        println!("Use --replace and I will terminate it for you.");
+        return Ok(());
+    }
+
+    logging_init(&mut args);
+
+    log::info!(
+        "Welcome to {} version {}!",
+        env!("CARGO_PKG_NAME"),
+        env!("WLX_BUILD"),
+    );
+    log::info!("It is {}.", chrono::Local::now().format("%c"));
+
+    #[cfg(feature = "openvr")]
+    if args.uninstall {
+        crate::backend::openvr::openvr_uninstall();
+        return Ok(());
+    }
+
+    let running = Arc::new(AtomicBool::new(true));
+    let _ = ctrlc::set_handler({
+        let running = running.clone();
+        move || {
+            running.store(false, Ordering::Relaxed);
+        }
+    });
+
+    auto_run(running, args);
+
+    Ok(())
+}
+
+#[allow(unused_mut)]
+fn auto_run(running: Arc<AtomicBool>, args: Args) {
+    use backend::common::BackendError;
+
+    let mut tried_xr = false;
+    let mut tried_vr = false;
+
+    #[cfg(feature = "openxr")]
+    if !args_get_openvr(&args) {
+        use crate::backend::openxr::openxr_run;
+        tried_xr = true;
+        match openxr_run(running.clone(), args.show, args.headless) {
+            Ok(()) => return,
+            Err(BackendError::NotSupported) => (),
+            Err(e) => {
+                log::error!("{e:?}");
+                return;
+            }
+        }
+    }
+
+    #[cfg(feature = "openvr")]
+    if !args_get_openxr(&args) {
+        use crate::backend::openvr::openvr_run;
+        tried_vr = true;
+        match openvr_run(running, args.show, args.headless) {
+            Ok(()) => return,
+            Err(BackendError::NotSupported) => (),
+            Err(e) => {
+                log::error!("{e:?}");
+                return;
+            }
+        }
+    }
+
+    log::error!("No more backends to try");
+
+    let instructions = match (tried_xr, tried_vr) {
+        (true, true) => "Make sure that Monado, WiVRn or SteamVR is running.",
+        (false, true) => "Make sure that SteamVR is running.",
+        (true, false) => "Make sure that Monado or WiVRn is running.",
+        _ => "Check your launch arguments.",
+    };
+
+    let instructions = format!("Could not connect to runtime.\n{instructions}");
+
+    let _ = DbusNotificationSender::new()
+        .and_then(|s| s.notify_send("WlxOverlay-S", &instructions, 1, 0, 0, false));
+
+    #[cfg(not(any(feature = "openvr", feature = "openxr")))]
+    compile_error!("No VR support! Enable either openvr or openxr features!");
+
+    #[cfg(not(any(feature = "wayland", feature = "x11")))]
+    compile_error!("No desktop support! Enable either wayland or x11 features!");
+}
+
+#[allow(dead_code, unused_variables)]
+const fn args_get_openvr(args: &Args) -> bool {
+    #[cfg(feature = "openvr")]
+    let ret = args.openvr;
+
+    #[cfg(not(feature = "openvr"))]
+    let ret = false;
+
+    ret
+}
+
+#[allow(dead_code, unused_variables)]
+const fn args_get_openxr(args: &Args) -> bool {
+    #[cfg(feature = "openxr")]
+    let ret = args.openxr;
+
+    #[cfg(not(feature = "openxr"))]
+    let ret = false;
+
+    ret
+}
+
+fn logging_init(args: &mut Args) {
+    let log_file_path = args
+        .log_to
+        .take()
+        .or_else(|| std::env::var("WLX_LOGFILE").ok())
+        .unwrap_or_else(|| String::from("/tmp/wlx.log"));
+
+    let file_writer = match std::fs::OpenOptions::new()
+        .create(true)
+        .write(true)
+        .truncate(true)
+        .open(&log_file_path)
+    {
+        Ok(file) => {
+            println!("Logging to {}", &log_file_path);
+            Some(file)
+        }
+        Err(e) => {
+            println!("Failed to open log file (path: {e:?}): {log_file_path}");
+            None
+        }
+    };
+
+    let registry = tracing_subscriber::registry()
+        .with(
+            tracing_subscriber::fmt::layer()
+                .pretty()
+                .with_writer(std::io::stderr),
+        )
+        .with(
+            /* read RUST_LOG env var */
+            EnvFilter::builder()
+                .with_default_directive(LevelFilter::INFO.into())
+                .from_env_lossy()
+                .add_directive("zbus=warn".parse().unwrap())
+                .add_directive("wlx_capture::wayland=info".parse().unwrap())
+                .add_directive("smithay=debug".parse().unwrap()), /* GLES render spam */
+        );
+
+    if let Some(writer) = file_writer {
+        registry
+            .with(
+                tracing_subscriber::fmt::layer()
+                    .with_file(true)
+                    .with_line_number(true)
+                    .with_writer(writer)
+                    .with_ansi(false),
+            )
+            .init();
+    } else {
+        registry.init();
+    }
+
+    log_panics::init();
+}
+
+fn ensure_single_instance(replace: bool) -> bool {
+    let mut path =
+        std::env::var("XDG_RUNTIME_DIR").map_or_else(|_| PathBuf::from("/tmp"), PathBuf::from);
+    path.push("wlx-overlay-s.pid");
+
+    if path.exists() {
+        // load contents
+        if let Ok(pid_str) = std::fs::read_to_string(&path) {
+            if let Ok(pid) = pid_str.trim().parse::<u32>() {
+                let mut system = sysinfo::System::new();
+                system.refresh_processes(
+                    sysinfo::ProcessesToUpdate::Some(&[Pid::from_u32(pid)]),
+                    false,
+                );
+                if let Some(proc) = system.process(sysinfo::Pid::from_u32(pid)) {
+                    if replace {
+                        proc.kill_with(sysinfo::Signal::Term);
+                        proc.wait();
+                    } else {
+                        return false;
+                    }
+                }
+            }
+        }
+    }
+
+    let pid = std::process::id().to_string();
+    std::fs::write(path, pid).unwrap();
+
+    true
+}

wlx-overlay-s/src/overlays/anchor.rs

@@ -0,0 +1,75 @@
+use glam::Vec3A;
+use std::sync::{Arc, LazyLock};
+use wgui::parser::parse_color_hex;
+use wgui::renderer_vk::text::{FontWeight, TextStyle};
+use wgui::taffy;
+use wgui::taffy::prelude::{length, percent};
+use wgui::widget::rectangle::{Rectangle, RectangleParams};
+use wgui::widget::text::{TextLabel, TextParams};
+use wgui::widget::util::WLength;
+
+use crate::backend::overlay::{OverlayData, OverlayState, Positioning, Z_ORDER_ANCHOR};
+use crate::gui::panel::GuiPanel;
+use crate::state::AppState;
+
+pub static ANCHOR_NAME: LazyLock<Arc<str>> = LazyLock::new(|| Arc::from("anchor"));
+
+pub fn create_anchor<O>(app: &mut AppState) -> anyhow::Result<OverlayData<O>>
+where
+    O: Default,
+{
+    let mut panel = GuiPanel::new_blank(app, 200, 200)?;
+
+    let (rect, _) = panel.layout.add_child(
+        panel.layout.root_widget,
+        Rectangle::create(RectangleParams {
+            color: wgui::drawing::Color::new(0., 0., 0., 0.),
+            border_color: parse_color_hex("#ffff00").unwrap(),
+            border: 2.0,
+            round: WLength::Percent(1.0),
+            ..Default::default()
+        })
+        .unwrap(),
+        taffy::Style {
+            size: taffy::Size {
+                width: percent(1.0),
+                height: percent(1.0),
+            },
+            align_items: Some(taffy::AlignItems::Center),
+            justify_content: Some(taffy::JustifyContent::Center),
+            padding: length(4.0),
+            ..Default::default()
+        },
+    )?;
+
+    let _ = panel.layout.add_child(
+        rect,
+        TextLabel::create(TextParams {
+            content: "Center".into(),
+            style: TextStyle {
+                weight: Some(FontWeight::Bold),
+                size: Some(36.0),
+                color: parse_color_hex("#ffff00"),
+                ..Default::default()
+            },
+        })
+        .unwrap(),
+        taffy::style::Style::DEFAULT,
+    );
+
+    Ok(OverlayData {
+        state: OverlayState {
+            name: ANCHOR_NAME.clone(),
+            want_visible: false,
+            interactable: false,
+            grabbable: false,
+            z_order: Z_ORDER_ANCHOR,
+            spawn_scale: 0.1,
+            spawn_point: Vec3A::NEG_Z * 0.5,
+            positioning: Positioning::Static,
+            ..Default::default()
+        },
+        backend: Box::new(panel),
+        ..Default::default()
+    })
+}

wlx-overlay-s/src/overlays/custom.rs

@@ -0,0 +1,36 @@
+use std::sync::Arc;
+
+use glam::Vec3A;
+
+use crate::{
+    backend::overlay::{OverlayBackend, OverlayState},
+    gui::panel::GuiPanel,
+    state::AppState,
+};
+
+const SETTINGS_NAME: &str = "settings";
+
+pub fn create_custom(
+    app: &mut AppState,
+    name: Arc<str>,
+) -> Option<(OverlayState, Box<dyn OverlayBackend>)> {
+    return None;
+
+    unreachable!();
+
+    let panel = GuiPanel::new_blank(&app, 200, 200).ok()?;
+
+    let state = OverlayState {
+        name,
+        want_visible: true,
+        interactable: true,
+        grabbable: true,
+        spawn_scale: 0.1, //TODO: this
+        spawn_point: Vec3A::from_array([0., 0., -0.5]),
+        //interaction_transform: ui_transform(config.size),
+        ..Default::default()
+    };
+    let backend = Box::new(panel);
+
+    Some((state, backend))
+}

wlx-overlay-s/src/overlays/keyboard.rs

@@ -0,0 +1,515 @@
+use std::{
+    collections::HashMap,
+    process::{Child, Command},
+    str::FromStr,
+    sync::{Arc, LazyLock},
+};
+
+use crate::{
+    backend::{
+        input::{InteractionHandler, PointerMode},
+        overlay::{
+            FrameMeta, OverlayBackend, OverlayData, OverlayRenderer, OverlayState, Positioning,
+            ShouldRender,
+        },
+    },
+    config::{self, ConfigType},
+    graphics::CommandBuffers,
+    gui::panel::GuiPanel,
+    hid::{
+        get_key_type, KeyModifier, KeyType, VirtualKey, XkbKeymap, ALT, CTRL, KEYS_TO_MODS, META,
+        NUM_LOCK, SHIFT, SUPER,
+    },
+    state::{AppState, KeyboardFocus},
+};
+use glam::{vec2, vec3a, Affine2, Vec2Swizzles, Vec4};
+use regex::Regex;
+use serde::{Deserialize, Serialize};
+use vulkano::image::view::ImageView;
+use wgui::{
+    parser::parse_color_hex,
+    taffy::{self, prelude::length},
+    widget::{
+        div::Div,
+        rectangle::{Rectangle, RectangleParams},
+        util::WLength,
+    },
+};
+
+const PIXELS_PER_UNIT: f32 = 80.;
+const BUTTON_PADDING: f32 = 4.;
+const AUTO_RELEASE_MODS: [KeyModifier; 5] = [SHIFT, CTRL, ALT, SUPER, META];
+
+pub const KEYBOARD_NAME: &str = "kbd";
+
+fn send_key(app: &mut AppState, key: VirtualKey, down: bool) {
+    match app.keyboard_focus {
+        KeyboardFocus::PhysicalScreen => {
+            app.hid_provider.send_key(key, down);
+        }
+        KeyboardFocus::WayVR =>
+        {
+            #[cfg(feature = "wayvr")]
+            if let Some(wayvr) = &app.wayvr {
+                wayvr.borrow_mut().data.state.send_key(key as u32, down);
+            }
+        }
+    }
+}
+
+fn set_modifiers(app: &mut AppState, mods: u8) {
+    match app.keyboard_focus {
+        KeyboardFocus::PhysicalScreen => {
+            app.hid_provider.set_modifiers(mods);
+        }
+        KeyboardFocus::WayVR =>
+        {
+            #[cfg(feature = "wayvr")]
+            if let Some(wayvr) = &app.wayvr {
+                wayvr.borrow_mut().data.state.set_modifiers(mods);
+            }
+        }
+    }
+}
+
+#[allow(clippy::too_many_lines)]
+pub fn create_keyboard<O>(
+    app: &AppState,
+    mut keymap: Option<XkbKeymap>,
+) -> anyhow::Result<OverlayData<O>>
+where
+    O: Default,
+{
+    let size = vec2(
+        LAYOUT.row_size * PIXELS_PER_UNIT,
+        (LAYOUT.main_layout.len() as f32) * PIXELS_PER_UNIT,
+    );
+
+    let data = KeyboardData {
+        modifiers: 0,
+        alt_modifier: match LAYOUT.alt_modifier {
+            AltModifier::Shift => SHIFT,
+            AltModifier::Ctrl => CTRL,
+            AltModifier::Alt => ALT,
+            AltModifier::Super => SUPER,
+            AltModifier::Meta => META,
+            _ => 0,
+        },
+        processes: vec![],
+    };
+
+    let padding = 4f32;
+
+    let mut panel = GuiPanel::new_blank(
+        app,
+        padding.mul_add(2.0, size.x) as u32,
+        padding.mul_add(2.0, size.y) as u32,
+    )?;
+
+    let (background, _) = panel.layout.add_child(
+        panel.layout.root_widget,
+        Rectangle::create(RectangleParams {
+            color: wgui::drawing::Color::new(0., 0., 0., 0.6),
+            round: WLength::Units(4.0),
+            ..Default::default()
+        })
+        .unwrap(),
+        taffy::Style {
+            flex_direction: taffy::FlexDirection::Column,
+            padding: length(padding),
+            ..Default::default()
+        },
+    )?;
+
+    let has_altgr = keymap
+        .as_ref()
+        .is_some_and(super::super::hid::XkbKeymap::has_altgr);
+
+    if !LAYOUT.auto_labels.unwrap_or(true) {
+        keymap = None;
+    }
+
+    for row in 0..LAYOUT.key_sizes.len() {
+        let (div, _) = panel.layout.add_child(
+            background,
+            Div::create().unwrap(),
+            taffy::Style {
+                flex_direction: taffy::FlexDirection::Row,
+                ..Default::default()
+            },
+        )?;
+
+        for col in 0..LAYOUT.key_sizes[row].len() {
+            let my_size = LAYOUT.key_sizes[row][col];
+            let my_size = taffy::Size {
+                width: length(PIXELS_PER_UNIT * my_size),
+                height: length(PIXELS_PER_UNIT),
+            };
+
+            if let Some(key) = LAYOUT.main_layout[row][col].as_ref() {
+                let mut label = Vec::with_capacity(2);
+                let mut maybe_state: Option<KeyButtonData> = None;
+                let mut cap_type = KeyCapType::Regular;
+
+                if let Ok(vk) = VirtualKey::from_str(key) {
+                    if let Some(keymap) = keymap.as_ref() {
+                        match get_key_type(vk) {
+                            KeyType::Symbol => {
+                                let label0 = keymap.label_for_key(vk, 0);
+                                let label1 = keymap.label_for_key(vk, SHIFT);
+
+                                if label0.chars().next().is_some_and(char::is_alphabetic) {
+                                    label.push(label1);
+                                    if has_altgr {
+                                        cap_type = KeyCapType::RegularAltGr;
+                                        label.push(keymap.label_for_key(vk, META));
+                                    } else {
+                                        cap_type = KeyCapType::Regular;
+                                    }
+                                } else {
+                                    label.push(label0);
+                                    label.push(label1);
+                                    if has_altgr {
+                                        label.push(keymap.label_for_key(vk, META));
+                                        cap_type = KeyCapType::ReversedAltGr;
+                                    } else {
+                                        cap_type = KeyCapType::Reversed;
+                                    }
+                                }
+                            }
+                            KeyType::NumPad => {
+                                label.push(keymap.label_for_key(vk, NUM_LOCK));
+                            }
+                            KeyType::Other => {}
+                        }
+                    }
+
+                    if let Some(mods) = KEYS_TO_MODS.get(vk) {
+                        maybe_state = Some(KeyButtonData::Modifier {
+                            modifier: *mods,
+                            sticky: false,
+                        });
+                    } else {
+                        maybe_state = Some(KeyButtonData::Key { vk, pressed: false });
+                    }
+                } else if let Some(macro_verbs) = LAYOUT.macros.get(key) {
+                    maybe_state = Some(KeyButtonData::Macro {
+                        verbs: key_events_for_macro(macro_verbs),
+                    });
+                } else if let Some(exec_args) = LAYOUT.exec_commands.get(key) {
+                    if exec_args.is_empty() {
+                        log::error!("Keyboard: EXEC args empty for {key}");
+                    } else {
+                        let mut iter = exec_args.iter().cloned();
+                        if let Some(program) = iter.next() {
+                            maybe_state = Some(KeyButtonData::Exec {
+                                program,
+                                args: iter.by_ref().take_while(|arg| arg[..] != *"null").collect(),
+                                release_program: iter.next(),
+                                release_args: iter.collect(),
+                            });
+                        }
+                    }
+                } else {
+                    log::error!("Unknown key: {key}");
+                }
+
+                if let Some(state) = maybe_state {
+                    if label.is_empty() {
+                        label = LAYOUT.label_for_key(key);
+                    }
+                    let _ = panel.layout.add_child(
+                        div,
+                        Rectangle::create(RectangleParams {
+                            border_color: parse_color_hex("#dddddd").unwrap(),
+                            border: 2.0,
+                            round: WLength::Units(4.0),
+                            ..Default::default()
+                        })
+                        .unwrap(),
+                        taffy::Style {
+                            size: my_size,
+                            min_size: my_size,
+                            max_size: my_size,
+                            ..Default::default()
+                        },
+                    )?;
+                } else {
+                    let _ = panel.layout.add_child(
+                        div,
+                        Div::create().unwrap(),
+                        taffy::Style {
+                            size: my_size,
+                            min_size: my_size,
+                            max_size: my_size,
+                            ..Default::default()
+                        },
+                    )?;
+                }
+            }
+        }
+    }
+
+    let interaction_transform = Affine2::from_translation(vec2(0.5, 0.5))
+        * Affine2::from_scale(vec2(1., -size.x as f32 / size.y as f32));
+
+    let width = LAYOUT.row_size * 0.05 * app.session.config.keyboard_scale;
+
+    Ok(OverlayData {
+        state: OverlayState {
+            name: KEYBOARD_NAME.into(),
+            grabbable: true,
+            recenter: true,
+            positioning: Positioning::Anchored,
+            interactable: true,
+            spawn_scale: width,
+            spawn_point: vec3a(0., -0.5, 0.),
+            interaction_transform,
+            ..Default::default()
+        },
+        backend: Box::new(KeyboardBackend { panel }),
+        ..Default::default()
+    })
+}
+
+struct KeyboardData {
+    modifiers: KeyModifier,
+    alt_modifier: KeyModifier,
+    processes: Vec<Child>,
+}
+
+const KEY_AUDIO_WAV: &[u8] = include_bytes!("../res/421581.wav");
+
+fn key_click(app: &mut AppState) {
+    if app.session.config.keyboard_sound_enabled {
+        app.audio.play(KEY_AUDIO_WAV);
+    }
+}
+
+enum KeyButtonData {
+    Key {
+        vk: VirtualKey,
+        pressed: bool,
+    },
+    Modifier {
+        modifier: KeyModifier,
+        sticky: bool,
+    },
+    Macro {
+        verbs: Vec<(VirtualKey, bool)>,
+    },
+    Exec {
+        program: String,
+        args: Vec<String>,
+        release_program: Option<String>,
+        release_args: Vec<String>,
+    },
+}
+
+static LAYOUT: LazyLock<Layout> = LazyLock::new(Layout::load_from_disk);
+
+static MACRO_REGEX: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"^([A-Za-z0-9_-]+)(?: +(UP|DOWN))?$").unwrap()); // want panic
+
+#[derive(Debug, Default, Clone, Copy, Deserialize, Serialize)]
+#[repr(usize)]
+pub enum AltModifier {
+    #[default]
+    None,
+    Shift,
+    Ctrl,
+    Alt,
+    Super,
+    Meta,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[allow(clippy::struct_field_names)]
+pub struct Layout {
+    name: String,
+    row_size: f32,
+    key_sizes: Vec<Vec<f32>>,
+    main_layout: Vec<Vec<Option<String>>>,
+    alt_modifier: AltModifier,
+    exec_commands: HashMap<String, Vec<String>>,
+    macros: HashMap<String, Vec<String>>,
+    labels: HashMap<String, Vec<String>>,
+    auto_labels: Option<bool>,
+}
+
+impl Layout {
+    fn load_from_disk() -> Self {
+        let mut layout = config::load_known_yaml::<Self>(ConfigType::Keyboard);
+        layout.post_load();
+        layout
+    }
+
+    fn post_load(&mut self) {
+        for i in 0..self.key_sizes.len() {
+            let row = &self.key_sizes[i];
+            let width: f32 = row.iter().sum();
+            assert!(
+                (width - self.row_size).abs() < 0.001,
+                "Row {} has a width of {}, but the row size is {}",
+                i,
+                width,
+                self.row_size
+            );
+        }
+
+        for i in 0..self.main_layout.len() {
+            let row = &self.main_layout[i];
+            let width = row.len();
+            assert!(
+                (width == self.key_sizes[i].len()),
+                "Row {} has {} keys, needs to have {} according to key_sizes",
+                i,
+                width,
+                self.key_sizes[i].len()
+            );
+        }
+    }
+
+    fn label_for_key(&self, key: &str) -> Vec<String> {
+        if let Some(label) = self.labels.get(key) {
+            return label.clone();
+        }
+        if key.is_empty() {
+            return vec![];
+        }
+        if key.len() == 1 {
+            return vec![key.to_string().to_lowercase()];
+        }
+        let mut key = key;
+        if key.starts_with("KP_") {
+            key = &key[3..];
+        }
+        if key.contains('_') {
+            key = key.split('_').next().unwrap_or_else(|| {
+                log::error!("keyboard.yaml: Key '{key}' must not start or end with '_'!");
+                "???"
+            });
+        }
+        vec![format!(
+            "{}{}",
+            key.chars().next().unwrap().to_uppercase(), // safe because we checked is_empty
+            &key[1..].to_lowercase()
+        )]
+    }
+}
+
+fn key_events_for_macro(macro_verbs: &Vec<String>) -> Vec<(VirtualKey, bool)> {
+    let mut key_events = vec![];
+    for verb in macro_verbs {
+        if let Some(caps) = MACRO_REGEX.captures(verb) {
+            if let Ok(virtual_key) = VirtualKey::from_str(&caps[1]) {
+                if let Some(state) = caps.get(2) {
+                    if state.as_str() == "UP" {
+                        key_events.push((virtual_key, false));
+                    } else if state.as_str() == "DOWN" {
+                        key_events.push((virtual_key, true));
+                    } else {
+                        log::error!(
+                            "Unknown key state in macro: {}, looking for UP or DOWN.",
+                            state.as_str()
+                        );
+                        return vec![];
+                    }
+                } else {
+                    key_events.push((virtual_key, true));
+                    key_events.push((virtual_key, false));
+                }
+            } else {
+                log::error!("Unknown virtual key: {}", &caps[1]);
+                return vec![];
+            }
+        }
+    }
+    key_events
+}
+
+struct KeyboardBackend {
+    panel: GuiPanel,
+}
+
+impl OverlayBackend for KeyboardBackend {
+    fn set_interaction(&mut self, interaction: Box<dyn crate::backend::input::InteractionHandler>) {
+        self.panel.set_interaction(interaction);
+    }
+    fn set_renderer(&mut self, renderer: Box<dyn crate::backend::overlay::OverlayRenderer>) {
+        self.panel.set_renderer(renderer);
+    }
+}
+
+impl InteractionHandler for KeyboardBackend {
+    fn on_pointer(
+        &mut self,
+        app: &mut AppState,
+        hit: &crate::backend::input::PointerHit,
+        pressed: bool,
+    ) {
+        self.panel.on_pointer(app, hit, pressed);
+    }
+    fn on_scroll(
+        &mut self,
+        app: &mut AppState,
+        hit: &crate::backend::input::PointerHit,
+        delta_y: f32,
+        delta_x: f32,
+    ) {
+        self.panel.on_scroll(app, hit, delta_y, delta_x);
+    }
+    fn on_left(&mut self, app: &mut AppState, pointer: usize) {
+        self.panel.on_left(app, pointer);
+    }
+    fn on_hover(
+        &mut self,
+        app: &mut AppState,
+        hit: &crate::backend::input::PointerHit,
+    ) -> Option<crate::backend::input::Haptics> {
+        self.panel.on_hover(app, hit)
+    }
+}
+
+impl OverlayRenderer for KeyboardBackend {
+    fn init(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.panel.init(app)
+    }
+    fn should_render(&mut self, app: &mut AppState) -> anyhow::Result<ShouldRender> {
+        self.panel.should_render(app)
+    }
+    fn render(
+        &mut self,
+        app: &mut AppState,
+        tgt: Arc<ImageView>,
+        buf: &mut CommandBuffers,
+        alpha: f32,
+    ) -> anyhow::Result<bool> {
+        self.panel.render(app, tgt, buf, alpha)
+    }
+    fn frame_meta(&mut self) -> Option<FrameMeta> {
+        self.panel.frame_meta()
+    }
+    fn pause(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        set_modifiers(app, 0);
+        self.panel.pause(app)
+    }
+    fn resume(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.panel.resume(app)
+    }
+}
+
+pub enum KeyCapType {
+    /// Label is in center of keycap
+    Regular,
+    /// Label on the top
+    /// AltGr symbol on bottom
+    RegularAltGr,
+    /// Primary symbol on bottom
+    /// Shift symbol on top
+    Reversed,
+    /// Primary symbol on bottom-left
+    /// Shift symbol on top-left
+    /// AltGr symbol on bottom-right
+    ReversedAltGr,
+}

wlx-overlay-s/src/overlays/mirror.rs

@@ -0,0 +1,158 @@
+use std::{
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+use futures::{Future, FutureExt};
+use vulkano::image::view::ImageView;
+use wlx_capture::pipewire::{pipewire_select_screen, PipewireCapture, PipewireSelectScreenResult};
+
+use crate::{
+    backend::{
+        common::OverlaySelector,
+        overlay::{
+            ui_transform, FrameMeta, OverlayBackend, OverlayRenderer, OverlayState, ShouldRender,
+            SplitOverlayBackend,
+        },
+        task::TaskType,
+    },
+    graphics::CommandBuffers,
+    state::{AppSession, AppState},
+};
+
+use super::screen::ScreenRenderer;
+type PinnedSelectorFuture = core::pin::Pin<
+    Box<dyn Future<Output = Result<PipewireSelectScreenResult, wlx_capture::pipewire::AshpdError>>>,
+>;
+
+pub struct MirrorRenderer {
+    name: Arc<str>,
+    renderer: Option<ScreenRenderer>,
+    selector: Option<PinnedSelectorFuture>,
+    last_extent: [u32; 3],
+}
+impl MirrorRenderer {
+    pub fn new(name: Arc<str>) -> Self {
+        let selector = Box::pin(pipewire_select_screen(None, false, false, false, false));
+        Self {
+            name,
+            renderer: None,
+            selector: Some(selector),
+            last_extent: [0; 3],
+        }
+    }
+}
+
+impl OverlayRenderer for MirrorRenderer {
+    fn init(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn should_render(&mut self, app: &mut AppState) -> anyhow::Result<ShouldRender> {
+        if let Some(mut selector) = self.selector.take() {
+            let maybe_pw_result = match selector
+                .poll_unpin(&mut Context::from_waker(futures::task::noop_waker_ref()))
+            {
+                Poll::Ready(result) => result,
+                Poll::Pending => {
+                    self.selector = Some(selector);
+                    return Ok(ShouldRender::Unable);
+                }
+            };
+
+            match maybe_pw_result {
+                Ok(pw_result) => {
+                    let node_id = pw_result.streams.first().unwrap().node_id; // streams guaranteed to have at least one element
+                    log::info!("{}: PipeWire node selected: {}", self.name.clone(), node_id);
+                    let capture = PipewireCapture::new(self.name.clone(), node_id);
+                    self.renderer = Some(ScreenRenderer::new_raw(
+                        self.name.clone(),
+                        Box::new(capture),
+                    ));
+                    app.tasks.enqueue(TaskType::Overlay(
+                        OverlaySelector::Name(self.name.clone()),
+                        Box::new(|app, o| {
+                            o.grabbable = true;
+                            o.interactable = true;
+                            o.reset(app, false);
+                        }),
+                    ));
+                }
+                Err(e) => {
+                    log::warn!("Failed to create mirror due to PipeWire error: {e:?}");
+                    self.renderer = None;
+                    // drop self
+                    app.tasks
+                        .enqueue(TaskType::DropOverlay(OverlaySelector::Name(
+                            self.name.clone(),
+                        )));
+                }
+            }
+        }
+        self.renderer
+            .as_mut()
+            .map_or(Ok(ShouldRender::Unable), |r| r.should_render(app))
+    }
+    fn render(
+        &mut self,
+        app: &mut AppState,
+        tgt: Arc<ImageView>,
+        buf: &mut CommandBuffers,
+        alpha: f32,
+    ) -> anyhow::Result<bool> {
+        let mut result = false;
+        if let Some(renderer) = self.renderer.as_mut() {
+            result = renderer.render(app, tgt, buf, alpha)?;
+            if let Some(meta) = renderer.frame_meta() {
+                let extent = meta.extent;
+                if self.last_extent != extent {
+                    self.last_extent = extent;
+                    // resized
+                    app.tasks.enqueue(TaskType::Overlay(
+                        OverlaySelector::Name(self.name.clone()),
+                        Box::new(move |_app, o| {
+                            o.interaction_transform = ui_transform([extent[0], extent[1]]);
+                        }),
+                    ));
+                }
+            }
+        }
+
+        Ok(result)
+    }
+    fn pause(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        if let Some(renderer) = self.renderer.as_mut() {
+            renderer.pause(app)?;
+        }
+        Ok(())
+    }
+    fn resume(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        if let Some(renderer) = self.renderer.as_mut() {
+            renderer.resume(app)?;
+        }
+        Ok(())
+    }
+
+    fn frame_meta(&mut self) -> Option<FrameMeta> {
+        self.renderer.as_mut().and_then(ScreenRenderer::frame_meta)
+    }
+}
+
+pub fn new_mirror(
+    name: Arc<str>,
+    show_hide: bool,
+    session: &AppSession,
+) -> (OverlayState, Box<dyn OverlayBackend>) {
+    let state = OverlayState {
+        name: name.clone(),
+        show_hide,
+        want_visible: true,
+        spawn_scale: 0.5 * session.config.desktop_view_scale,
+        ..Default::default()
+    };
+    let backend = Box::new(SplitOverlayBackend {
+        renderer: Box::new(MirrorRenderer::new(name)),
+        ..Default::default()
+    });
+
+    (state, backend)
+}

wlx-overlay-s/src/overlays/mod.rs

@@ -0,0 +1,11 @@
+pub mod anchor;
+pub mod custom;
+pub mod keyboard;
+#[cfg(feature = "wayland")]
+pub mod mirror;
+pub mod screen;
+pub mod toast;
+pub mod watch;
+
+#[cfg(feature = "wayvr")]
+pub mod wayvr;

wlx-overlay-s/src/overlays/toast.rs

@@ -0,0 +1,268 @@
+use std::{
+    f32::consts::PI,
+    ops::Add,
+    sync::{Arc, LazyLock},
+    time::Instant,
+};
+
+use glam::{vec3a, Quat};
+use idmap_derive::IntegerId;
+use serde::{Deserialize, Serialize};
+use wgui::{
+    parser::parse_color_hex,
+    renderer_vk::text::{FontWeight, TextStyle},
+    taffy::{
+        self,
+        prelude::{auto, length, percent},
+    },
+    widget::{
+        rectangle::{Rectangle, RectangleParams},
+        text::{TextLabel, TextParams},
+        util::WLength,
+    },
+};
+
+use crate::{
+    backend::{
+        common::OverlaySelector,
+        overlay::{OverlayBackend, OverlayState, Positioning, Z_ORDER_TOAST},
+        task::TaskType,
+    },
+    gui::panel::GuiPanel,
+    state::{AppState, LeftRight},
+};
+
+const FONT_SIZE: isize = 16;
+const PADDING: (f32, f32) = (25., 7.);
+const PIXELS_TO_METERS: f32 = 1. / 2000.;
+static TOAST_NAME: LazyLock<Arc<str>> = LazyLock::new(|| "toast".into());
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub enum DisplayMethod {
+    Hide,
+    Center,
+    Watch,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, IntegerId, Serialize, Deserialize)]
+pub enum ToastTopic {
+    System,
+    DesktopNotification,
+    XSNotification,
+    IpdChange,
+}
+
+pub struct Toast {
+    pub title: String,
+    pub body: String,
+    pub opacity: f32,
+    pub timeout: f32,
+    pub sound: bool,
+    pub topic: ToastTopic,
+}
+
+#[allow(dead_code)]
+impl Toast {
+    pub const fn new(topic: ToastTopic, title: String, body: String) -> Self {
+        Self {
+            title,
+            body,
+            opacity: 1.0,
+            timeout: 3.0,
+            sound: false,
+            topic,
+        }
+    }
+    pub const fn with_timeout(mut self, timeout: f32) -> Self {
+        self.timeout = timeout;
+        self
+    }
+    pub const fn with_opacity(mut self, opacity: f32) -> Self {
+        self.opacity = opacity;
+        self
+    }
+    pub const fn with_sound(mut self, sound: bool) -> Self {
+        self.sound = sound;
+        self
+    }
+    pub fn submit(self, app: &mut AppState) {
+        self.submit_at(app, Instant::now());
+    }
+    pub fn submit_at(self, app: &mut AppState, instant: Instant) {
+        let selector = OverlaySelector::Name(TOAST_NAME.clone());
+
+        let destroy_at = instant.add(std::time::Duration::from_secs_f32(self.timeout));
+
+        let has_sound = self.sound && app.session.config.notifications_sound_enabled;
+        if has_sound {
+            app.audio.play(app.toast_sound);
+        }
+
+        // drop any toast that was created before us.
+        // (DropOverlay only drops overlays that were
+        // created before current frame)
+        app.tasks
+            .enqueue_at(TaskType::DropOverlay(selector.clone()), instant);
+
+        // CreateOverlay only creates the overlay if
+        // the selector doesn't exist yet, so in case
+        // multiple toasts are submitted for the same
+        // frame, only the first one gets created
+        app.tasks.enqueue_at(
+            TaskType::CreateOverlay(
+                selector,
+                Box::new(move |app| {
+                    let mut maybe_toast = new_toast(self, app);
+                    if let Some((state, _)) = maybe_toast.as_mut() {
+                        state.auto_movement(app);
+                        app.tasks.enqueue_at(
+                            // at timeout, drop the overlay by ID instead
+                            // in order to avoid dropping any newer toasts
+                            TaskType::DropOverlay(OverlaySelector::Id(state.id)),
+                            destroy_at,
+                        );
+                    }
+                    maybe_toast
+                }),
+            ),
+            instant,
+        );
+    }
+}
+
+#[allow(clippy::too_many_lines)]
+fn new_toast(toast: Toast, app: &mut AppState) -> Option<(OverlayState, Box<dyn OverlayBackend>)> {
+    let current_method = app
+        .session
+        .toast_topics
+        .get(toast.topic)
+        .copied()
+        .unwrap_or(DisplayMethod::Hide);
+
+    let (spawn_point, spawn_rotation, positioning) = match current_method {
+        DisplayMethod::Hide => return None,
+        DisplayMethod::Center => (
+            vec3a(0., -0.2, -0.5),
+            Quat::IDENTITY,
+            Positioning::FollowHead { lerp: 0.1 },
+        ),
+        DisplayMethod::Watch => {
+            let mut watch_pos = app.session.config.watch_pos + vec3a(-0.005, -0.05, 0.02);
+            let mut watch_rot = app.session.config.watch_rot;
+            let relative_to = match app.session.config.watch_hand {
+                LeftRight::Left => Positioning::FollowHand { hand: 0, lerp: 1.0 },
+                LeftRight::Right => {
+                    watch_pos.x = -watch_pos.x;
+                    watch_rot = watch_rot * Quat::from_rotation_x(PI) * Quat::from_rotation_z(PI);
+                    Positioning::FollowHand { hand: 1, lerp: 1.0 }
+                }
+            };
+            (watch_pos, watch_rot, relative_to)
+        }
+    };
+
+    let title = if toast.title.is_empty() {
+        "Notification".into()
+    } else {
+        toast.title
+    };
+
+    let mut panel = GuiPanel::new_blank(app, 600, 200).ok()?;
+
+    let (rect, _) = panel
+        .layout
+        .add_child(
+            panel.layout.root_widget,
+            Rectangle::create(RectangleParams {
+                color: parse_color_hex("#1e2030").unwrap(),
+                border_color: parse_color_hex("#5e7090").unwrap(),
+                border: 1.0,
+                round: WLength::Units(4.0),
+                ..Default::default()
+            })
+            .unwrap(),
+            taffy::Style {
+                align_items: Some(taffy::AlignItems::Center),
+                justify_content: Some(taffy::JustifyContent::Center),
+                padding: length(1.0),
+                ..Default::default()
+            },
+        )
+        .ok()?;
+
+    let _ = panel.layout.add_child(
+        rect,
+        TextLabel::create(TextParams {
+            content: title,
+            style: TextStyle {
+                color: parse_color_hex("#ffffff"),
+                ..Default::default()
+            },
+        })
+        .unwrap(),
+        taffy::Style {
+            size: taffy::Size {
+                width: percent(1.0),
+                height: auto(),
+            },
+            ..Default::default()
+        },
+    );
+
+    let _ = panel.layout.add_child(
+        rect,
+        TextLabel::create(TextParams {
+            content: toast.body,
+            style: TextStyle {
+                weight: Some(FontWeight::Bold),
+                color: parse_color_hex("#eeeeee"),
+                ..Default::default()
+            },
+        })
+        .unwrap(),
+        taffy::Style {
+            size: taffy::Size {
+                width: percent(1.0),
+                height: auto(),
+            },
+            ..Default::default()
+        },
+    );
+
+    let state = OverlayState {
+        name: TOAST_NAME.clone(),
+        want_visible: true,
+        spawn_scale: (panel.width as f32) * PIXELS_TO_METERS,
+        spawn_rotation,
+        spawn_point,
+        z_order: Z_ORDER_TOAST,
+        positioning,
+        ..Default::default()
+    };
+    let backend = Box::new(panel);
+
+    Some((state, backend))
+}
+
+fn msg_err(app: &mut AppState, message: &str) {
+    Toast::new(ToastTopic::System, "Error".into(), message.into())
+        .with_timeout(3.)
+        .submit(app);
+}
+
+// Display the same error in the terminal and as a toast in VR.
+// Formatted as "Failed to XYZ: Object is not defined"
+pub fn error_toast<ErrorType>(app: &mut AppState, title: &str, err: ErrorType)
+where
+    ErrorType: std::fmt::Display + std::fmt::Debug,
+{
+    log::error!("{title}: {err:?}"); // More detailed version (use Debug)
+
+    // Brief version (use Display)
+    msg_err(app, &format!("{title}: {err}"));
+}
+
+pub fn error_toast_str(app: &mut AppState, message: &str) {
+    log::error!("{message}");
+    msg_err(app, message);
+}

wlx-overlay-s/src/overlays/watch.rs

@@ -0,0 +1,100 @@
+use glam::Vec3A;
+use wgui::{
+    parser::parse_color_hex,
+    taffy::{
+        self,
+        prelude::{length, percent},
+    },
+    widget::{
+        rectangle::{Rectangle, RectangleParams},
+        util::WLength,
+    },
+};
+
+use crate::{
+    backend::overlay::{ui_transform, OverlayData, OverlayState, Positioning, Z_ORDER_WATCH},
+    gui::panel::GuiPanel,
+    state::AppState,
+};
+
+pub const WATCH_NAME: &str = "watch";
+
+pub fn create_watch<O>(app: &mut AppState) -> anyhow::Result<OverlayData<O>>
+where
+    O: Default,
+{
+    let mut panel = GuiPanel::new_blank(app, 400, 200)?;
+
+    let (_, _) = panel.layout.add_child(
+        panel.layout.root_widget,
+        Rectangle::create(RectangleParams {
+            color: wgui::drawing::Color::new(0., 0., 0., 0.5),
+            border_color: parse_color_hex("#00ffff").unwrap(),
+            border: 2.0,
+            round: WLength::Units(4.0),
+            ..Default::default()
+        })
+        .unwrap(),
+        taffy::Style {
+            size: taffy::Size {
+                width: percent(1.0),
+                height: percent(1.0),
+            },
+            align_items: Some(taffy::AlignItems::Center),
+            justify_content: Some(taffy::JustifyContent::Center),
+            padding: length(4.0),
+            ..Default::default()
+        },
+    )?;
+
+    let positioning = Positioning::FollowHand {
+        hand: app.session.config.watch_hand as _,
+        lerp: 1.0,
+    };
+
+    Ok(OverlayData {
+        state: OverlayState {
+            name: WATCH_NAME.into(),
+            want_visible: true,
+            interactable: true,
+            z_order: Z_ORDER_WATCH,
+            spawn_scale: 0.115, //TODO:configurable
+            spawn_point: app.session.config.watch_pos,
+            spawn_rotation: app.session.config.watch_rot,
+            interaction_transform: ui_transform([400, 200]),
+            positioning,
+            ..Default::default()
+        },
+        backend: Box::new(panel),
+        ..Default::default()
+    })
+}
+
+pub fn watch_fade<D>(app: &mut AppState, watch: &mut OverlayData<D>)
+where
+    D: Default,
+{
+    if watch.state.saved_transform.is_some() {
+        watch.state.want_visible = false;
+        return;
+    }
+
+    let to_hmd = (watch.state.transform.translation - app.input_state.hmd.translation).normalize();
+    let watch_normal = watch
+        .state
+        .transform
+        .transform_vector3a(Vec3A::NEG_Z)
+        .normalize();
+    let dot = to_hmd.dot(watch_normal);
+
+    if dot < app.session.config.watch_view_angle_min {
+        watch.state.want_visible = false;
+    } else {
+        watch.state.want_visible = true;
+
+        watch.state.alpha = (dot - app.session.config.watch_view_angle_min)
+            / (app.session.config.watch_view_angle_max - app.session.config.watch_view_angle_min);
+        watch.state.alpha += 0.1;
+        watch.state.alpha = watch.state.alpha.clamp(0., 1.);
+    }
+}

wlx-overlay-s/src/overlays/wayvr.rs

@@ -0,0 +1,970 @@
+use glam::{vec3a, Affine2, Vec3, Vec3A};
+use std::{cell::RefCell, collections::HashMap, rc::Rc, sync::Arc};
+use vulkano::{
+    command_buffer::CommandBufferUsage,
+    format::Format,
+    image::{view::ImageView, Image, ImageTiling, SubresourceLayout},
+    pipeline::graphics::input_assembly::PrimitiveTopology,
+};
+use wayvr_ipc::packet_server::{self, PacketServer, WvrStateChanged};
+use wgui::gfx::{pipeline::WGfxPipeline, WGfx};
+use wlx_capture::frame::{DmabufFrame, FourCC, FrameFormat, FramePlane};
+
+use crate::{
+    backend::{
+        common::{OverlayContainer, OverlaySelector},
+        input::{self, InteractionHandler},
+        overlay::{
+            ui_transform, FrameMeta, OverlayData, OverlayID, OverlayRenderer, OverlayState,
+            ShouldRender, SplitOverlayBackend, Z_ORDER_DASHBOARD,
+        },
+        task::TaskType,
+        wayvr::{
+            self, display,
+            server_ipc::{gen_args_vec, gen_env_vec},
+            WayVR, WayVRAction, WayVRDisplayClickAction,
+        },
+    },
+    config_wayvr,
+    graphics::{dmabuf::WGfxDmabuf, CommandBuffers, ExtentExt, Vert2Uv},
+    state::{self, AppState, KeyboardFocus},
+};
+
+use super::toast::error_toast;
+
+// Hard-coded for now
+const DASHBOARD_WIDTH: u16 = 1920;
+const DASHBOARD_HEIGHT: u16 = 1080;
+const DASHBOARD_DISPLAY_NAME: &str = "_DASHBOARD";
+
+pub struct WayVRContext {
+    wayvr: Rc<RefCell<WayVRData>>,
+    display: wayvr::display::DisplayHandle,
+}
+
+impl WayVRContext {
+    pub const fn new(wvr: Rc<RefCell<WayVRData>>, display: wayvr::display::DisplayHandle) -> Self {
+        Self {
+            wayvr: wvr,
+            display,
+        }
+    }
+}
+
+struct OverlayToCreate {
+    pub conf_display: config_wayvr::WayVRDisplay,
+    pub disp_handle: display::DisplayHandle,
+}
+
+pub struct WayVRData {
+    display_handle_map: HashMap<display::DisplayHandle, OverlayID>,
+    overlays_to_create: Vec<OverlayToCreate>,
+    dashboard_executed: bool,
+    pub data: WayVR,
+    pending_haptics: Option<input::Haptics>,
+}
+
+impl WayVRData {
+    pub fn new(config: wayvr::Config) -> anyhow::Result<Self> {
+        Ok(Self {
+            display_handle_map: HashMap::default(),
+            data: WayVR::new(config)?,
+            overlays_to_create: Vec::new(),
+            dashboard_executed: false,
+            pending_haptics: None,
+        })
+    }
+
+    fn get_unique_display_name(&self, mut candidate: String) -> String {
+        let mut num = 0;
+
+        while !self
+            .data
+            .state
+            .displays
+            .vec
+            .iter()
+            .flatten()
+            .any(|d| d.obj.name == candidate)
+        {
+            if num > 0 {
+                candidate = format!("{candidate} ({num})");
+            }
+            num += 1;
+        }
+
+        candidate
+    }
+}
+
+pub struct WayVRInteractionHandler {
+    context: Rc<RefCell<WayVRContext>>,
+    mouse_transform: Affine2,
+}
+
+impl WayVRInteractionHandler {
+    pub const fn new(context: Rc<RefCell<WayVRContext>>, mouse_transform: Affine2) -> Self {
+        Self {
+            context,
+            mouse_transform,
+        }
+    }
+}
+
+impl InteractionHandler for WayVRInteractionHandler {
+    fn on_hover(
+        &mut self,
+        _app: &mut state::AppState,
+        hit: &input::PointerHit,
+    ) -> Option<input::Haptics> {
+        let ctx = self.context.borrow();
+
+        let wayvr = &mut ctx.wayvr.borrow_mut();
+
+        if let Some(disp) = wayvr.data.state.displays.get(&ctx.display) {
+            let pos = self.mouse_transform.transform_point2(hit.uv);
+            let x = ((pos.x * f32::from(disp.width)) as i32).max(0);
+            let y = ((pos.y * f32::from(disp.height)) as i32).max(0);
+
+            let ctx = self.context.borrow();
+            wayvr
+                .data
+                .state
+                .send_mouse_move(ctx.display, x as u32, y as u32);
+        }
+
+        wayvr.pending_haptics.take()
+    }
+
+    fn on_left(&mut self, _app: &mut state::AppState, _pointer: usize) {
+        // Ignore event
+    }
+
+    fn on_pointer(&mut self, _app: &mut state::AppState, hit: &input::PointerHit, pressed: bool) {
+        if let Some(index) = match hit.mode {
+            input::PointerMode::Left => Some(wayvr::MouseIndex::Left),
+            input::PointerMode::Middle => Some(wayvr::MouseIndex::Center),
+            input::PointerMode::Right => Some(wayvr::MouseIndex::Right),
+            _ => {
+                // Unknown pointer event, ignore
+                None
+            }
+        } {
+            let ctx = self.context.borrow();
+            let wayvr = &mut ctx.wayvr.borrow_mut().data;
+            if pressed {
+                wayvr.state.send_mouse_down(ctx.display, index);
+            } else {
+                wayvr.state.send_mouse_up(index);
+            }
+        }
+    }
+
+    fn on_scroll(
+        &mut self,
+        _app: &mut state::AppState,
+        _hit: &input::PointerHit,
+        delta_y: f32,
+        delta_x: f32,
+    ) {
+        let ctx = self.context.borrow();
+        ctx.wayvr
+            .borrow_mut()
+            .data
+            .state
+            .send_mouse_scroll(delta_y, delta_x);
+    }
+}
+
+struct ImageData {
+    vk_image: Arc<Image>,
+    vk_image_view: Arc<ImageView>,
+}
+
+pub struct WayVRRenderer {
+    pipeline: Arc<WGfxPipeline<Vert2Uv>>,
+    image: Option<ImageData>,
+    context: Rc<RefCell<WayVRContext>>,
+    graphics: Arc<WGfx>,
+    resolution: [u16; 2],
+}
+
+impl WayVRRenderer {
+    pub fn new(
+        app: &state::AppState,
+        wvr: Rc<RefCell<WayVRData>>,
+        display: wayvr::display::DisplayHandle,
+        resolution: [u16; 2],
+    ) -> anyhow::Result<Self> {
+        let pipeline = app.gfx.create_pipeline(
+            app.gfx_extras.shaders.get("vert_quad").unwrap().clone(), // want panic
+            app.gfx_extras.shaders.get("frag_srgb").unwrap().clone(), // want panic
+            app.gfx.surface_format,
+            None,
+            PrimitiveTopology::TriangleStrip,
+            false,
+        )?;
+
+        Ok(Self {
+            pipeline,
+            context: Rc::new(RefCell::new(WayVRContext::new(wvr, display))),
+            graphics: app.gfx.clone(),
+            image: None,
+            resolution,
+        })
+    }
+}
+
+fn get_or_create_display_by_name(
+    app: &mut AppState,
+    wayvr: &mut WayVRData,
+    disp_name: &str,
+) -> anyhow::Result<display::DisplayHandle> {
+    let disp_handle =
+        if let Some(disp) = WayVR::get_display_by_name(&wayvr.data.state.displays, disp_name) {
+            disp
+        } else {
+            let conf_display = app
+                .session
+                .wayvr_config
+                .get_display(disp_name)
+                .ok_or_else(|| anyhow::anyhow!("Cannot find display named \"{}\"", disp_name))?
+                .clone();
+
+            let disp_handle = wayvr.data.state.create_display(
+                conf_display.width,
+                conf_display.height,
+                disp_name,
+                conf_display.primary.unwrap_or(false),
+            )?;
+
+            wayvr.overlays_to_create.push(OverlayToCreate {
+                conf_display,
+                disp_handle,
+            });
+
+            disp_handle
+        };
+
+    Ok(disp_handle)
+}
+
+pub fn executable_exists_in_path(command: &str) -> bool {
+    let Ok(path) = std::env::var("PATH") else {
+        return false; // very unlikely to happen
+    };
+    for dir in path.split(':') {
+        let exec_path = std::path::PathBuf::from(dir).join(command);
+        if exec_path.exists() && exec_path.is_file() {
+            return true; // executable found
+        }
+    }
+    false
+}
+
+fn toggle_dashboard<O>(
+    app: &mut AppState,
+    overlays: &mut OverlayContainer<O>,
+    wayvr: &mut WayVRData,
+) -> anyhow::Result<()>
+where
+    O: Default,
+{
+    let Some(conf_dash) = app.session.wayvr_config.dashboard.clone() else {
+        anyhow::bail!("Dashboard is not configured");
+    };
+
+    if !wayvr.dashboard_executed && !executable_exists_in_path(&conf_dash.exec) {
+        anyhow::bail!("Executable \"{}\" not found", &conf_dash.exec);
+    }
+
+    let (newly_created, disp_handle) = wayvr.data.state.get_or_create_dashboard_display(
+        DASHBOARD_WIDTH,
+        DASHBOARD_HEIGHT,
+        DASHBOARD_DISPLAY_NAME,
+    )?;
+
+    if newly_created {
+        log::info!("Creating dashboard overlay");
+
+        let mut overlay = create_overlay::<O>(
+            app,
+            wayvr,
+            DASHBOARD_DISPLAY_NAME,
+            OverlayToCreate {
+                disp_handle,
+                conf_display: config_wayvr::WayVRDisplay {
+                    attach_to: None,
+                    width: DASHBOARD_WIDTH,
+                    height: DASHBOARD_HEIGHT,
+                    scale: None,
+                    rotation: None,
+                    pos: None,
+                    primary: None,
+                },
+            },
+        )?;
+
+        overlay.state.curvature = Some(0.15);
+        overlay.state.want_visible = true;
+        overlay.state.spawn_scale = 2.0;
+        overlay.state.spawn_point = vec3a(0.0, -0.35, -1.75);
+        overlay.state.z_order = Z_ORDER_DASHBOARD;
+        overlay.state.reset(app, true);
+
+        overlays.add(overlay);
+
+        let args_vec = &conf_dash
+            .args
+            .as_ref()
+            .map_or_else(Vec::new, |args| gen_args_vec(args.as_str()));
+
+        let env_vec = &conf_dash
+            .env
+            .as_ref()
+            .map_or_else(Vec::new, |env| gen_env_vec(env));
+
+        let mut userdata = HashMap::new();
+        userdata.insert(String::from("type"), String::from("dashboard"));
+
+        // Start dashboard specified in the WayVR config
+        let _process_handle_unused = wayvr.data.state.spawn_process(
+            disp_handle,
+            &conf_dash.exec,
+            args_vec,
+            env_vec,
+            conf_dash.working_dir.as_deref(),
+            userdata,
+        )?;
+
+        wayvr.dashboard_executed = true;
+
+        return Ok(());
+    }
+
+    let display = wayvr.data.state.displays.get(&disp_handle).unwrap(); // safe
+    let Some(overlay_id) = display.overlay_id else {
+        anyhow::bail!("Overlay ID not set for dashboard display");
+    };
+
+    let cur_visibility = !display.visible;
+
+    wayvr
+        .data
+        .ipc_server
+        .broadcast(PacketServer::WvrStateChanged(if cur_visibility {
+            WvrStateChanged::DashboardShown
+        } else {
+            WvrStateChanged::DashboardHidden
+        }));
+
+    app.tasks.enqueue(TaskType::Overlay(
+        OverlaySelector::Id(overlay_id),
+        Box::new(move |app, o| {
+            // Toggle visibility
+            o.want_visible = cur_visibility;
+            if cur_visibility {
+                o.reset(app, true);
+            }
+        }),
+    ));
+
+    Ok(())
+}
+
+fn create_overlay<O>(
+    app: &mut AppState,
+    data: &mut WayVRData,
+    name: &str,
+    cell: OverlayToCreate,
+) -> anyhow::Result<OverlayData<O>>
+where
+    O: Default,
+{
+    let conf_display = &cell.conf_display;
+    let disp_handle = cell.disp_handle;
+
+    let mut overlay = create_wayvr_display_overlay::<O>(
+        app,
+        conf_display.width,
+        conf_display.height,
+        disp_handle,
+        conf_display.scale.unwrap_or(1.0),
+        name,
+    )?;
+
+    data.display_handle_map
+        .insert(disp_handle, overlay.state.id);
+
+    if let Some(attach_to) = &conf_display.attach_to {
+        overlay.state.positioning = attach_to.get_positioning();
+    }
+
+    if let Some(rot) = &conf_display.rotation {
+        overlay.state.spawn_rotation =
+            glam::Quat::from_axis_angle(Vec3::from_slice(&rot.axis), f32::to_radians(rot.angle));
+    }
+
+    if let Some(pos) = &conf_display.pos {
+        overlay.state.spawn_point = Vec3A::from_slice(pos);
+    }
+
+    let display = data.data.state.displays.get_mut(&disp_handle).unwrap(); // Never fails
+    display.overlay_id = Some(overlay.state.id);
+
+    Ok(overlay)
+}
+
+fn create_queued_displays<O>(
+    app: &mut AppState,
+    data: &mut WayVRData,
+    overlays: &mut OverlayContainer<O>,
+) -> anyhow::Result<()>
+where
+    O: Default,
+{
+    let overlays_to_create = std::mem::take(&mut data.overlays_to_create);
+
+    for cell in overlays_to_create {
+        let Some(disp) = data.data.state.displays.get(&cell.disp_handle) else {
+            continue; // this shouldn't happen
+        };
+
+        let name = disp.name.clone();
+
+        let overlay = create_overlay::<O>(app, data, name.as_str(), cell)?;
+        overlays.add(overlay); // Insert freshly created WayVR overlay into wlx stack
+    }
+
+    Ok(())
+}
+
+#[allow(clippy::too_many_lines)]
+pub fn tick_events<O>(app: &mut AppState, overlays: &mut OverlayContainer<O>) -> anyhow::Result<()>
+where
+    O: Default,
+{
+    let Some(r_wayvr) = app.wayvr.clone() else {
+        return Ok(());
+    };
+
+    let mut wayvr = r_wayvr.borrow_mut();
+
+    while let Some(signal) = wayvr.data.state.signals.read() {
+        match signal {
+            wayvr::WayVRSignal::DisplayVisibility(display_handle, visible) => {
+                if let Some(overlay_id) = wayvr.display_handle_map.get(&display_handle) {
+                    let overlay_id = *overlay_id;
+                    wayvr
+                        .data
+                        .state
+                        .set_display_visible(display_handle, visible);
+                    app.tasks.enqueue(TaskType::Overlay(
+                        OverlaySelector::Id(overlay_id),
+                        Box::new(move |_app, o| {
+                            o.want_visible = visible;
+                        }),
+                    ));
+                }
+            }
+            wayvr::WayVRSignal::DisplayWindowLayout(display_handle, layout) => {
+                wayvr.data.state.set_display_layout(display_handle, layout);
+            }
+            wayvr::WayVRSignal::BroadcastStateChanged(packet) => {
+                wayvr
+                    .data
+                    .ipc_server
+                    .broadcast(packet_server::PacketServer::WvrStateChanged(packet));
+            }
+            wayvr::WayVRSignal::DropOverlay(overlay_id) => {
+                app.tasks
+                    .enqueue(TaskType::DropOverlay(OverlaySelector::Id(overlay_id)));
+            }
+            wayvr::WayVRSignal::Haptics(haptics) => {
+                wayvr.pending_haptics = Some(haptics);
+            }
+        }
+    }
+
+    let res = wayvr.data.tick_events(app)?;
+    drop(wayvr);
+
+    for result in res {
+        match result {
+            wayvr::TickTask::NewExternalProcess(request) => {
+                let config = &app.session.wayvr_config;
+
+                let disp_name = request.env.display_name.map_or_else(
+                    || {
+                        config
+                            .get_default_display()
+                            .map(|(display_name, _)| display_name)
+                    },
+                    |display_name| {
+                        config
+                            .get_display(display_name.as_str())
+                            .map(|_| display_name)
+                    },
+                );
+
+                if let Some(disp_name) = disp_name {
+                    let mut wayvr = r_wayvr.borrow_mut();
+
+                    log::info!("Registering external process with PID {}", request.pid);
+
+                    let disp_handle = get_or_create_display_by_name(app, &mut wayvr, &disp_name)?;
+
+                    wayvr
+                        .data
+                        .state
+                        .add_external_process(disp_handle, request.pid);
+
+                    wayvr
+                        .data
+                        .state
+                        .manager
+                        .add_client(wayvr::client::WayVRClient {
+                            client: request.client,
+                            display_handle: disp_handle,
+                            pid: request.pid,
+                        });
+                }
+            }
+            wayvr::TickTask::NewDisplay(cpar, disp_handle) => {
+                log::info!("Creating new display with name \"{}\"", cpar.name);
+
+                let mut wayvr = r_wayvr.borrow_mut();
+
+                let unique_name = wayvr.get_unique_display_name(cpar.name);
+
+                let disp_handle = match disp_handle {
+                    Some(d) => d,
+                    None => wayvr.data.state.create_display(
+                        cpar.width,
+                        cpar.height,
+                        &unique_name,
+                        false,
+                    )?,
+                };
+
+                wayvr.overlays_to_create.push(OverlayToCreate {
+                    disp_handle,
+                    conf_display: config_wayvr::WayVRDisplay {
+                        attach_to: Some(config_wayvr::AttachTo::from_packet(&cpar.attach_to)),
+                        width: cpar.width,
+                        height: cpar.height,
+                        pos: None,
+                        primary: None,
+                        rotation: None,
+                        scale: cpar.scale,
+                    },
+                });
+            }
+        }
+    }
+
+    let mut wayvr = r_wayvr.borrow_mut();
+    create_queued_displays(app, &mut wayvr, overlays)?;
+
+    Ok(())
+}
+
+impl WayVRRenderer {
+    fn ensure_software_data(
+        &mut self,
+        data: &wayvr::egl_data::RenderSoftwarePixelsData,
+    ) -> anyhow::Result<()> {
+        let mut upload = self
+            .graphics
+            .create_xfer_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+
+        let tex = upload.upload_image(
+            u32::from(data.width),
+            u32::from(data.height),
+            Format::R8G8B8A8_UNORM,
+            &data.data,
+        )?;
+
+        // FIXME: can we use _buffers_ here?
+        upload.build_and_execute_now()?;
+
+        //buffers.push(upload.build()?);
+        self.image = Some(ImageData {
+            vk_image: tex.clone(),
+            vk_image_view: ImageView::new_default(tex).unwrap(),
+        });
+        Ok(())
+    }
+
+    fn ensure_dmabuf_data(
+        &mut self,
+        data: &wayvr::egl_data::RenderDMAbufData,
+    ) -> anyhow::Result<()> {
+        if self.image.is_some() {
+            return Ok(()); // already initialized and automatically updated due to direct zero-copy textue access
+        }
+
+        // First init
+        let mut planes = [FramePlane::default(); 4];
+        planes[0].fd = Some(data.fd);
+        planes[0].offset = data.offset as u32;
+        planes[0].stride = data.stride;
+
+        let ctx = self.context.borrow_mut();
+        let wayvr = ctx.wayvr.borrow_mut();
+        let Some(disp) = wayvr.data.state.displays.get(&ctx.display) else {
+            anyhow::bail!("Failed to fetch WayVR display")
+        };
+
+        let frame = DmabufFrame {
+            format: FrameFormat {
+                width: u32::from(disp.width),
+                height: u32::from(disp.height),
+                fourcc: FourCC {
+                    value: data.mod_info.fourcc,
+                },
+                modifier: data.mod_info.modifiers[0], /* possibly not proper? */
+                ..Default::default()
+            },
+            num_planes: 1,
+            planes,
+            ..Default::default()
+        };
+
+        drop(wayvr);
+
+        let layouts: Vec<SubresourceLayout> = vec![SubresourceLayout {
+            offset: data.offset as _,
+            size: 0,
+            row_pitch: data.stride as _,
+            array_pitch: None,
+            depth_pitch: None,
+        }];
+
+        let tex = self.graphics.dmabuf_texture_ex(
+            frame,
+            ImageTiling::DrmFormatModifier,
+            layouts,
+            &data.mod_info.modifiers,
+        )?;
+
+        self.image = Some(ImageData {
+            vk_image: tex.clone(),
+            vk_image_view: ImageView::new_default(tex).unwrap(),
+        });
+        Ok(())
+    }
+}
+
+impl OverlayRenderer for WayVRRenderer {
+    fn init(&mut self, _app: &mut state::AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+
+    fn pause(&mut self, _app: &mut state::AppState) -> anyhow::Result<()> {
+        let ctx = self.context.borrow_mut();
+        let wayvr = &mut ctx.wayvr.borrow_mut().data;
+        wayvr.state.set_display_visible(ctx.display, false);
+        Ok(())
+    }
+
+    fn resume(&mut self, _app: &mut state::AppState) -> anyhow::Result<()> {
+        let ctx = self.context.borrow_mut();
+        let wayvr = &mut ctx.wayvr.borrow_mut().data;
+        wayvr.state.set_display_visible(ctx.display, true);
+        Ok(())
+    }
+
+    fn should_render(&mut self, _app: &mut AppState) -> anyhow::Result<ShouldRender> {
+        let ctx = self.context.borrow();
+        let mut wayvr = ctx.wayvr.borrow_mut();
+        let redrawn = match wayvr.data.render_display(ctx.display) {
+            Ok(r) => r,
+            Err(e) => {
+                log::error!("render_display failed: {e}");
+                return Ok(ShouldRender::Unable);
+            }
+        };
+
+        if redrawn {
+            Ok(ShouldRender::Should)
+        } else {
+            Ok(ShouldRender::Can)
+        }
+    }
+
+    fn render(
+        &mut self,
+        app: &mut state::AppState,
+        tgt: Arc<ImageView>,
+        buf: &mut CommandBuffers,
+        alpha: f32,
+    ) -> anyhow::Result<bool> {
+        let ctx = self.context.borrow();
+        let wayvr = ctx.wayvr.borrow_mut();
+
+        let data = wayvr
+            .data
+            .state
+            .get_render_data(ctx.display)
+            .ok_or_else(|| anyhow::anyhow!("Failed to fetch render data"))?
+            .clone();
+
+        drop(wayvr);
+        drop(ctx);
+
+        match data {
+            wayvr::egl_data::RenderData::Dmabuf(data) => {
+                self.ensure_dmabuf_data(&data)?;
+            }
+            wayvr::egl_data::RenderData::Software(data) => {
+                if let Some(new_frame) = &data {
+                    self.ensure_software_data(new_frame)?;
+                }
+            }
+        }
+
+        let Some(image) = self.image.as_ref() else {
+            return Ok(false);
+        };
+
+        let set0 = self.pipeline.uniform_sampler(
+            0,
+            image.vk_image_view.clone(),
+            app.gfx.texture_filter,
+        )?;
+
+        let set1 = self.pipeline.uniform_buffer_upload(1, vec![alpha])?;
+
+        let pass = self.pipeline.create_pass(
+            tgt.extent_f32(),
+            app.gfx_extras.quad_verts.clone(),
+            0..4,
+            0..1,
+            vec![set0, set1],
+        )?;
+
+        let mut cmd_buffer = app
+            .gfx
+            .create_gfx_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+        cmd_buffer.begin_rendering(tgt)?;
+        cmd_buffer.run_ref(&pass)?;
+        cmd_buffer.end_rendering()?;
+        buf.push(cmd_buffer.build()?);
+
+        Ok(true)
+    }
+
+    fn frame_meta(&mut self) -> Option<FrameMeta> {
+        Some(FrameMeta {
+            extent: [self.resolution[0] as u32, self.resolution[1] as u32, 1],
+            ..Default::default()
+        })
+    }
+}
+
+#[allow(dead_code)]
+pub fn create_wayvr_display_overlay<O>(
+    app: &mut state::AppState,
+    display_width: u16,
+    display_height: u16,
+    display_handle: wayvr::display::DisplayHandle,
+    display_scale: f32,
+    name: &str,
+) -> anyhow::Result<OverlayData<O>>
+where
+    O: Default,
+{
+    let transform = ui_transform([u32::from(display_width), u32::from(display_height)]);
+
+    let state = OverlayState {
+        name: format!("WayVR - {name}").into(),
+        keyboard_focus: Some(KeyboardFocus::WayVR),
+        want_visible: true,
+        interactable: true,
+        grabbable: true,
+        spawn_scale: display_scale,
+        spawn_point: vec3a(0.0, -0.1, -1.0),
+        interaction_transform: transform,
+        ..Default::default()
+    };
+
+    let wayvr = app.get_wayvr()?;
+
+    let renderer = WayVRRenderer::new(app, wayvr, display_handle, [display_width, display_height])?;
+    let context = renderer.context.clone();
+
+    let backend = Box::new(SplitOverlayBackend {
+        renderer: Box::new(renderer),
+        interaction: Box::new(WayVRInteractionHandler::new(context, Affine2::IDENTITY)),
+    });
+
+    Ok(OverlayData {
+        state,
+        backend,
+        ..Default::default()
+    })
+}
+
+fn show_display<O>(wayvr: &mut WayVRData, overlays: &mut OverlayContainer<O>, display_name: &str)
+where
+    O: Default,
+{
+    if let Some(display) = WayVR::get_display_by_name(&wayvr.data.state.displays, display_name) {
+        if let Some(overlay_id) = wayvr.display_handle_map.get(&display) {
+            if let Some(overlay) = overlays.mut_by_id(*overlay_id) {
+                overlay.state.want_visible = true;
+            }
+        }
+
+        wayvr.data.state.set_display_visible(display, true);
+    }
+}
+
+fn action_app_click<O>(
+    app: &mut AppState,
+    overlays: &mut OverlayContainer<O>,
+    catalog_name: &Arc<str>,
+    app_name: &Arc<str>,
+) -> anyhow::Result<()>
+where
+    O: Default,
+{
+    let wayvr = app.get_wayvr()?;
+
+    let catalog = app
+        .session
+        .wayvr_config
+        .get_catalog(catalog_name)
+        .ok_or_else(|| anyhow::anyhow!("Failed to get catalog \"{}\"", catalog_name))?
+        .clone();
+
+    if let Some(app_entry) = catalog.get_app(app_name) {
+        let mut wayvr = wayvr.borrow_mut();
+
+        let disp_handle = get_or_create_display_by_name(
+            app,
+            &mut wayvr,
+            &app_entry.target_display.to_lowercase(),
+        )?;
+
+        let args_vec = &app_entry
+            .args
+            .as_ref()
+            .map_or_else(Vec::new, |args| gen_args_vec(args.as_str()));
+
+        let env_vec = &app_entry
+            .env
+            .as_ref()
+            .map_or_else(Vec::new, |env| gen_env_vec(env));
+
+        // Terminate existing process if required
+        if let Some(process_handle) =
+            wayvr
+                .data
+                .state
+                .process_query(disp_handle, &app_entry.exec, args_vec, env_vec)
+        {
+            // Terminate process
+            wayvr.data.terminate_process(process_handle);
+        } else {
+            // Spawn process
+            wayvr.data.state.spawn_process(
+                disp_handle,
+                &app_entry.exec,
+                args_vec,
+                env_vec,
+                None,
+                HashMap::default(),
+            )?;
+
+            show_display::<O>(&mut wayvr, overlays, app_entry.target_display.as_str());
+        }
+    }
+
+    Ok(())
+}
+
+pub fn action_display_click<O>(
+    app: &mut AppState,
+    overlays: &mut OverlayContainer<O>,
+    display_name: &Arc<str>,
+    action: &WayVRDisplayClickAction,
+) -> anyhow::Result<()>
+where
+    O: Default,
+{
+    let wayvr = app.get_wayvr()?;
+    let mut wayvr = wayvr.borrow_mut();
+
+    let Some(handle) = WayVR::get_display_by_name(&wayvr.data.state.displays, display_name) else {
+        return Ok(());
+    };
+
+    let Some(display) = wayvr.data.state.displays.get_mut(&handle) else {
+        return Ok(());
+    };
+
+    let Some(overlay_id) = display.overlay_id else {
+        return Ok(());
+    };
+
+    let Some(overlay) = overlays.mut_by_id(overlay_id) else {
+        return Ok(());
+    };
+
+    match action {
+        WayVRDisplayClickAction::ToggleVisibility => {
+            // Toggle visibility
+            overlay.state.want_visible = !overlay.state.want_visible;
+        }
+        WayVRDisplayClickAction::Reset => {
+            // Show it at the front
+            overlay.state.want_visible = true;
+            overlay.state.reset(app, true);
+        }
+    }
+
+    Ok(())
+}
+
+pub fn wayvr_action<O>(app: &mut AppState, overlays: &mut OverlayContainer<O>, action: &WayVRAction)
+where
+    O: Default,
+{
+    match action {
+        WayVRAction::AppClick {
+            catalog_name,
+            app_name,
+        } => {
+            if let Err(e) = action_app_click(app, overlays, catalog_name, app_name) {
+                // Happens if something went wrong with initialization
+                // or input exec path is invalid. Do nothing, just print an error
+                error_toast(app, "action_app_click failed", e);
+            }
+        }
+        WayVRAction::DisplayClick {
+            display_name,
+            action,
+        } => {
+            if let Err(e) = action_display_click::<O>(app, overlays, display_name, action) {
+                error_toast(app, "action_display_click failed", e);
+            }
+        }
+        WayVRAction::ToggleDashboard => {
+            let wayvr = match app.get_wayvr() {
+                Ok(wayvr) => wayvr,
+                Err(e) => {
+                    log::error!("WayVR Error: {e:?}");
+                    return;
+                }
+            };
+
+            let mut wayvr = wayvr.borrow_mut();
+
+            if let Err(e) = toggle_dashboard::<O>(app, overlays, &mut wayvr) {
+                error_toast(app, "toggle_dashboard failed", e);
+            }
+        }
+    }
+}

wlx-overlay-s/src/res/actions.json

@@ -0,0 +1,93 @@
+{
+  "actions": [
+    {
+      "name": "/actions/default/in/Click",
+      "type": "boolean",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/Grab",
+      "type": "boolean",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/Scroll",
+      "type": "vector2",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/ShowHide",
+      "type": "boolean",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/AltClick",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/ClickModifierRight",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/ClickModifierMiddle",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/MoveMouse",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/SpaceDrag",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/SpaceRotate",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/LeftHand",
+      "type": "pose",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/RightHand",
+      "type": "pose",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/out/HapticsLeft",
+      "type": "vibration"
+    },
+    {
+      "name": "/actions/default/out/HapticsRight",
+      "type": "vibration"
+    }
+  ],
+  "action_sets": [
+    {
+      "name": "/actions/default",
+      "usage": "leftright"
+    }
+  ],
+  "default_bindings": [
+    {
+      "controller_type": "knuckles",
+      "binding_url": "actions_binding_knuckles.json"
+    },
+    {
+      "controller_type": "oculus_touch",
+      "binding_url": "actions_binding_oculus.json"
+    },
+    {
+      "controller_type": "vive_controller",
+      "binding_url": "actions_binding_vive.json"
+    }
+  ],
+  "localization": []
+}

wlx-overlay-s/src/res/actions_binding_knuckles.json

@@ -0,0 +1,179 @@
+{
+   "action_manifest_version" : 0,
+   "app_key" : "galister.wlxoverlay-s",
+   "bindings" : {
+      "/actions/default" : {
+         "haptics" : [
+            {
+               "output" : "/actions/default/out/hapticsleft",
+               "path" : "/user/hand/left/output/haptic"
+            },
+            {
+               "output" : "/actions/default/out/hapticsright",
+               "path" : "/user/hand/right/output/haptic"
+            }
+         ],
+         "poses" : [
+            {
+               "output" : "/actions/default/in/lefthand",
+               "path" : "/user/hand/left/pose/tip"
+            },
+            {
+               "output" : "/actions/default/in/righthand",
+               "path" : "/user/hand/right/pose/tip"
+            }
+         ],
+         "sources" : [
+            {
+               "inputs" : {
+                  "double" : {
+                     "output" : "/actions/default/in/showhide"
+                  },
+                  "touch" : {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/b"
+            },
+            {
+               "path": "/user/hand/left/input/a",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/b",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/a",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  },
+                  "touch": {
+                     "output": "/actions/default/in/movemouse"
+                  }
+               },
+               "mode" : "button",
+               "parameters" : {
+                  "click_activate_threshold" : "0.35",
+                  "click_deactivate_threshold" : "0.31"
+               },
+               "path" : "/user/hand/left/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  },
+                  "touch": {
+                     "output": "/actions/default/in/movemouse"
+                  }
+               },
+               "mode" : "button",
+               "parameters" : {
+                  "click_activate_threshold" : "0.35",
+                  "click_deactivate_threshold" : "0.31"
+               },
+               "path" : "/user/hand/right/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/altclick"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/right/input/trackpad"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/spacedrag"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/trackpad"
+            },
+            {
+               "inputs" : {
+                  "grab" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode" : "grab",
+               "parameters" : {
+                  "value_hold_threshold" : "1.3",
+                  "value_release_threshold" : "1.1"
+               },
+               "path" : "/user/hand/left/input/grip"
+            },
+            {
+               "inputs" : {
+                  "grab" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode" : "grab",
+               "parameters" : {
+                  "value_hold_threshold" : "1.3",
+                  "value_release_threshold" : "1.1"
+               },
+               "path" : "/user/hand/right/input/grip"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/left/input/thumbstick"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/right/input/thumbstick"
+            }
+         ]
+      }
+   },
+   "category" : "steamvr_input",
+   "controller_type" : "knuckles",
+   "description" : "Ver1",
+   "interaction_profile" : "",
+   "name" : "WlxOverlay configuration for Index Controller",
+   "options" : {
+      "mirror_actions" : false,
+      "simulated_controller_type" : "none"
+   },
+   "simulated_actions" : []
+}

wlx-overlay-s/src/res/actions_binding_oculus.json

@@ -0,0 +1,145 @@
+{
+   "action_manifest_version" : 0,
+   "app_key" : "galister.wlxoverlay-s",
+   "bindings" : {
+      "/actions/default" : {
+         "haptics" : [
+            {
+               "output" : "/actions/default/out/hapticsleft",
+               "path" : "/user/hand/left/output/haptic"
+            },
+            {
+               "output" : "/actions/default/out/hapticsright",
+               "path" : "/user/hand/right/output/haptic"
+            }
+         ],
+         "poses" : [
+            {
+               "output" : "/actions/default/in/lefthand",
+               "path" : "/user/hand/left/pose/tip"
+            },
+            {
+               "output" : "/actions/default/in/righthand",
+               "path" : "/user/hand/right/pose/tip"
+            }
+         ],
+         "sources" : [
+            {
+               "inputs" : {
+                  "double" : {
+                     "output" : "/actions/default/in/showhide"
+                  },
+                  "touch" : {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/y"
+            },
+            {
+               "path": "/user/hand/left/input/x",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/b",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/a",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  },
+                  "touch": {
+                     "output": "/actions/default/in/movemouse"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/left/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  },
+                  "touch": {
+                     "output": "/actions/default/in/movemouse"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/right/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/left/input/grip"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/right/input/grip"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/left/input/joystick"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/right/input/joystick"
+            }
+         ]
+      }
+   },
+   "category" : "steamvr_input",
+   "controller_type" : "oculus_touch",
+   "description" : "Ver1",
+   "interaction_profile" : "",
+   "name" : "WlxOverlay configuration for Oculus Touch Controller",
+   "options" : {
+      "mirror_actions" : false,
+      "simulated_controller_type" : "none"
+   },
+   "simulated_actions" : []
+}

wlx-overlay-s/src/res/actions_binding_vive.json

@@ -0,0 +1,139 @@
+{
+   "action_manifest_version" : 0,
+   "app_key" : "galister.wlxoverlay-s",
+   "bindings" : {
+      "/actions/default": {
+         "poses": [
+            {
+               "path": "/user/hand/left/pose/tip",
+               "output": "/actions/default/in/lefthand"
+            },
+            {
+               "path": "/user/hand/right/pose/tip",
+               "output": "/actions/default/in/righthand"
+            }
+         ],
+         "haptics": [
+            {
+               "output": "/actions/default/out/hapticsleft",
+               "path": "/user/hand/left/output/haptic"
+            },
+            {
+               "output": "/actions/default/out/hapticsright",
+               "path": "/user/hand/right/output/haptic"
+            }
+         ],
+         "sources": [
+            {
+               "path": "/user/hand/left/input/grip",
+               "mode": "button",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/grab"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/grip",
+               "mode": "button",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/grab"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/trigger",
+               "mode": "button",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/click"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/trigger",
+               "mode": "trigger",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/click"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/trackpad",
+               "mode": "scroll",
+               "parameters": {
+                  "scroll_mode": "discrete"
+               },
+               "inputs": {
+                  "scroll": {
+                     "output": "/actions/default/in/scroll"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/trackpad",
+               "mode": "scroll",
+               "parameters": {
+                  "scroll_mode": "discrete"
+               },
+               "inputs": {
+                  "scroll": {
+                     "output": "/actions/default/in/scroll"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/application_menu",
+               "mode": "button",
+               "inputs": {
+                  "double": {
+                     "output": "/actions/default/in/showhide"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/trackpad",
+               "mode": "dpad",
+               "parameters": {
+                  "sub_mode": "touch"
+               },
+               "inputs": {
+                  "west": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  },
+                  "east": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/trackpad",
+               "mode": "dpad",
+               "parameters": {
+                  "sub_mode": "touch"
+               },
+               "inputs": {
+                  "west": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  },
+                  "east": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            }
+         ]
+      }
+   },
+   "category" : "steamvr_input",
+   "controller_type" : "vive_controller",
+   "description" : "Ver1",
+   "interaction_profile" : "",
+   "name" : "WlxOverlay configuration for Vive Controller",
+   "options" : {
+      "mirror_actions" : false,
+      "simulated_controller_type" : "none"
+   },
+   "simulated_actions" : []
+}

wlx-overlay-s/src/res/anchor.yaml

@@ -0,0 +1,29 @@
+# looking to make changes? 
+# drop me in ~/.config/wlxoverlay/anchor.yaml
+#
+
+width: 0.1
+
+size: [200, 200]
+
+# +X: right, +Y: up, +Z: back
+spawn_pos: [0, 0, -1]
+
+elements:
+  - type: Panel
+    rect: [98, 0, 4, 200]
+    corner_radius: 0
+    bg_color: "#ffff00"
+
+  - type: Panel
+    rect: [0, 98, 200, 4]
+    corner_radius: 0
+    bg_color: "#ffff00"
+
+  - type: Label
+    rect: [8, 90, 600, 70]
+    corner_radius: 0
+    font_size: 18
+    fg_color: "#ffff00"
+    source: Static
+    text: Center

wlx-overlay-s/src/res/config.yaml

@@ -0,0 +1,27 @@
+# For how much time mouse motion events should be stopped after clicking?
+# Prevents accidental dragging various GUI elements or links, making it easier to click
+# Default: 300
+click_freeze_time_ms: 300
+
+# Default: true
+keyboard_sound_enabled: true
+
+# Alter default scale of various overlays
+# Default: 1.0
+keyboard_scale: 1.0
+desktop_view_scale: 1.0
+watch_scale: 1.0
+
+# Enable / disable sliding windows back and forth with the scroll action
+# Default: true
+allow_sliding: true
+
+# Enable / disable realigning the working set windows when they are shown/hidden
+# Default: true
+realign_on_showhide: true
+
+# When enabled, the mouse pointer will not be moved on the screen, unless the trigger is touched
+# allowing for moving both pointers off the screens to the keyboard, while keeping the cursor position
+# unchanged, for when the desktop is configured to move the focus with the mouse cursor
+# Default: false
+focus_follows_mouse_mode: false

wlx-overlay-s/src/res/keyboard.yaml

@@ -0,0 +1,121 @@
+---
+# looking to make changes? 
+# drop me in ~/.config/wlxoverlay/keyboard.yaml
+
+# This file contains all data needed to generate the keyboard.
+# You can create any layout, as long as:
+# - All keys are rectangular with 1 unit of height.
+# This means:
+# - We're limited to the flat & boring ANSI enter key.
+# - Numpad + and Enter might not look so great.
+
+# *** Important ***
+# The keyboard layout uses virtual key codes, so they are layout-independent.
+# For example, Q on a French layout actually results in A.
+# If you're using a non-english layout, chances are you only need to edit the label section below.
+
+# Not used for anything right now
+name: "en-us_full"
+
+# How many units of key size in each row? 1 = standard letter key size
+row_size: 23
+
+# Specifies the size of each key. The sum of any given row must equal RowSize
+key_sizes:
+    - [1.5,0.5, 1, 1, 1, 1,0.5,1, 1, 1, 1,0.5,1, 1, 1, 1, 0.5, 1, 1, 1, 0.5, 1, 1, 1, 1]
+    - [1,   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2,     0.5, 1, 1, 1, 0.5,    1, 1, 1, 1]
+    - [1.5,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.5,  0.5, 1, 1, 1, 0.5,    1, 1, 1, 1]
+    - [1.75,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2.25,   4,                    1, 1, 1, 1]
+    - [1.25, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2.75,    1.5,    1, 1.5,       1, 1, 1, 1]
+    - [1.25, 1.25, 1.25, 6.25, 1.25, 1.25, 1.25, 1.25, 0.5, 1, 1, 1, 0.5,    2,    1, 1]
+
+# The main (blue) layout of the keyboard.
+# Accepted are:
+# - virtual keys. For a full list, look at enum VirtualKey in https://github.com/galister/wlx-overlay-s/blob/main/src/hid.rs
+# - exec_commands (defined below)
+# - macros (defined below)
+# - ~ (null) will leave an empty space with the corresponding size from key_sizes
+main_layout:
+    - ["Escape", ~, "F1", "F2", "F3", "F4", ~, "F5", "F6", "F7", "F8", ~, "F9", "F10", "F11", "F12", ~, "Print", "Scroll", "Pause", ~, "COPY", "PASTE", ~, "KILL"]
+    - ["Oem3", "N1", "N2", "N3", "N4", "N5", "N6", "N7", "N8", "N9", "N0", "Minus", "Plus", "BackSpace", ~, "Insert", "Home", "Prior", ~, "NumLock", "KP_Divide", "KP_Multiply", "KP_Subtract"]
+    - ["Tab", "Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P", "Oem4", "Oem6", "Oem5", ~, "Delete", "End", "Next", ~, "KP_7", "KP_8", "KP_9", "KP_Add"]
+    - ["XF86Favorites", "A", "S", "D", "F", "G", "H", "J", "K", "L", "Oem1", "Oem7", "Return", ~, "KP_4", "KP_5", "KP_6", ~]
+    - ["LShift", "Oem102", "Z", "X", "C", "V", "B", "N", "M", "Comma", "Period", "Oem2", "RShift", ~, "Up", ~, "KP_1", "KP_2", "KP_3", "KP_Enter"]
+    - ["LCtrl", "LSuper", "LAlt", "Space", "Meta", "RSuper", "Menu", "RCtrl", ~, "Left", "Down", "Right", ~, "KP_0", "KP_Decimal", ~]
+
+# When using the purple pointer...
+# None   - No special functionality when using purple pointer (Default)
+# Shift  - Use same functionality as the orange pointer
+# Ctrl   - Use Main layout with Ctrl modifier
+# Alt    - Use Main layout with Alt modifier
+# Super  - Use Main layout with Super (WinKey) modifier
+# Meta   - Use Main layout with Meta (AltGr) modifier
+alt_modifier: None
+
+# Shell commands to be used in a layout.
+# Value is an array of string arguments.
+exec_commands:
+    STT: [ "whisper_stt", "--lang", "en" ]
+
+# Series of keypresses to be used in a layout.
+# Format: keyName [DOWN|UP]
+# keyName must be a valid virtual key from the VirtualKey enum (see above)
+# DOWN|UP: can be omitted for an implicit "keyName DOWN, keyName UP"
+macros:
+    KILL: [ "LSuper DOWN", "LCtrl DOWN", "Escape", "LCtrl UP", "LSuper UP" ]
+    COPY: [ "LCtrl DOWN", "C", "LCtrl UP" ]
+    PASTE: [ "LCtrl DOWN", "V", "LCtrl UP" ]
+
+# Custom labels to use.
+# Key: element of main_layout
+# Value: Array of strings. 0th element is the upper row, 1st element is lower row.
+# For empty labels, use [] (do not use ~)
+labels:
+    "Escape":         ["Esc"]
+    "Prior":          ["PgUp"]
+    "Next":           ["PgDn"]
+    "NumLock":        ["Num"]
+    "Space":          []
+    "LAlt":           ["Alt"]
+    "LCtrl":          ["Ctrl"]
+    "RCtrl":          ["Ctrl"]
+    "LSuper":         ["Super"]
+    "RSuper":         ["Super"]
+    "LShift":         ["Shift"]
+    "RShift":         ["Shift"]
+    "Insert":         ["Ins"]
+    "Delete":         ["Del"]
+    "BackSpace":      ["<<"]
+    "KP_Divide":      [" /"]
+    "KP_Add":         [" +"]
+    "KP_Multiply":    [" *"]
+    "KP_Decimal":     [" ."]
+    "KP_Subtract":    [" -"]
+    "KP_Enter":       ["Ent"]
+    "Print":          ["Prn"]
+    "Scroll":         ["Scr"]
+    "Pause":          ["Brk"]
+    "XF86Favorites":  ["Menu"] # fallback labels below
+    "N1":             ["1", "!"]
+    "N2":             ["2", "@"]
+    "N3":             ["3", "#"]
+    "N4":             ["4", "$"]
+    "N5":             ["5", "%"]
+    "N6":             ["6", "^"]
+    "N7":             ["7", "&"]
+    "N8":             ["8", "*"]
+    "N9":             ["9", "("]
+    "N0":             ["0", ")"]
+    "Minus":          ["-", "_"]
+    "Plus":           ["=", "+"]
+    "Comma":          [" ,",  "<"]
+    "Period":         [" .",  ">"]
+    "Oem1":           [" ;",  ":"]
+    "Oem2":           [" /",  "?"]
+    "Oem3":           ["`", "~"]
+    "Oem4":           [" [",  "{"]
+    "Oem5":           [" \\", "|"]
+    "Oem6":           [" ]",  "}"]
+    "Oem7":           [" '",  "\""]
+    "Oem102":         [" \\", "|"] 
+

wlx-overlay-s/src/res/settings.yaml

@@ -0,0 +1,670 @@
+# looking to make changes? 
+# drop me in ~/.config/wlxoverlay/settings.yaml
+#
+
+width: 0.3
+
+size: [600, 700]
+
+# +X: right, +Y: up, +Z: back
+spawn_pos: [0, -0.1, -0.5]
+
+elements:
+  - type: Panel
+    rect: [0, 0, 600, 800]
+    corner_radius: 8
+    bg_color: "#1e2030"
+
+  - type: Label
+    rect: [15, 35, 600, 70]
+    corner_radius: 6
+    font_size: 24
+    fg_color: "#cad3f5"
+    source: Static
+    text: Settings
+
+  - type: Button
+    rect: [560, 0, 40, 40]
+    corner_radius: 8
+    font_size: 16
+    bg_color: "#ed8796"
+    fg_color: "#24273a"
+    text: X
+    click_down:
+      - type: Window
+        target: "settings"
+        action: Destroy
+
+  - type: Panel
+    rect: [50, 53, 500, 1]
+    corner_radius: 6
+    bg_color: "#6e738d"
+
+  ####### Watch Section #######
+
+  - type: Label
+    rect: [15, 85, 570, 24]
+    corner_radius: 6
+    font_size: 18
+    fg_color: "#cad3f5"
+    source: Static
+    text: Watch
+
+  - type: Panel
+    rect: [250, 105, 1, 100]
+    corner_radius: 6
+    bg_color: "#6e738d"
+
+  - type: Label
+    rect: [288, 105, 100, 24]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    source: Static
+    text: Visibility
+
+  - type: Button
+    rect: [270, 120, 100, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Hide"
+    click_down:
+      - type: Watch
+        action: Hide
+
+  - type: Button
+    rect: [270, 170, 100, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Swap Hand"
+    click_down:
+      - type: Watch
+        action: SwitchHands
+
+  - type: Panel
+    rect: [390, 105, 1, 100]
+    corner_radius: 6
+    bg_color: "#6e738d"
+
+  - type: Label
+    rect: [430, 105, 120, 24]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    source: Static
+    text: Watch Fade
+
+  - type: Button
+    rect: [410, 120, 140, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Cutoff Point"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          ViewAngle: {kind: "MaxOpacity", delta: 0.01}
+    scroll_down:
+      - type: Watch
+        action:
+          ViewAngle: {kind: "MaxOpacity", delta: -0.01}
+
+  - type: Button
+    rect: [410, 170, 140, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Cutoff Strength"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          ViewAngle: {kind: "MinOpacity", delta: 0.01}
+    scroll_down:
+      - type: Watch
+        action: 
+          ViewAngle: {kind: "MinOpacity", delta: -0.01}
+
+  - type: Label
+    rect: [25, 140, 90, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    source: Static
+    text: Rotation
+
+  - type: Button
+    rect: [108, 120, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "X"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Rotation: {axis: "X", delta: 0.25}
+    scroll_down:
+      - type: Watch
+        action:
+          Rotation: {axis: "X", delta: -0.25}
+
+  - type: Button
+    rect: [153, 120, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Y"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Rotation: {axis: "Y", delta: 0.25}
+    scroll_down:
+      - type: Watch
+        action:
+          Rotation: {axis: "Y", delta: -0.25}
+
+  - type: Button
+    rect: [198, 120, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Z"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Rotation: {axis: "Z", delta: 0.25}
+    scroll_down:
+      - type: Watch
+        action:
+          Rotation: {axis: "Z", delta: -0.25}
+
+  - type: Label
+    rect: [25, 190, 90, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    source: Static
+    text: Position
+
+  - type: Button
+    rect: [108, 170, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "X"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Position: {axis: "X", delta: 0.001}
+    scroll_down:
+      - type: Watch
+        action:
+          Position: {axis: "X", delta: -0.001}
+
+  - type: Button
+    rect: [153, 170, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Y"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Position: {axis: "Y", delta: 0.001}
+    scroll_down:
+      - type: Watch
+        action:
+          Position: {axis: "Y", delta: -0.001}
+
+  - type: Button
+    rect: [198, 170, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Z"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Position: {axis: "Z", delta: 0.001}
+    scroll_down:
+      - type: Watch
+        action:
+          Position: {axis: "Z", delta: -0.001}
+
+  - type: Panel
+    rect: [50, 220, 500, 1]
+    corner_radius: 6
+    bg_color: "#6e738d"
+
+  ####### Mirror Section #######
+  - type: Label
+    rect: [15, 255, 570, 24]
+    corner_radius: 6
+    font_size: 18
+    fg_color: "#cad3f5"
+    source: Static
+    text: Mirrors
+
+  - type: Label
+    rect: [25, 290, 30, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    source: Static
+    text: M1
+
+  - type: Button
+    rect: [60, 270, 110, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    bg_color: "#494d64"
+    text: "Show/Hide"
+    click_down: # ToggleVisible if exists, else create
+      - type: Overlay
+        target: M1
+        action: ToggleVisible # only fires if overlay exists
+      - type: Window
+        target: M1
+        action: ShowMirror # only fires if not exists
+
+  - type: Button
+    rect: [185, 270, 60, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    bg_color: "#494d64"
+    text: "Lock"
+    click_down:
+      - type: Overlay
+        target: M1
+        action: ToggleInteraction
+
+  - type: Button
+    rect: [258, 270, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#ed8796"
+    text: "X"
+    click_down:
+      - type: Window
+        target: M1
+        action: Destroy
+
+  - type: Label
+    rect: [25, 340, 30, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    source: Static
+    text: M2
+
+  - type: Button
+    rect: [60, 320, 110, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    bg_color: "#494d64"
+    text: "Show/Hide"
+    click_down:
+      - type: Overlay
+        target: M2
+        action: ToggleVisible
+      - type: Window
+        target: M2
+        action: ShowMirror
+
+  - type: Button
+    rect: [185, 320, 60, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    bg_color: "#494d64"
+    text: "Lock"
+    click_down:
+      - type: Overlay
+        target: M2
+        action: ToggleInteraction
+
+  - type: Button
+    rect: [258, 320, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#ed8796"
+    text: "X"
+    click_down:
+      - type: Window
+        target: M2
+        action: Destroy
+
+  - type: Label
+    rect: [25, 390, 30, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    source: Static
+    text: M3
+
+  - type: Button
+    rect: [60, 370, 110, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    bg_color: "#494d64"
+    text: "Show/Hide"
+    click_down:
+      - type: Overlay
+        target: M3
+        action: ToggleVisible
+      - type: Window
+        target: M3
+        action: ShowMirror
+
+  - type: Button
+    rect: [185, 370, 60, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    bg_color: "#494d64"
+    text: "Lock"
+    click_down:
+      - type: Overlay
+        target: M3
+        action: ToggleInteraction
+
+  - type: Button
+    rect: [258, 370, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#ed8796"
+    text: "X"
+    click_down:
+      - type: Window
+        target: M3
+        action: Destroy
+
+  - type: Panel
+    rect: [300, 240, 1, 200]
+    corner_radius: 6
+    bg_color: "#6e738d"
+
+  ####### Color Gain Section #######
+
+  - type: Label
+    rect: [325, 255, 90, 24]
+    corner_radius: 6
+    font_size: 18
+    fg_color: "#cad3f5"
+    source: Static
+    text: Color Gain
+
+  - type: Label
+    rect: [470, 255, 90, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    source: Static
+    text: (SteamVR)
+
+  - type: Button
+    rect: [330, 270, 60, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#cad3f5"
+    bg_color: "#494d64"
+    text: "All"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: ColorAdjust
+        channel: All
+        delta: 0.01
+    scroll_down:
+      - type: ColorAdjust
+        channel: All
+        delta: -0.01
+
+  - type: Button
+    rect: [405, 270, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#e78284"
+    text: "R"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: ColorAdjust
+        channel: R
+        delta: 0.01
+    scroll_down:
+      - type: ColorAdjust
+        channel: R
+        delta: -0.01
+
+  - type: Button
+    rect: [450, 270, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#a6d189"
+    text: "G"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: ColorAdjust
+        channel: G
+        delta: 0.01
+    scroll_down:
+      - type: ColorAdjust
+        channel: G
+        delta: -0.01
+
+  - type: Button
+    rect: [495, 270, 30, 30]
+    corner_radius: 15
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#8caaee"
+    text: "B"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: ColorAdjust
+        channel: B
+        delta: 0.01
+    scroll_down:
+      - type: ColorAdjust
+        channel: B
+        delta: -0.01
+
+  - type: Panel
+    rect: [325, 315, 225, 1]
+    corner_radius: 6
+    bg_color: "#6e738d"
+
+  ####### Playspace Section #######
+
+  - type: Label
+    rect: [325, 345, 90, 24]
+    corner_radius: 6
+    font_size: 18
+    fg_color: "#cad3f5"
+    source: Static
+    text: Playspace
+
+  - type: Button
+    rect: [330, 360, 220, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Fix Floor"
+    click_down:
+      - type: System
+        action: PlayspaceFixFloor
+      - type: Window
+        target: "settings"
+        action: Destroy
+
+  - type: Button
+    rect: [330, 410, 220, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Reset Offset"
+    click_down:
+      - type: System
+        action: PlayspaceResetOffset
+      - type: Window
+        target: "settings"
+        action: Destroy
+
+  ####### Notifications Section #######
+
+  - type: Panel
+    rect: [50, 460, 500, 1]
+    corner_radius: 6
+    bg_color: "#6e738d"
+
+  - type: Label
+    rect: [325, 490, 90, 24]
+    corner_radius: 6
+    font_size: 18
+    fg_color: "#cad3f5"
+    source: Static
+    text: Notifications
+
+  - type: Button
+    rect: [330, 505, 220, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#e64553"
+    text: "Enabled"
+    click_down:
+      - type: System
+        action: ToggleNotifications
+    highlight: Notifications
+
+  - type: Button
+    rect: [330, 555, 220, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#e64553"
+    text: "Sound Enabled"
+    click_down:
+      - type: System
+        action: ToggleNotificationSounds
+    highlight: NotificationSounds
+
+  ####### Behavior Section #######
+  - type: Label
+    rect: [15, 490, 570, 24]
+    corner_radius: 6
+    font_size: 18
+    fg_color: "#cad3f5"
+    source: Static
+    text: Behavior
+
+  - type: Button
+    rect: [30, 505, 220, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#e64553"
+    text: "Auto-Realign"
+    click_down:
+      - type: System
+        action: ToggleAutoRealign
+    highlight: AutoRealign
+
+  - type: Button
+    rect: [30, 555, 220, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#e64553"
+    text: "Grab+Scroll Slide"
+    click_down:
+      - type: System
+        action: ToggleAllowSliding
+    highlight: AllowSliding
+
+  ####### Footer Section #######
+
+  - type: Panel
+    rect: [50, 605, 500, 1]
+    corner_radius: 6
+    bg_color: "#6e738d"
+
+  - type: Button
+    rect: [330, 625, 220, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Save Config"
+    click_down:
+      - type: System
+        action: PersistConfig
+      - type: Toast
+        message: Settings saved successfully.
+
+  - type: Button
+    rect: [30, 625, 250, 30]
+    corner_radius: 6
+    font_size: 12
+    fg_color: "#24273a"
+    bg_color: "#eed49f"
+    text: "Save Overlay Layout"
+    click_down:
+      - type: System
+        action: PersistLayout
+      - type: Toast
+        message: Saved. You will see this layout on next startup.
+

wlx-overlay-s/src/res/watch.yaml

@@ -0,0 +1,192 @@
+# looking to make changes?
+# drop me in ~/.config/wlxoverlay/watch.yaml
+#
+
+width: 0.115
+
+size: [400, 200]
+
+elements:
+  # background panel
+  - type: Panel
+    rect: [0, 30, 400, 130]
+    corner_radius: 20
+    bg_color: "#24273a"
+
+  - type: Button
+    rect: [2, 162, 26, 36]
+    corner_radius: 4
+    font_size: 15
+    bg_color: "#c6a0f6"
+    fg_color: "#24273a"
+    text: "C"
+    click_up: # destroy if exists, otherwise create
+      - type: Window
+        target: settings
+        action: ShowUi # only triggers if not exists
+      - type: Window
+        target: settings
+        action: Destroy # only triggers if exists since before current frame
+
+  # Dashboard toggle button
+  - type: Button
+    rect: [32, 162, 48, 36]
+    corner_radius: 4
+    font_size: 15
+    bg_color: "#2288FF"
+    fg_color: "#24273a"
+    text: "Dash"
+    click_up:
+      - type: WayVR
+        action: ToggleDashboard
+
+  # Keyboard button
+  - type: Button
+    rect: [84, 162, 48, 36]
+    corner_radius: 4
+    font_size: 15
+    fg_color: "#24273a"
+    bg_color: "#a6da95"
+    text: Kbd
+    click_up:
+      - type: Overlay
+        target: "kbd"
+        action: ToggleVisible
+    long_click_up:
+      - type: Overlay
+        target: "kbd"
+        action: Reset
+    right_up:
+      - type: Overlay
+        target: "kbd"
+        action: ToggleImmovable
+    middle_up:
+      - type: Overlay
+        target: "kbd"
+        action: ToggleInteraction
+    scroll_up:
+      - type: Overlay
+        target: "kbd"
+        action:
+          Opacity: { delta: 0.025 }
+    scroll_down:
+      - type: Overlay
+        target: "kbd"
+        action:
+          Opacity: { delta: -0.025 }
+
+  # bottom row, of keyboard + overlays
+  - type: OverlayList
+    rect: [134, 160, 266, 40]
+    corner_radius: 4
+    font_size: 15
+    fg_color: "#cad3f5"
+    bg_color: "#1e2030"
+    layout: Horizontal
+    click_up: ToggleVisible
+    long_click_up: Reset
+    right_up: ToggleImmovable
+    middle_up: ToggleInteraction
+    scroll_up:
+      Opacity: { delta: 0.025 }
+    scroll_down:
+      Opacity: { delta: -0.025 }
+
+  # local clock
+  - type: Label
+    rect: [19, 90, 200, 50]
+    corner_radius: 4
+    font_size: 46 # Use 32 for 12-hour time
+    fg_color: "#cad3f5"
+    source: Clock
+    format: "%H:%M" # 23:59
+    #format: "%I:%M %p" # 11:59 PM
+
+  # local date
+  - type: Label
+    rect: [20, 117, 200, 20]
+    corner_radius: 4
+    font_size: 14
+    fg_color: "#cad3f5"
+    source: Clock
+    format: "%x" # local date representation
+
+  # local day-of-week
+  - type: Label
+    rect: [20, 137, 200, 50]
+    corner_radius: 4
+    font_size: 14
+    fg_color: "#cad3f5"
+    source: Clock
+    format: "%A" # Tuesday
+    #format: "%a" # Tue
+
+  # alt clock 1
+  - type: Label
+    rect: [210, 90, 200, 50]
+    corner_radius: 4
+    font_size: 24 # Use 18 for 12-hour time
+    fg_color: "#8bd5ca"
+    source: Clock
+    timezone: 0
+    format: "%H:%M" # 23:59
+    #format: "%I:%M %p" # 11:59 PM
+  - type: Label
+    rect: [210, 60, 200, 50]
+    corner_radius: 4
+    font_size: 14
+    fg_color: "#8bd5ca"
+    source: Timezone
+    timezone: 0
+
+  # alt clock 2
+  - type: Label
+    rect: [210, 150, 200, 50]
+    corner_radius: 4
+    font_size: 24 # Use 18 for 12-hour time
+    fg_color: "#b7bdf8"
+    source: Clock
+    timezone: 1
+    format: "%H:%M" # 23:59
+    #format: "%I:%M %p" # 11:59 PM
+  - type: Label
+    rect: [210, 120, 200, 50]
+    corner_radius: 4
+    font_size: 14
+    fg_color: "#b7bdf8"
+    source: Timezone
+    timezone: 1
+
+  # batteries
+  - type: BatteryList
+    rect: [0, 5, 400, 30]
+    corner_radius: 4
+    font_size: 16
+    fg_color: "#8bd5ca"
+    fg_color_low: "#B06060"
+    fg_color_charging: "#6080A0"
+    num_devices: 9
+    layout: Horizontal
+    low_threshold: 33
+
+  # volume buttons
+  - type: Button
+    rect: [315, 52, 70, 32]
+    corner_radius: 4
+    font_size: 13
+    fg_color: "#cad3f5"
+    bg_color: "#5b6078"
+    text: "Vol +"
+    click_down:
+      - type: Exec
+        command: ["pactl", "set-sink-volume", "@DEFAULT_SINK@", "+5%"]
+  - type: Button
+    rect: [315, 116, 70, 32]
+    corner_radius: 4
+    font_size: 13
+    fg_color: "#cad3f5"
+    bg_color: "#5b6078"
+    text: "Vol -"
+    click_down:
+      - type: Exec
+        command: ["pactl", "set-sink-volume", "@DEFAULT_SINK@", "-5%"]

wlx-overlay-s/src/res/wayvr.yaml

@@ -0,0 +1,80 @@
+# This is an example WayVR panel configuration. It demonstrates all the capabilities of this module.
+# looking to make changes?
+# drop me in ~/.config/wlxoverlay/wayvr.yaml
+#
+
+version: 1
+
+# If your gpu has some issues with zero-copy textures, you can set this option to "software".
+#
+# Possible options:
+# "dmabuf": Use zero-copy texture access (from EGL to Vulkan) - no performance impact
+# "software": Read pixel data to memory via glReadPixels() every time a content has been updated. Minor performance impact on large resolutions
+blit_method: "dmabuf"
+
+# Set to true if you want to make Wyland server instantly available.
+# By default, WayVR starts only when it's needed.
+# (this option is primarily used for remote starting external processes and development purposes)
+run_compositor_at_start: false 
+
+# Automatically close overlays with zero window count?
+auto_hide: true
+
+# For how long an overlay should be visible in case if there are no windows present? (in milliseconds, auto_hide needs to be enabled)
+# This value shouldn't be set at 0, because some programs could re-initialize a window during startup (splash screens for example)
+auto_hide_delay: 750
+
+# In milliseconds
+keyboard_repeat_delay: 200
+
+# Chars per second
+keyboard_repeat_rate: 50
+
+# WayVR-compatible dashboard.
+# For now, there is only one kind of dashboard with WayVR IPC support (WayVR Dashboard).
+#
+# Build instructions: https://github.com/olekolek1000/wayvr-dashboard
+#
+# exec: Executable path, for example "/home/USER/wayvr-dashboard/src-tauri/target/release/wayvr-dashboard"
+# or just "wayvr-dashboard" if you have it installed from your package manager.
+dashboard:
+  exec: "wayvr-dashboard"
+  args: ""
+  env: []
+
+displays:
+  watch:
+    width: 400
+    height: 600
+    scale: 0.4
+    attach_to: "HandRight" # HandLeft, HandRight
+    pos: [0.0, 0.0, 0.125]
+    rotation: {axis: [1.0, 0.0, 0.0], angle: -45.0}
+  disp1:
+    width: 640
+    height: 480
+    primary: true # Required if you want to attach external processes (not spawned by WayVR itself) without WAYVR_DISPLAY_NAME set
+  disp2:
+    width: 1280
+    height: 720
+    scale: 2.0
+
+catalogs:
+  default_catalog:
+    apps:
+      - name: "Calc"
+        target_display: "disp1"
+        exec: "kcalc"
+        env: ["FOO=bar"]
+        shown_at_start: false
+
+      - name: "htop"
+        target_display: "watch"
+        exec: "konsole"
+        args: "-e htop"
+
+      - name: "Browser"
+        target_display: "disp2"
+        exec: "cage"
+        args: "chromium -- --incognito"
+        

wlx-overlay-s/src/shaders/color.frag

@@ -0,0 +1,24 @@
+#version 310 es
+precision highp float;
+
+layout (location = 0) in vec2 in_uv;
+layout (location = 0) out vec4 out_color;
+
+layout (set = 0, binding = 0) uniform ColorBlock {
+    uniform vec4 in_color;
+    uniform vec2 corner_radius;
+};
+
+void main()
+{
+    out_color.r = corner_radius.r;
+    out_color = in_color;
+
+    vec2 uv_circ = ((1. - corner_radius) - (abs(in_uv + vec2(-0.5)) * 2.))/corner_radius;
+    float dist = length(uv_circ);
+
+    out_color.a = mix(out_color.a, 0.,
+            float(dist > 1.)
+            * float(uv_circ.x < 0.)
+            * float(uv_circ.y < 0.));
+}

wlx-overlay-s/src/shaders/grid.frag

@@ -0,0 +1,19 @@
+#version 310 es
+precision highp float;
+
+layout (location = 0) in vec2 in_uv;
+layout (location = 0) out vec4 out_color;
+
+void main()
+{
+    float fade = max(1.0 - 2.0 * length(in_uv.xy + vec2(-0.5, -0.5)), 0.0);
+    float grid;
+
+    if (fract(in_uv.x / 0.0005) < 0.01 || fract(in_uv.y / 0.0005) < 0.01) {
+        grid = 1.0;
+    } else {
+        grid = 0.0;
+    }
+    out_color = vec4(1.0, 1.0, 1.0, grid * fade);
+}
+

wlx-overlay-s/src/shaders/mod.rs

@@ -0,0 +1,34 @@
+pub mod vert_quad {
+    vulkano_shaders::shader! {
+        ty: "vertex",
+        path: "src/shaders/quad.vert"
+    }
+}
+
+pub mod frag_color {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        path: "src/shaders/color.frag",
+    }
+}
+
+pub mod frag_grid {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        path: "src/shaders/grid.frag",
+    }
+}
+
+pub mod frag_screen {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        path: "src/shaders/screen.frag",
+    }
+}
+
+pub mod frag_srgb {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        path: "src/shaders/srgb.frag",
+    }
+}

wlx-overlay-s/src/shaders/quad.vert

@@ -0,0 +1,11 @@
+#version 310 es
+precision highp float;
+
+layout (location = 0) in vec2 in_pos;
+layout (location = 1) in vec2 in_uv;
+layout (location = 0) out vec2 out_uv;
+
+void main() {
+    out_uv = in_uv;
+    gl_Position = vec4(in_pos * 2. - 1., 0., 1.);
+}

wlx-overlay-s/src/shaders/screen.frag

@@ -0,0 +1,17 @@
+#version 310 es
+precision highp float;
+
+layout (location = 0) in vec2 in_uv;
+layout (location = 0) out vec4 out_color;
+
+layout (set = 0, binding = 0) uniform sampler2D in_texture;
+layout (set = 1, binding = 0) uniform AlphaBlock {
+    uniform float alpha;
+};
+
+void main()
+{
+    out_color = texture(in_texture, in_uv);
+    out_color.a = alpha;
+}
+

wlx-overlay-s/src/shaders/srgb.frag

@@ -0,0 +1,24 @@
+#version 310 es
+precision highp float;
+
+layout (location = 0) in vec2 in_uv;
+layout (location = 0) out vec4 out_color;
+
+layout (set = 0, binding = 0) uniform sampler2D in_texture;
+layout (set = 1, binding = 0) uniform AlphaBlock {
+    uniform float alpha;
+};
+
+
+void main()
+{
+    out_color = texture(in_texture, in_uv);
+    
+    bvec4 cutoff = lessThan(out_color, vec4(0.04045));
+    vec4 higher = pow((out_color + vec4(0.055))/vec4(1.055), vec4(2.4));
+    vec4 lower = out_color/vec4(12.92);
+
+    out_color = mix(higher, lower, cutoff);
+    out_color.a *= alpha;
+}
+

wlx-overlay-s/src/state.rs

@@ -0,0 +1,234 @@
+use glam::Affine3A;
+use idmap::IdMap;
+use rodio::{Decoder, OutputStream, OutputStreamHandle, Source};
+use serde::{Deserialize, Serialize};
+use smallvec::{smallvec, SmallVec};
+use std::{io::Cursor, sync::Arc};
+use vulkano::image::view::ImageView;
+use wgui::gfx::WGfx;
+
+#[cfg(feature = "wayvr")]
+use {
+    crate::config_wayvr::{self, WayVRConfig},
+    crate::overlays::wayvr::WayVRData,
+    std::{cell::RefCell, rc::Rc},
+};
+
+#[cfg(feature = "osc")]
+use crate::backend::osc::OscSender;
+
+use crate::{
+    backend::{input::InputState, overlay::OverlayID, task::TaskContainer},
+    config::{AStrMap, GeneralConfig},
+    config_io,
+    graphics::WGfxExtras,
+    hid::HidProvider,
+    overlays::toast::{DisplayMethod, ToastTopic},
+};
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum KeyboardFocus {
+    PhysicalScreen,
+
+    #[allow(dead_code)] // Not available if "wayvr" feature is disabled
+    WayVR, // (for now without wayland window id data, it's handled internally),
+}
+
+pub struct AppState {
+    pub session: AppSession,
+    pub tasks: TaskContainer,
+
+    pub gfx: Arc<WGfx>,
+    pub gfx_extras: WGfxExtras,
+
+    pub input_state: InputState,
+    pub hid_provider: Box<dyn HidProvider>,
+    pub audio: AudioOutput,
+    pub screens: SmallVec<[ScreenMeta; 8]>,
+    pub anchor: Affine3A,
+    pub sprites: AStrMap<Arc<ImageView>>,
+    pub keyboard_focus: KeyboardFocus,
+    pub toast_sound: &'static [u8],
+
+    #[cfg(feature = "osc")]
+    pub osc_sender: Option<OscSender>,
+
+    #[cfg(feature = "wayvr")]
+    pub wayvr: Option<Rc<RefCell<WayVRData>>>, // Dynamically created if requested
+}
+
+impl AppState {
+    pub fn from_graphics(gfx: Arc<WGfx>, gfx_extras: WGfxExtras) -> anyhow::Result<Self> {
+        // insert shared resources
+        #[cfg(feature = "wayvr")]
+        let mut tasks = TaskContainer::new();
+
+        #[cfg(not(feature = "wayvr"))]
+        let tasks = TaskContainer::new();
+
+        let session = AppSession::load();
+
+        #[cfg(feature = "wayvr")]
+        let wayvr = session
+            .wayvr_config
+            .post_load(&session.config, &mut tasks)?;
+
+        #[cfg(feature = "osc")]
+        let osc_sender = crate::backend::osc::OscSender::new(session.config.osc_out_port).ok();
+
+        let toast_sound_wav = Self::try_load_bytes(
+            &session.config.notification_sound,
+            include_bytes!("res/557297.wav"),
+        );
+
+        Ok(Self {
+            session,
+            tasks,
+            gfx,
+            gfx_extras,
+            input_state: InputState::new(),
+            hid_provider: crate::hid::initialize(),
+            audio: AudioOutput::new(),
+            screens: smallvec![],
+            anchor: Affine3A::IDENTITY,
+            sprites: AStrMap::new(),
+            keyboard_focus: KeyboardFocus::PhysicalScreen,
+            toast_sound: toast_sound_wav,
+
+            #[cfg(feature = "osc")]
+            osc_sender,
+
+            #[cfg(feature = "wayvr")]
+            wayvr,
+        })
+    }
+
+    #[cfg(feature = "wayvr")]
+    #[allow(dead_code)]
+    pub fn get_wayvr(&mut self) -> anyhow::Result<Rc<RefCell<WayVRData>>> {
+        if let Some(wvr) = &self.wayvr {
+            Ok(wvr.clone())
+        } else {
+            let wayvr = Rc::new(RefCell::new(WayVRData::new(
+                WayVRConfig::get_wayvr_config(&self.session.config, &self.session.wayvr_config)?,
+            )?));
+            self.wayvr = Some(wayvr.clone());
+            Ok(wayvr)
+        }
+    }
+
+    pub fn try_load_bytes(path: &str, fallback_data: &'static [u8]) -> &'static [u8] {
+        if path.is_empty() {
+            return fallback_data;
+        }
+
+        let real_path = config_io::get_config_root().join(path);
+
+        if std::fs::File::open(real_path.clone()).is_err() {
+            log::warn!("Could not open file at: {path}");
+            return fallback_data;
+        }
+
+        match std::fs::read(real_path) {
+            // Box is used here to work around `f`'s limited lifetime
+            Ok(f) => Box::leak(Box::new(f)).as_slice(),
+            Err(e) => {
+                log::warn!("Failed to read file at: {path}");
+                log::warn!("{e:?}");
+                fallback_data
+            }
+        }
+    }
+}
+
+pub struct AppSession {
+    pub config: GeneralConfig,
+
+    #[cfg(feature = "wayvr")]
+    pub wayvr_config: WayVRConfig,
+
+    pub toast_topics: IdMap<ToastTopic, DisplayMethod>,
+}
+
+impl AppSession {
+    pub fn load() -> Self {
+        let config_root_path = config_io::ConfigRoot::Generic.ensure_dir();
+        log::info!("Config root path: {}", config_root_path.display());
+        let config = GeneralConfig::load_from_disk();
+
+        let mut toast_topics = IdMap::new();
+        toast_topics.insert(ToastTopic::System, DisplayMethod::Center);
+        toast_topics.insert(ToastTopic::DesktopNotification, DisplayMethod::Center);
+        toast_topics.insert(ToastTopic::XSNotification, DisplayMethod::Center);
+
+        config.notification_topics.iter().for_each(|(k, v)| {
+            toast_topics.insert(*k, *v);
+        });
+
+        #[cfg(feature = "wayvr")]
+        let wayvr_config = config_wayvr::load_wayvr();
+
+        Self {
+            config,
+            #[cfg(feature = "wayvr")]
+            wayvr_config,
+            toast_topics,
+        }
+    }
+}
+
+pub struct AudioOutput {
+    audio_stream: Option<(OutputStream, OutputStreamHandle)>,
+    first_try: bool,
+}
+
+impl AudioOutput {
+    pub const fn new() -> Self {
+        Self {
+            audio_stream: None,
+            first_try: true,
+        }
+    }
+
+    fn get_handle(&mut self) -> Option<&OutputStreamHandle> {
+        if self.audio_stream.is_none() && self.first_try {
+            self.first_try = false;
+            if let Ok((stream, handle)) = OutputStream::try_default() {
+                self.audio_stream = Some((stream, handle));
+            } else {
+                log::error!("Failed to open audio stream. Audio will not work.");
+                return None;
+            }
+        }
+        self.audio_stream.as_ref().map(|(_, h)| h)
+    }
+
+    pub fn play(&mut self, wav_bytes: &'static [u8]) {
+        let Some(handle) = self.get_handle() else {
+            return;
+        };
+        let cursor = Cursor::new(wav_bytes);
+        let source = match Decoder::new_wav(cursor) {
+            Ok(source) => source,
+            Err(e) => {
+                log::error!("Failed to play sound: {e:?}");
+                return;
+            }
+        };
+        let _ = handle.play_raw(source.convert_samples());
+    }
+}
+
+pub struct ScreenMeta {
+    pub name: Arc<str>,
+    pub id: OverlayID,
+    pub native_handle: u32,
+}
+
+#[derive(Serialize, Deserialize, Clone, Copy, Default)]
+#[repr(u8)]
+pub enum LeftRight {
+    #[default]
+    Left,
+    Right,
+}