initial commit

src/backend/common.rs

@@ -1,152 +1,158 @@
-use std::{collections::VecDeque, time::Instant};
+use std::{
+    collections::{BinaryHeap, VecDeque},
+    sync::Arc,
+    time::Instant,
+};
 
-use glam::{Affine3A, Vec2, Vec3, Vec3A};
-use ovr_overlay::TrackedDeviceIndex;
+use idmap::IdMap;
 
-pub struct InputState<TState, THand> {
-    pub hmd: Affine3A,
-    pub pointers: [Pointer<THand>; 2],
-    pub devices: Vec<TrackedDevice>,
-    pub(super) data: TState,
+use crate::{
+    overlays::{
+        keyboard::create_keyboard,
+        screen::{get_screens_wayland, get_screens_x11},
+        watch::create_watch,
+    },
+    state::AppState,
+};
+
+use super::overlay::{OverlayData, OverlayState};
+
+pub struct OverlayContainer<T>
+where
+    T: Default,
+{
+    overlays: IdMap<usize, OverlayData<T>>,
 }
 
-impl<TState, THand> InputState<TState, THand> {
-    pub fn pre_update(&mut self) {
-        self.pointers[0].before = self.pointers[0].now;
-        self.pointers[1].before = self.pointers[1].now;
+impl<T> OverlayContainer<T>
+where
+    T: Default,
+{
+    pub fn new(app: &mut AppState) -> Self {
+        let mut overlays = IdMap::new();
+
+        let screens = if std::env::var("WAYLAND_DISPLAY").is_ok() {
+            get_screens_wayland(&app.session)
+        } else {
+            get_screens_x11()
+        };
+
+        let watch = create_watch::<T>(&app, &screens);
+        overlays.insert(watch.state.id, watch);
+
+        let keyboard = create_keyboard(&app);
+        overlays.insert(keyboard.state.id, keyboard);
+
+        let mut first = true;
+        for mut screen in screens {
+            if first {
+                screen.state.want_visible = true;
+                first = false;
+            }
+            overlays.insert(screen.state.id, screen);
+        }
+
+        Self { overlays }
     }
 
-    pub fn post_update(&mut self) {
-        for hand in &mut self.pointers {
-            if hand.now.click_modifier_right {
-                hand.interaction.mode = PointerMode::Right;
-                continue;
-            }
-
-            if hand.now.click_modifier_middle {
-                hand.interaction.mode = PointerMode::Middle;
-                continue;
-            }
-
-            let hmd_up = self.hmd.transform_vector3a(Vec3A::Y);
-            let dot =
-                hmd_up.dot(hand.pose.transform_vector3a(Vec3A::X)) * (1.0 - 2.0 * hand.hand as f32);
-
-            hand.interaction.mode = if dot < -0.85 {
-                PointerMode::Right
-            } else if dot > 0.7 {
-                PointerMode::Middle
-            } else {
-                PointerMode::Left
-            };
-
-            let middle_click_orientation = false;
-            let right_click_orientation = false;
-            match hand.interaction.mode {
-                PointerMode::Middle => {
-                    if !middle_click_orientation {
-                        hand.interaction.mode = PointerMode::Left;
-                    }
-                }
-                PointerMode::Right => {
-                    if !right_click_orientation {
-                        hand.interaction.mode = PointerMode::Left;
-                    }
-                }
-                _ => {}
-            };
+    pub fn mut_by_selector(&mut self, selector: &OverlaySelector) -> Option<&mut OverlayData<T>> {
+        match selector {
+            OverlaySelector::Id(id) => self.mut_by_id(*id),
+            OverlaySelector::Name(name) => self.mut_by_name(name),
         }
     }
+
+    pub fn get_by_id<'a>(&'a mut self, id: usize) -> Option<&'a OverlayData<T>> {
+        self.overlays.get(&id)
+    }
+
+    pub fn mut_by_id<'a>(&'a mut self, id: usize) -> Option<&'a mut OverlayData<T>> {
+        self.overlays.get_mut(&id)
+    }
+
+    pub fn get_by_name<'a>(&'a mut self, name: &str) -> Option<&'a OverlayData<T>> {
+        self.overlays.values().find(|o| *o.state.name == *name)
+    }
+
+    pub fn mut_by_name<'a>(&'a mut self, name: &str) -> Option<&'a mut OverlayData<T>> {
+        self.overlays.values_mut().find(|o| *o.state.name == *name)
+    }
+
+    pub fn iter<'a>(&'a self) -> impl Iterator<Item = &'a OverlayData<T>> {
+        self.overlays.values()
+    }
+
+    pub fn iter_mut<'a>(&'a mut self) -> impl Iterator<Item = &'a mut OverlayData<T>> {
+        self.overlays.values_mut()
+    }
 }
 
-pub struct Pointer<THand> {
-    pub hand: usize,
-    pub pose: Affine3A,
-    pub now: PointerState,
-    pub before: PointerState,
-    pub(super) interaction: InteractionState,
-    pub(super) data: THand,
+pub enum OverlaySelector {
+    Id(usize),
+    Name(Arc<str>),
 }
 
-#[derive(Debug, Clone, Copy, Default)]
-pub struct PointerState {
-    pub scroll: f32,
-    pub click: bool,
-    pub grab: bool,
-    pub alt_click: bool,
-    pub show_hide: bool,
-    pub space_drag: bool,
-    pub click_modifier_right: bool,
-    pub click_modifier_middle: bool,
+struct AppTask {
+    pub not_before: Instant,
+    pub task: TaskType,
 }
 
-pub struct InteractionState {
-    pub mode: PointerMode,
-    pub grabbed: Option<GrabData>,
-    pub clicked_id: Option<usize>,
-    pub hovered_id: Option<usize>,
-    pub release_actions: VecDeque<Box<dyn Fn()>>,
-    pub next_push: Instant,
+impl PartialEq<AppTask> for AppTask {
+    fn eq(&self, other: &Self) -> bool {
+        self.not_before == other.not_before
+    }
+}
+impl PartialOrd<AppTask> for AppTask {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.not_before.cmp(&other.not_before).reverse())
+    }
+}
+impl Eq for AppTask {}
+impl Ord for AppTask {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.not_before.cmp(&other.not_before).reverse()
+    }
 }
 
-impl Default for InteractionState {
-    fn default() -> Self {
+pub enum TaskType {
+    Global(Box<dyn FnOnce(&mut AppState) + Send>),
+    Overlay(
+        OverlaySelector,
+        Box<dyn FnOnce(&mut AppState, &mut OverlayState) + Send>,
+    ),
+}
+
+pub struct TaskContainer {
+    tasks: BinaryHeap<AppTask>,
+}
+
+impl TaskContainer {
+    pub fn new() -> Self {
         Self {
-            mode: PointerMode::Left,
-            grabbed: None,
-            clicked_id: None,
-            hovered_id: None,
-            release_actions: VecDeque::new(),
-            next_push: Instant::now(),
+            tasks: BinaryHeap::new(),
+        }
+    }
+
+    pub fn enqueue(&mut self, task: TaskType) {
+        self.tasks.push(AppTask {
+            not_before: Instant::now(),
+            task,
+        });
+    }
+
+    pub fn enqueue_at(&mut self, task: TaskType, not_before: Instant) {
+        self.tasks.push(AppTask { not_before, task });
+    }
+
+    pub fn retrieve_due(&mut self, dest_buf: &mut VecDeque<TaskType>) {
+        let now = Instant::now();
+
+        while let Some(task) = self.tasks.peek() {
+            if task.not_before > now {
+                break;
+            }
+
+            dest_buf.push_back(self.tasks.pop().unwrap().task);
         }
     }
 }
-
-pub struct PointerHit {
-    pub hand: usize,
-    pub mode: PointerMode,
-    pub primary: bool,
-    pub uv: Vec2,
-    pub dist: f32,
-}
-
-struct RayHit {
-    idx: usize,
-    ray_pos: Vec3,
-    hit_pos: Vec3,
-    uv: Vec2,
-    dist: f32,
-}
-
-#[derive(Debug, Clone, Copy, Default)]
-pub struct GrabData {
-    pub offset: Vec3,
-    pub grabbed_id: usize,
-}
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, Default)]
-pub enum PointerMode {
-    #[default]
-    Left,
-    Right,
-    Middle,
-}
-
-pub struct TrackedDevice {
-    pub index: TrackedDeviceIndex,
-    pub valid: bool,
-    pub soc: Option<f32>,
-    pub charging: bool,
-    pub role: TrackedDeviceRole,
-}
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum TrackedDeviceRole {
-    None,
-    Hmd,
-    LeftHand,
-    RightHand,
-    Tracker,
-}

src/backend/input.rs

@@ -0,0 +1,362 @@
+use std::{collections::VecDeque, time::Instant};
+
+use glam::{Affine3A, Vec2, Vec3A};
+use log::warn;
+use ovr_overlay::TrackedDeviceIndex;
+use tinyvec::array_vec;
+
+use crate::state::AppState;
+
+use super::{common::OverlayContainer, overlay::OverlayData};
+
+pub struct TrackedDevice {
+    pub index: TrackedDeviceIndex,
+    pub valid: bool,
+    pub soc: Option<f32>,
+    pub charging: bool,
+    pub role: TrackedDeviceRole,
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TrackedDeviceRole {
+    None,
+    Hmd,
+    LeftHand,
+    RightHand,
+    Tracker,
+}
+
+pub struct InputState<TState, THand> {
+    pub hmd: Affine3A,
+    pub pointers: [Pointer<THand>; 2],
+    pub devices: Vec<TrackedDevice>,
+    pub(super) data: TState,
+}
+
+impl<TState, THand> InputState<TState, THand> {
+    pub fn pre_update(&mut self) {
+        self.pointers[0].before = self.pointers[0].now;
+        self.pointers[1].before = self.pointers[1].now;
+    }
+
+    pub fn post_update(&mut self) {
+        for hand in &mut self.pointers {
+            if hand.now.click_modifier_right {
+                hand.interaction.mode = PointerMode::Right;
+                continue;
+            }
+
+            if hand.now.click_modifier_middle {
+                hand.interaction.mode = PointerMode::Middle;
+                continue;
+            }
+
+            let hmd_up = self.hmd.transform_vector3a(Vec3A::Y);
+            let dot =
+                hmd_up.dot(hand.pose.transform_vector3a(Vec3A::X)) * (1.0 - 2.0 * hand.hand as f32);
+
+            hand.interaction.mode = if dot < -0.85 {
+                PointerMode::Right
+            } else if dot > 0.7 {
+                PointerMode::Middle
+            } else {
+                PointerMode::Left
+            };
+
+            let middle_click_orientation = false;
+            let right_click_orientation = false;
+            match hand.interaction.mode {
+                PointerMode::Middle => {
+                    if !middle_click_orientation {
+                        hand.interaction.mode = PointerMode::Left;
+                    }
+                }
+                PointerMode::Right => {
+                    if !right_click_orientation {
+                        hand.interaction.mode = PointerMode::Left;
+                    }
+                }
+                _ => {}
+            };
+        }
+    }
+}
+
+pub struct InteractionState {
+    pub mode: PointerMode,
+    pub grabbed: Option<GrabData>,
+    pub clicked_id: Option<usize>,
+    pub hovered_id: Option<usize>,
+    pub release_actions: VecDeque<Box<dyn Fn()>>,
+    pub next_push: Instant,
+}
+
+impl Default for InteractionState {
+    fn default() -> Self {
+        Self {
+            mode: PointerMode::Left,
+            grabbed: None,
+            clicked_id: None,
+            hovered_id: None,
+            release_actions: VecDeque::new(),
+            next_push: Instant::now(),
+        }
+    }
+}
+
+pub struct Pointer<THand> {
+    pub idx: usize,
+    pub hand: u8,
+    pub pose: Affine3A,
+    pub now: PointerState,
+    pub before: PointerState,
+    pub(super) interaction: InteractionState,
+    pub(super) data: THand,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct PointerState {
+    pub scroll: f32,
+    pub click: bool,
+    pub grab: bool,
+    pub alt_click: bool,
+    pub show_hide: bool,
+    pub space_drag: bool,
+    pub click_modifier_right: bool,
+    pub click_modifier_middle: bool,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct PointerHit {
+    pub pointer: usize,
+    pub overlay: usize,
+    pub mode: PointerMode,
+    pub primary: bool,
+    pub uv: Vec2,
+    pub dist: f32,
+}
+
+pub trait InteractionHandler {
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit);
+    fn on_left(&mut self, app: &mut AppState, pointer: usize);
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool);
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32);
+}
+
+pub struct DummyInteractionHandler;
+
+impl InteractionHandler for DummyInteractionHandler {
+    fn on_left(&mut self, _app: &mut AppState, _pointer: usize) {}
+    fn on_hover(&mut self, _app: &mut AppState, _hit: &PointerHit) {}
+    fn on_pointer(&mut self, _app: &mut AppState, _hit: &PointerHit, _pressed: bool) {}
+    fn on_scroll(&mut self, _app: &mut AppState, _hit: &PointerHit, _delta: f32) {}
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+struct RayHit {
+    overlay: usize,
+    hit_pos: Vec3A,
+    dist: f32,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct GrabData {
+    pub offset: Vec3A,
+    pub grabbed_id: usize,
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, Default)]
+pub enum PointerMode {
+    #[default]
+    Left,
+    Right,
+    Middle,
+}
+
+impl<THand> Pointer<THand> {
+    pub fn interact<O>(&mut self, overlays: &mut OverlayContainer<O>, app: &mut AppState)
+    where
+        O: Default,
+    {
+        if let Some(grab_data) = self.interaction.grabbed {
+            if let Some(grabbed) = overlays.mut_by_id(grab_data.grabbed_id) {
+                self.handle_grabbed(grabbed, grab_data.offset);
+            } else {
+                warn!("Grabbed overlay {} does not exist", grab_data.grabbed_id);
+                self.interaction.grabbed = None;
+            }
+            return;
+        }
+
+        let Some(mut hit) = self.get_nearest_hit(overlays) else {
+            if let Some(hovered_id) = self.interaction.hovered_id.take() {
+                if let Some(hovered) = overlays.mut_by_id(hovered_id) {
+                    hovered.backend.on_left(app, self.idx);
+                }
+                self.interaction.hovered_id = None;
+            }
+            if let Some(clicked_id) = self.interaction.clicked_id.take() {
+                if let Some(clicked) = overlays.mut_by_id(clicked_id) {
+                    let hit = PointerHit {
+                        pointer: self.idx,
+                        overlay: clicked_id,
+                        mode: self.interaction.mode,
+                        ..Default::default()
+                    };
+                    clicked.backend.on_pointer(app, &hit, false);
+                }
+            }
+            return;
+        };
+
+        if let Some(hovered_id) = self.interaction.hovered_id {
+            if hovered_id != hit.overlay {
+                if let Some(old_hovered) = overlays.mut_by_id(hovered_id) {
+                    if Some(self.idx) == old_hovered.primary_pointer {
+                        old_hovered.primary_pointer = None;
+                    }
+                    old_hovered.backend.on_left(app, self.idx);
+                }
+            }
+        }
+        let Some(hovered) = overlays.mut_by_id(hit.overlay) else {
+            warn!("Hit overlay {} does not exist", hit.overlay);
+            return;
+        };
+
+        self.interaction.hovered_id = Some(hit.overlay);
+
+        if let Some(primary_pointer) = hovered.primary_pointer {
+            if hit.pointer < primary_pointer {
+                hovered.primary_pointer = Some(hit.pointer);
+                hit.primary = true;
+            }
+        } else {
+            hovered.primary_pointer = Some(hit.pointer);
+            hit.primary = true;
+        }
+        hovered.backend.on_hover(app, &hit);
+
+        if self.now.scroll.abs() > 0.1 {
+            hovered.backend.on_scroll(app, &hit, self.now.scroll);
+        }
+
+        if self.now.click && !self.before.click {
+            self.interaction.clicked_id = Some(hit.overlay);
+            hovered.backend.on_pointer(app, &hit, true);
+        } else if !self.now.click && self.before.click {
+            if let Some(clicked_id) = self.interaction.clicked_id.take() {
+                if let Some(clicked) = overlays.mut_by_id(clicked_id) {
+                    clicked.backend.on_pointer(app, &hit, false);
+                }
+            } else {
+                hovered.backend.on_pointer(app, &hit, false);
+            }
+        }
+    }
+
+    fn get_nearest_hit<O>(&mut self, overlays: &mut OverlayContainer<O>) -> Option<PointerHit>
+    where
+        O: Default,
+    {
+        let mut hits = array_vec!([RayHit; 8]);
+
+        for overlay in overlays.iter() {
+            if !overlay.state.want_visible {
+                continue;
+            }
+
+            if let Some(hit) = self.ray_test(overlay.state.id, &overlay.state.transform) {
+                hits.try_push(hit);
+            }
+        }
+
+        hits.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap());
+
+        for hit in hits.iter() {
+            let uv = overlays
+                .get_by_id(hit.overlay)
+                .unwrap() // this is safe
+                .state
+                .transform
+                .inverse()
+                .transform_point3a(hit.hit_pos)
+                .truncate();
+
+            if uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0 {
+                continue;
+            }
+
+            return Some(PointerHit {
+                pointer: self.idx,
+                overlay: hit.overlay,
+                mode: self.interaction.mode,
+                primary: false,
+                uv,
+                dist: hit.dist,
+            });
+        }
+
+        None
+    }
+
+    fn start_grab<O>(&mut self, overlay: &mut OverlayData<O>)
+    where
+        O: Default,
+    {
+        let offset = self
+            .pose
+            .inverse()
+            .transform_point3a(overlay.state.transform.translation);
+
+        self.interaction.grabbed = Some(GrabData {
+            offset,
+            grabbed_id: overlay.state.id,
+        });
+    }
+    fn handle_grabbed<O>(&mut self, overlay: &mut OverlayData<O>, offset: Vec3A)
+    where
+        O: Default,
+    {
+        if self.now.grab {
+            overlay.state.transform.translation = self.pose.transform_point3a(offset);
+
+            if self.now.click && !self.before.click {
+                warn!("todo: click-while-grabbed");
+            }
+
+            match self.interaction.mode {
+                PointerMode::Left => {
+                    overlay.state.transform.translation.y += self.now.scroll * 0.01;
+                }
+                _ => {
+                    overlay.state.transform.matrix3 = overlay
+                        .state
+                        .transform
+                        .matrix3
+                        .mul_scalar(1.0 + 0.01 * self.now.scroll);
+                }
+            }
+        } else {
+            overlay.state.spawn_point = overlay.state.transform.translation;
+            self.interaction.grabbed = None;
+        }
+    }
+    fn ray_test(&self, overlay: usize, plane: &Affine3A) -> Option<RayHit> {
+        let plane_normal = plane.transform_vector3a(Vec3A::NEG_Z);
+        let ray_dir = self.pose.transform_vector3a(Vec3A::NEG_Z);
+
+        let d = plane.translation.dot(-plane_normal);
+        let dist = -(d + self.pose.translation.dot(plane_normal)) / ray_dir.dot(plane_normal);
+
+        let hit_pos = self.pose.translation + ray_dir * dist;
+
+        Some(RayHit {
+            overlay,
+            hit_pos,
+            dist,
+        })
+    }
+}

src/backend/mod.rs

@@ -1,3 +1,5 @@
 pub mod common;
+pub mod input;
 pub mod openvr;
 pub mod openxr;
+pub mod overlay;

src/backend/openvr/input.rs

@@ -1,4 +1,4 @@
-use std::array;
+use std::{array, io::Write, path::Path};
 
 use glam::Affine3A;
 use ovr_overlay::{
@@ -11,7 +11,7 @@ use ovr_overlay::{
     TrackedDeviceIndex,
 };
 
-use crate::backend::common::{
+use crate::backend::input::{
     InputState, InteractionState, Pointer, PointerState, TrackedDevice, TrackedDeviceRole,
 };
 
@@ -96,7 +96,8 @@ impl InputState<OpenVrInputState, OpenVrHandState> {
             .collect::<Result<_, &'static str>>()?;
 
         let hands: [Pointer<OpenVrHandState>; 2] = array::from_fn(|i| Pointer::<OpenVrHandState> {
-            hand: i,
+            idx: i,
+            hand: i as u8,
             now: PointerState::default(),
             before: PointerState::default(),
             pose: Affine3A::IDENTITY,
@@ -286,3 +287,30 @@ fn copy_from_hmd(in_mat: &HmdMatrix34_t, out_mat: &mut Affine3A) {
     out_mat.w_axis[1] = in_mat.m[1][3];
     out_mat.w_axis[2] = in_mat.m[2][3];
 }
+
+pub fn action_manifest_path() -> &'static Path {
+    let action_path = "/tmp/wlxoverlay-s/actions.json";
+    std::fs::create_dir_all("/tmp/wlxoverlay-s").unwrap();
+
+    std::fs::File::create(action_path)
+        .unwrap()
+        .write_all(include_bytes!("../../res/actions.json"))
+        .unwrap();
+
+    std::fs::File::create("/tmp/wlxoverlay-s/actions_binding_knuckles.json")
+        .unwrap()
+        .write_all(include_bytes!("../../res/actions_binding_knuckles.json"))
+        .unwrap();
+
+    std::fs::File::create("/tmp/wlxoverlay-s/actions_binding_vive.json")
+        .unwrap()
+        .write_all(include_bytes!("../../res/actions_binding_vive.json"))
+        .unwrap();
+
+    std::fs::File::create("/tmp/wlxoverlay-s/actions_binding_oculus.json")
+        .unwrap()
+        .write_all(include_bytes!("../../res/actions_binding_oculus.json"))
+        .unwrap();
+
+    Path::new(action_path)
+}

src/backend/openvr/mod.rs

@@ -1,4 +1,5 @@
 use std::{
+    collections::VecDeque,
     path::Path,
     time::{Duration, Instant},
 };
@@ -8,45 +9,73 @@ use ovr_overlay::{
     sys::{ETrackedDeviceProperty, EVRApplicationType, EVREventType},
     TrackedDeviceIndex,
 };
+use vulkano::{
+    device::{physical::PhysicalDevice, DeviceExtensions},
+    instance::InstanceExtensions,
+    Handle, VulkanObject,
+};
 
-use super::common::InputState;
+use crate::state::AppState;
+
+use self::{input::action_manifest_path, overlay::OpenVrOverlayData};
+
+use super::{
+    common::{OverlayContainer, TaskType},
+    input::InputState,
+};
 
 pub mod input;
 pub mod overlay;
 
-fn openvr_run() {
+pub fn openvr_run() {
     let app_type = EVRApplicationType::VRApplication_Overlay;
     let Ok(context) = ovr_overlay::Context::init(app_type) else {
             error!("Failed to initialize OpenVR");
             return;
         };
 
-    let mut overlay = context.overlay_mngr();
-    let mut settings = context.settings_mngr();
-    let mut input = context.input_mngr();
-    let mut system = context.system_mngr();
-    let mut compositor = context.compositor_mngr();
+    let mut overlay_mngr = context.overlay_mngr();
+    //let mut settings_mngr = context.settings_mngr();
+    let mut input_mngr = context.input_mngr();
+    let mut system_mngr = context.system_mngr();
+    let mut compositor_mngr = context.compositor_mngr();
 
-    let Ok(_) = input.set_action_manifest(Path::new("resources/actions.json")) else {
-        error!("Failed to set action manifest");
+    let device_extensions_fn = |device: &PhysicalDevice| {
+        let names = compositor_mngr.get_vulkan_device_extensions_required(device.handle().as_raw());
+        let ext = DeviceExtensions::from_iter(names.iter().map(|s| s.as_str()));
+        ext
+    };
+
+    let mut compositor_mngr = context.compositor_mngr();
+    let instance_extensions = {
+        let names = compositor_mngr.get_vulkan_instance_extensions_required();
+        InstanceExtensions::from_iter(names.iter().map(|s| s.as_str()))
+    };
+
+    let mut state = AppState::new(instance_extensions, device_extensions_fn);
+    let mut overlays = OverlayContainer::<OpenVrOverlayData>::new(&mut state);
+
+    if let Err(e) = input_mngr.set_action_manifest(action_manifest_path()) {
+        error!("Failed to set action manifest: {}", e.description());
         return;
     };
 
-    let Ok(mut input_state) = InputState::new(&mut input) else {
+    let Ok(mut input) = InputState::new(&mut input_mngr) else {
         error!("Failed to initialize input");
         return;
     };
 
-    let Ok(refresh_rate) = system.get_tracked_device_property::<f32>(TrackedDeviceIndex::HMD, ETrackedDeviceProperty::Prop_DisplayFrequency_Float) else {
+    let Ok(refresh_rate) = system_mngr.get_tracked_device_property::<f32>(TrackedDeviceIndex::HMD, ETrackedDeviceProperty::Prop_DisplayFrequency_Float) else {
         error!("Failed to get display refresh rate");
         return;
     };
 
     let frame_time = (1000.0 / refresh_rate).floor() * 0.001;
     let mut next_device_update = Instant::now();
+    let mut due_tasks = VecDeque::with_capacity(4);
 
     loop {
-        while let Some(event) = system.poll_next_event() {
+        while let Some(event) = system_mngr.poll_next_event() {
             match event.event_type {
                 EVREventType::VREvent_Quit => {
                     info!("Received quit event, shutting down.");
@@ -61,35 +90,57 @@ fn openvr_run() {
             }
 
             if next_device_update <= Instant::now() {
-                input_state.update_devices(&mut system);
+                input.update_devices(&mut system_mngr);
                 next_device_update = Instant::now() + Duration::from_secs(30);
             }
 
-            input_state.pre_update();
-            input_state.update(&mut input, &mut system);
-            input_state.post_update();
+            state.tasks.retrieve_due(&mut due_tasks);
+            while let Some(task) = due_tasks.pop_front() {
+                match task {
+                    TaskType::Global(f) => f(&mut state),
+                    TaskType::Overlay(sel, f) => {
+                        if let Some(o) = overlays.mut_by_selector(&sel) {
+                            f(&mut state, &mut o.state);
+                        }
+                    }
+                }
+            }
 
-            // task scheduler
+            input.pre_update();
+            input.update(&mut input_mngr, &mut system_mngr);
+            input.post_update();
 
-            // after input
+            input
+                .pointers
+                .iter_mut()
+                .for_each(|p| p.interact(&mut overlays, &mut state));
 
-            // interactions
+            overlays
+                .iter_mut()
+                .for_each(|o| o.after_input(&mut overlay_mngr, &mut state));
 
-            // show overlays
+            log::debug!("Rendering frame");
+
+            overlays
+                .iter_mut()
+                .filter(|o| o.state.want_visible)
+                .for_each(|o| o.render(&mut state));
+
+            log::debug!("Rendering overlays");
+
+            overlays
+                .iter_mut()
+                .for_each(|o| o.after_render(&mut overlay_mngr, &state.graphics));
 
             // chaperone
 
-            // render overlays
-
-            // hide overlays
-
-            // close font handles
+            // close font handles?
 
             // playspace moved end frame
 
             let mut seconds_since_vsync = 0f32;
             std::thread::sleep(Duration::from_secs_f32(
-                if system.get_time_since_last_vsync(&mut seconds_since_vsync, &mut 0u64) {
+                if system_mngr.get_time_since_last_vsync(&mut seconds_since_vsync, &mut 0u64) {
                     frame_time - seconds_since_vsync
                 } else {
                     0.011

src/backend/openvr/overlay.rs

@@ -1,15 +1,207 @@
-use ovr_overlay::sys::VRVulkanTextureData_t;
+use glam::Vec4;
+use ovr_overlay::{
+    overlay::{OverlayHandle, OverlayManager},
+    sys::VRVulkanTextureData_t,
+};
+use vulkano::{
+    command_buffer::{
+        synced::{
+            SyncCommandBuffer, SyncCommandBufferBuilder, SyncCommandBufferBuilderExecuteCommands,
+        },
+        AutoCommandBufferBuilder, CommandBufferExecFuture,
+    },
+    image::{ImageAccess, ImageLayout},
+    sync::{future::NowFuture, ImageMemoryBarrier},
+    Handle, VulkanObject,
+};
 
-use crate::overlays::OverlayData;
+use crate::{backend::overlay::OverlayData, graphics::WlxGraphics, state::AppState};
 
-pub(super) struct OpenVrOverlayManager {
-    pub(super) overlays: Vec<OpenVrOverlay>,
-}
-
-pub(super) struct OpenVrOverlay {
+#[derive(Default)]
+pub(super) struct OpenVrOverlayData {
+    handle: Option<OverlayHandle>,
+    last_image: Option<u64>,
     pub(super) visible: bool,
-    pub(super) color: [f32; 4],
-    overlay: OverlayData,
-    handle: u32,
-    ovr_texture: VRVulkanTextureData_t,
+    pub(super) color: Vec4,
+    pub(super) curvature: f32,
+    pub(super) sort_order: u32,
+}
+
+impl OverlayData<OpenVrOverlayData> {
+    pub fn initialize(
+        &mut self,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> OverlayHandle {
+        let key = format!("wlx-{}", self.state.name);
+        let handle = match overlay.create_overlay(&key, &key) {
+            Ok(handle) => handle,
+            Err(e) => {
+                panic!("Failed to create overlay: {}", e);
+            }
+        };
+        log::debug!("{}: initialize", self.state.name);
+
+        self.data.handle = Some(handle);
+
+        self.init(app);
+
+        self.upload_width(overlay);
+        self.upload_color(overlay);
+        self.upload_curvature(overlay);
+
+        handle
+    }
+
+    pub fn after_input(&mut self, overlay: &mut OverlayManager, app: &mut AppState) {
+        if self.state.want_visible && !self.data.visible {
+            self.show(overlay, app);
+        } else if !self.state.want_visible && self.data.visible {
+            self.hide(overlay);
+        }
+    }
+
+    pub fn after_render(&mut self, overlay: &mut OverlayManager, graphics: &WlxGraphics) {
+        if self.data.visible {
+            self.upload_texture(overlay, graphics);
+        }
+    }
+
+    fn show(&mut self, overlay: &mut OverlayManager, app: &mut AppState) {
+        let handle = match self.data.handle {
+            Some(handle) => handle,
+            None => self.initialize(overlay, app),
+        };
+        log::debug!("{}: show", self.state.name);
+        if let Err(e) = overlay.set_visibility(handle, true) {
+            panic!("Failed to show overlay: {}", e);
+        }
+        self.data.visible = true;
+    }
+
+    fn hide(&mut self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            return;
+        };
+        log::debug!("{}: hide", self.state.name);
+        if let Err(e) = overlay.set_visibility(handle, false) {
+            panic!("Failed to hide overlay: {}", e);
+        }
+        self.data.visible = false;
+    }
+
+    fn upload_color(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_opacity(handle, self.data.color.w) {
+            panic!("Failed to set overlay opacity: {}", e);
+        }
+        if let Err(e) = overlay.set_tint(
+            handle,
+            ovr_overlay::ColorTint {
+                r: self.data.color.x,
+                g: self.data.color.y,
+                b: self.data.color.z,
+                a: 1.0,
+            },
+        ) {
+            panic!("Failed to set overlay tint: {}", e);
+        }
+    }
+
+    fn upload_width(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_width(handle, self.state.width) {
+            panic!("Failed to set overlay width: {}", e);
+        }
+    }
+
+    fn upload_curvature(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_curvature(handle, self.data.curvature) {
+            panic!("Failed to set overlay curvature: {}", e);
+        }
+    }
+
+    fn upload_sort_order(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_sort_order(handle, self.data.sort_order) {
+            panic!("Failed to set overlay z order: {}", e);
+        }
+    }
+
+    fn upload_texture(&mut self, overlay: &mut OverlayManager, graphics: &WlxGraphics) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+
+        let Some(view) = self.backend.view() else {
+            log::debug!("{}: Not rendered", self.state.name);
+            return;
+        };
+
+        let image = view.image().inner().image.clone();
+
+        let raw_image = image.handle().as_raw();
+
+        if let Some(last_image) = self.data.last_image {
+            if last_image == raw_image {
+                return;
+            }
+        }
+
+        let Some(format) = image.format() else {
+            panic!("{}: Image format is None", self.state.name);
+        };
+
+        let dimensions = image.dimensions();
+
+        let mut texture = VRVulkanTextureData_t {
+            m_nImage: raw_image,
+            m_nFormat: format as _,
+            m_nWidth: dimensions.width(),
+            m_nHeight: dimensions.height(),
+            m_nSampleCount: image.samples() as u32,
+            m_pDevice: graphics.device.handle().as_raw() as *mut _,
+            m_pPhysicalDevice: graphics.device.physical_device().handle().as_raw() as *mut _,
+            m_pInstance: graphics.instance.handle().as_raw() as *mut _,
+            m_pQueue: graphics.queue.handle().as_raw() as *mut _,
+            m_nQueueFamilyIndex: graphics.queue.queue_family_index(),
+        };
+
+        graphics
+            .transition_layout(
+                image.clone(),
+                ImageLayout::ColorAttachmentOptimal,
+                ImageLayout::TransferSrcOptimal,
+            )
+            .wait(None)
+            .unwrap();
+
+        log::info!("nImage: {}, nFormat: {:?}, nWidth: {}, nHeight: {}, nSampleCount: {}, nQueueFamilyIndex: {}", texture.m_nImage, format, texture.m_nWidth, texture.m_nHeight, texture.m_nSampleCount, texture.m_nQueueFamilyIndex);
+        if let Err(e) = overlay.set_image_vulkan(handle, &mut texture) {
+            panic!("Failed to set overlay texture: {}", e);
+        }
+
+        graphics
+            .transition_layout(
+                image,
+                ImageLayout::TransferSrcOptimal,
+                ImageLayout::ColorAttachmentOptimal,
+            )
+            .wait(None)
+            .unwrap();
+    }
 }

src/backend/overlay.rs

@@ -0,0 +1,170 @@
+use std::sync::{
+    atomic::{AtomicUsize, Ordering},
+    Arc,
+};
+
+use glam::{Affine3A, Quat, Vec3A};
+use vulkano::image::ImageViewAbstract;
+
+use crate::state::AppState;
+
+use super::input::{DummyInteractionHandler, InteractionHandler, PointerHit};
+
+static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(0);
+
+pub trait OverlayBackend: OverlayRenderer + InteractionHandler {}
+
+pub struct OverlayState {
+    pub id: usize,
+    pub name: Arc<str>,
+    pub width: f32,
+    pub size: (i32, i32),
+    pub want_visible: bool,
+    pub show_hide: bool,
+    pub grabbable: bool,
+    pub transform: Affine3A,
+    pub spawn_point: Vec3A,
+    pub spawn_rotation: Quat,
+    pub relative_to: RelativeTo,
+    pub primary_pointer: Option<usize>,
+    pub interaction_transform: Affine3A,
+}
+
+impl Default for OverlayState {
+    fn default() -> Self {
+        OverlayState {
+            id: AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed),
+            name: Arc::from(""),
+            width: 1.,
+            size: (0, 0),
+            want_visible: false,
+            show_hide: false,
+            grabbable: false,
+            relative_to: RelativeTo::None,
+            spawn_point: Vec3A::NEG_Z,
+            spawn_rotation: Quat::IDENTITY,
+            transform: Affine3A::IDENTITY,
+            primary_pointer: None,
+            interaction_transform: Affine3A::IDENTITY,
+        }
+    }
+}
+
+pub struct OverlayData<T>
+where
+    T: Default,
+{
+    pub state: OverlayState,
+    pub backend: Box<dyn OverlayBackend>,
+    pub primary_pointer: Option<usize>,
+    pub data: T,
+}
+
+impl<T> Default for OverlayData<T>
+where
+    T: Default,
+{
+    fn default() -> Self {
+        OverlayData {
+            state: Default::default(),
+            backend: Box::new(SplitOverlayBackend::default()),
+            primary_pointer: None,
+            data: Default::default(),
+        }
+    }
+}
+
+impl OverlayState {
+    pub fn reset(&mut self, _app: &mut AppState) {
+        todo!()
+    }
+}
+
+impl<T> OverlayData<T>
+where
+    T: Default,
+{
+    pub fn init(&mut self, app: &mut AppState) {
+        self.backend.init(app);
+    }
+    pub fn render(&mut self, app: &mut AppState) {
+        self.backend.render(app);
+    }
+    pub fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>> {
+        self.backend.view()
+    }
+}
+
+pub trait OverlayRenderer {
+    fn init(&mut self, app: &mut AppState);
+    fn pause(&mut self, app: &mut AppState);
+    fn resume(&mut self, app: &mut AppState);
+    fn render(&mut self, app: &mut AppState);
+    fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>>;
+}
+
+pub struct FallbackRenderer;
+
+impl OverlayRenderer for FallbackRenderer {
+    fn init(&mut self, _app: &mut AppState) {}
+    fn pause(&mut self, _app: &mut AppState) {}
+    fn resume(&mut self, _app: &mut AppState) {}
+    fn render(&mut self, _app: &mut AppState) {}
+    fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>> {
+        unimplemented!()
+    }
+}
+// Boilerplate and dummies
+
+pub enum RelativeTo {
+    None,
+    Head,
+    Hand(usize),
+}
+
+pub struct SplitOverlayBackend {
+    pub renderer: Box<dyn OverlayRenderer>,
+    pub interaction: Box<dyn InteractionHandler>,
+}
+
+impl Default for SplitOverlayBackend {
+    fn default() -> SplitOverlayBackend {
+        SplitOverlayBackend {
+            renderer: Box::new(FallbackRenderer),
+            interaction: Box::new(DummyInteractionHandler),
+        }
+    }
+}
+
+impl OverlayBackend for SplitOverlayBackend {}
+impl OverlayRenderer for SplitOverlayBackend {
+    fn init(&mut self, app: &mut AppState) {
+        self.renderer.init(app);
+    }
+    fn pause(&mut self, app: &mut AppState) {
+        self.renderer.pause(app);
+    }
+    fn resume(&mut self, app: &mut AppState) {
+        self.renderer.resume(app);
+    }
+    fn render(&mut self, app: &mut AppState) {
+        self.renderer.render(app);
+    }
+    fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>> {
+        self.renderer.view()
+    }
+}
+impl InteractionHandler for SplitOverlayBackend {
+    fn on_left(&mut self, app: &mut AppState, pointer: usize) {
+        self.interaction.on_left(app, pointer);
+    }
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) {
+        self.interaction.on_hover(app, hit);
+    }
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32) {
+        self.interaction.on_scroll(app, hit, delta);
+    }
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
+        self.interaction.on_pointer(app, hit, pressed);
+    }
+}