initial commit

src/backend/common.rs

@@ -1,152 +1,158 @@
-use std::{collections::VecDeque, time::Instant};
+use std::{
+    collections::{BinaryHeap, VecDeque},
+    sync::Arc,
+    time::Instant,
+};
 
-use glam::{Affine3A, Vec2, Vec3, Vec3A};
-use ovr_overlay::TrackedDeviceIndex;
+use idmap::IdMap;
 
-pub struct InputState<TState, THand> {
-    pub hmd: Affine3A,
-    pub pointers: [Pointer<THand>; 2],
-    pub devices: Vec<TrackedDevice>,
-    pub(super) data: TState,
+use crate::{
+    overlays::{
+        keyboard::create_keyboard,
+        screen::{get_screens_wayland, get_screens_x11},
+        watch::create_watch,
+    },
+    state::AppState,
+};
+
+use super::overlay::{OverlayData, OverlayState};
+
+pub struct OverlayContainer<T>
+where
+    T: Default,
+{
+    overlays: IdMap<usize, OverlayData<T>>,
 }
 
-impl<TState, THand> InputState<TState, THand> {
-    pub fn pre_update(&mut self) {
-        self.pointers[0].before = self.pointers[0].now;
-        self.pointers[1].before = self.pointers[1].now;
+impl<T> OverlayContainer<T>
+where
+    T: Default,
+{
+    pub fn new(app: &mut AppState) -> Self {
+        let mut overlays = IdMap::new();
+
+        let screens = if std::env::var("WAYLAND_DISPLAY").is_ok() {
+            get_screens_wayland(&app.session)
+        } else {
+            get_screens_x11()
+        };
+
+        let watch = create_watch::<T>(&app, &screens);
+        overlays.insert(watch.state.id, watch);
+
+        let keyboard = create_keyboard(&app);
+        overlays.insert(keyboard.state.id, keyboard);
+
+        let mut first = true;
+        for mut screen in screens {
+            if first {
+                screen.state.want_visible = true;
+                first = false;
+            }
+            overlays.insert(screen.state.id, screen);
+        }
+
+        Self { overlays }
     }
 
-    pub fn post_update(&mut self) {
-        for hand in &mut self.pointers {
-            if hand.now.click_modifier_right {
-                hand.interaction.mode = PointerMode::Right;
-                continue;
-            }
-
-            if hand.now.click_modifier_middle {
-                hand.interaction.mode = PointerMode::Middle;
-                continue;
-            }
-
-            let hmd_up = self.hmd.transform_vector3a(Vec3A::Y);
-            let dot =
-                hmd_up.dot(hand.pose.transform_vector3a(Vec3A::X)) * (1.0 - 2.0 * hand.hand as f32);
-
-            hand.interaction.mode = if dot < -0.85 {
-                PointerMode::Right
-            } else if dot > 0.7 {
-                PointerMode::Middle
-            } else {
-                PointerMode::Left
-            };
-
-            let middle_click_orientation = false;
-            let right_click_orientation = false;
-            match hand.interaction.mode {
-                PointerMode::Middle => {
-                    if !middle_click_orientation {
-                        hand.interaction.mode = PointerMode::Left;
-                    }
-                }
-                PointerMode::Right => {
-                    if !right_click_orientation {
-                        hand.interaction.mode = PointerMode::Left;
-                    }
-                }
-                _ => {}
-            };
+    pub fn mut_by_selector(&mut self, selector: &OverlaySelector) -> Option<&mut OverlayData<T>> {
+        match selector {
+            OverlaySelector::Id(id) => self.mut_by_id(*id),
+            OverlaySelector::Name(name) => self.mut_by_name(name),
         }
     }
+
+    pub fn get_by_id<'a>(&'a mut self, id: usize) -> Option<&'a OverlayData<T>> {
+        self.overlays.get(&id)
+    }
+
+    pub fn mut_by_id<'a>(&'a mut self, id: usize) -> Option<&'a mut OverlayData<T>> {
+        self.overlays.get_mut(&id)
+    }
+
+    pub fn get_by_name<'a>(&'a mut self, name: &str) -> Option<&'a OverlayData<T>> {
+        self.overlays.values().find(|o| *o.state.name == *name)
+    }
+
+    pub fn mut_by_name<'a>(&'a mut self, name: &str) -> Option<&'a mut OverlayData<T>> {
+        self.overlays.values_mut().find(|o| *o.state.name == *name)
+    }
+
+    pub fn iter<'a>(&'a self) -> impl Iterator<Item = &'a OverlayData<T>> {
+        self.overlays.values()
+    }
+
+    pub fn iter_mut<'a>(&'a mut self) -> impl Iterator<Item = &'a mut OverlayData<T>> {
+        self.overlays.values_mut()
+    }
 }
 
-pub struct Pointer<THand> {
-    pub hand: usize,
-    pub pose: Affine3A,
-    pub now: PointerState,
-    pub before: PointerState,
-    pub(super) interaction: InteractionState,
-    pub(super) data: THand,
+pub enum OverlaySelector {
+    Id(usize),
+    Name(Arc<str>),
 }
 
-#[derive(Debug, Clone, Copy, Default)]
-pub struct PointerState {
-    pub scroll: f32,
-    pub click: bool,
-    pub grab: bool,
-    pub alt_click: bool,
-    pub show_hide: bool,
-    pub space_drag: bool,
-    pub click_modifier_right: bool,
-    pub click_modifier_middle: bool,
+struct AppTask {
+    pub not_before: Instant,
+    pub task: TaskType,
 }
 
-pub struct InteractionState {
-    pub mode: PointerMode,
-    pub grabbed: Option<GrabData>,
-    pub clicked_id: Option<usize>,
-    pub hovered_id: Option<usize>,
-    pub release_actions: VecDeque<Box<dyn Fn()>>,
-    pub next_push: Instant,
+impl PartialEq<AppTask> for AppTask {
+    fn eq(&self, other: &Self) -> bool {
+        self.not_before == other.not_before
+    }
+}
+impl PartialOrd<AppTask> for AppTask {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.not_before.cmp(&other.not_before).reverse())
+    }
+}
+impl Eq for AppTask {}
+impl Ord for AppTask {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.not_before.cmp(&other.not_before).reverse()
+    }
 }
 
-impl Default for InteractionState {
-    fn default() -> Self {
+pub enum TaskType {
+    Global(Box<dyn FnOnce(&mut AppState) + Send>),
+    Overlay(
+        OverlaySelector,
+        Box<dyn FnOnce(&mut AppState, &mut OverlayState) + Send>,
+    ),
+}
+
+pub struct TaskContainer {
+    tasks: BinaryHeap<AppTask>,
+}
+
+impl TaskContainer {
+    pub fn new() -> Self {
         Self {
-            mode: PointerMode::Left,
-            grabbed: None,
-            clicked_id: None,
-            hovered_id: None,
-            release_actions: VecDeque::new(),
-            next_push: Instant::now(),
+            tasks: BinaryHeap::new(),
+        }
+    }
+
+    pub fn enqueue(&mut self, task: TaskType) {
+        self.tasks.push(AppTask {
+            not_before: Instant::now(),
+            task,
+        });
+    }
+
+    pub fn enqueue_at(&mut self, task: TaskType, not_before: Instant) {
+        self.tasks.push(AppTask { not_before, task });
+    }
+
+    pub fn retrieve_due(&mut self, dest_buf: &mut VecDeque<TaskType>) {
+        let now = Instant::now();
+
+        while let Some(task) = self.tasks.peek() {
+            if task.not_before > now {
+                break;
+            }
+
+            dest_buf.push_back(self.tasks.pop().unwrap().task);
         }
     }
 }
-
-pub struct PointerHit {
-    pub hand: usize,
-    pub mode: PointerMode,
-    pub primary: bool,
-    pub uv: Vec2,
-    pub dist: f32,
-}
-
-struct RayHit {
-    idx: usize,
-    ray_pos: Vec3,
-    hit_pos: Vec3,
-    uv: Vec2,
-    dist: f32,
-}
-
-#[derive(Debug, Clone, Copy, Default)]
-pub struct GrabData {
-    pub offset: Vec3,
-    pub grabbed_id: usize,
-}
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, Default)]
-pub enum PointerMode {
-    #[default]
-    Left,
-    Right,
-    Middle,
-}
-
-pub struct TrackedDevice {
-    pub index: TrackedDeviceIndex,
-    pub valid: bool,
-    pub soc: Option<f32>,
-    pub charging: bool,
-    pub role: TrackedDeviceRole,
-}
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum TrackedDeviceRole {
-    None,
-    Hmd,
-    LeftHand,
-    RightHand,
-    Tracker,
-}

src/backend/input.rs

@@ -0,0 +1,362 @@
+use std::{collections::VecDeque, time::Instant};
+
+use glam::{Affine3A, Vec2, Vec3A};
+use log::warn;
+use ovr_overlay::TrackedDeviceIndex;
+use tinyvec::array_vec;
+
+use crate::state::AppState;
+
+use super::{common::OverlayContainer, overlay::OverlayData};
+
+pub struct TrackedDevice {
+    pub index: TrackedDeviceIndex,
+    pub valid: bool,
+    pub soc: Option<f32>,
+    pub charging: bool,
+    pub role: TrackedDeviceRole,
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TrackedDeviceRole {
+    None,
+    Hmd,
+    LeftHand,
+    RightHand,
+    Tracker,
+}
+
+pub struct InputState<TState, THand> {
+    pub hmd: Affine3A,
+    pub pointers: [Pointer<THand>; 2],
+    pub devices: Vec<TrackedDevice>,
+    pub(super) data: TState,
+}
+
+impl<TState, THand> InputState<TState, THand> {
+    pub fn pre_update(&mut self) {
+        self.pointers[0].before = self.pointers[0].now;
+        self.pointers[1].before = self.pointers[1].now;
+    }
+
+    pub fn post_update(&mut self) {
+        for hand in &mut self.pointers {
+            if hand.now.click_modifier_right {
+                hand.interaction.mode = PointerMode::Right;
+                continue;
+            }
+
+            if hand.now.click_modifier_middle {
+                hand.interaction.mode = PointerMode::Middle;
+                continue;
+            }
+
+            let hmd_up = self.hmd.transform_vector3a(Vec3A::Y);
+            let dot =
+                hmd_up.dot(hand.pose.transform_vector3a(Vec3A::X)) * (1.0 - 2.0 * hand.hand as f32);
+
+            hand.interaction.mode = if dot < -0.85 {
+                PointerMode::Right
+            } else if dot > 0.7 {
+                PointerMode::Middle
+            } else {
+                PointerMode::Left
+            };
+
+            let middle_click_orientation = false;
+            let right_click_orientation = false;
+            match hand.interaction.mode {
+                PointerMode::Middle => {
+                    if !middle_click_orientation {
+                        hand.interaction.mode = PointerMode::Left;
+                    }
+                }
+                PointerMode::Right => {
+                    if !right_click_orientation {
+                        hand.interaction.mode = PointerMode::Left;
+                    }
+                }
+                _ => {}
+            };
+        }
+    }
+}
+
+pub struct InteractionState {
+    pub mode: PointerMode,
+    pub grabbed: Option<GrabData>,
+    pub clicked_id: Option<usize>,
+    pub hovered_id: Option<usize>,
+    pub release_actions: VecDeque<Box<dyn Fn()>>,
+    pub next_push: Instant,
+}
+
+impl Default for InteractionState {
+    fn default() -> Self {
+        Self {
+            mode: PointerMode::Left,
+            grabbed: None,
+            clicked_id: None,
+            hovered_id: None,
+            release_actions: VecDeque::new(),
+            next_push: Instant::now(),
+        }
+    }
+}
+
+pub struct Pointer<THand> {
+    pub idx: usize,
+    pub hand: u8,
+    pub pose: Affine3A,
+    pub now: PointerState,
+    pub before: PointerState,
+    pub(super) interaction: InteractionState,
+    pub(super) data: THand,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct PointerState {
+    pub scroll: f32,
+    pub click: bool,
+    pub grab: bool,
+    pub alt_click: bool,
+    pub show_hide: bool,
+    pub space_drag: bool,
+    pub click_modifier_right: bool,
+    pub click_modifier_middle: bool,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct PointerHit {
+    pub pointer: usize,
+    pub overlay: usize,
+    pub mode: PointerMode,
+    pub primary: bool,
+    pub uv: Vec2,
+    pub dist: f32,
+}
+
+pub trait InteractionHandler {
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit);
+    fn on_left(&mut self, app: &mut AppState, pointer: usize);
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool);
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32);
+}
+
+pub struct DummyInteractionHandler;
+
+impl InteractionHandler for DummyInteractionHandler {
+    fn on_left(&mut self, _app: &mut AppState, _pointer: usize) {}
+    fn on_hover(&mut self, _app: &mut AppState, _hit: &PointerHit) {}
+    fn on_pointer(&mut self, _app: &mut AppState, _hit: &PointerHit, _pressed: bool) {}
+    fn on_scroll(&mut self, _app: &mut AppState, _hit: &PointerHit, _delta: f32) {}
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+struct RayHit {
+    overlay: usize,
+    hit_pos: Vec3A,
+    dist: f32,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct GrabData {
+    pub offset: Vec3A,
+    pub grabbed_id: usize,
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, Default)]
+pub enum PointerMode {
+    #[default]
+    Left,
+    Right,
+    Middle,
+}
+
+impl<THand> Pointer<THand> {
+    pub fn interact<O>(&mut self, overlays: &mut OverlayContainer<O>, app: &mut AppState)
+    where
+        O: Default,
+    {
+        if let Some(grab_data) = self.interaction.grabbed {
+            if let Some(grabbed) = overlays.mut_by_id(grab_data.grabbed_id) {
+                self.handle_grabbed(grabbed, grab_data.offset);
+            } else {
+                warn!("Grabbed overlay {} does not exist", grab_data.grabbed_id);
+                self.interaction.grabbed = None;
+            }
+            return;
+        }
+
+        let Some(mut hit) = self.get_nearest_hit(overlays) else {
+            if let Some(hovered_id) = self.interaction.hovered_id.take() {
+                if let Some(hovered) = overlays.mut_by_id(hovered_id) {
+                    hovered.backend.on_left(app, self.idx);
+                }
+                self.interaction.hovered_id = None;
+            }
+            if let Some(clicked_id) = self.interaction.clicked_id.take() {
+                if let Some(clicked) = overlays.mut_by_id(clicked_id) {
+                    let hit = PointerHit {
+                        pointer: self.idx,
+                        overlay: clicked_id,
+                        mode: self.interaction.mode,
+                        ..Default::default()
+                    };
+                    clicked.backend.on_pointer(app, &hit, false);
+                }
+            }
+            return;
+        };
+
+        if let Some(hovered_id) = self.interaction.hovered_id {
+            if hovered_id != hit.overlay {
+                if let Some(old_hovered) = overlays.mut_by_id(hovered_id) {
+                    if Some(self.idx) == old_hovered.primary_pointer {
+                        old_hovered.primary_pointer = None;
+                    }
+                    old_hovered.backend.on_left(app, self.idx);
+                }
+            }
+        }
+        let Some(hovered) = overlays.mut_by_id(hit.overlay) else {
+            warn!("Hit overlay {} does not exist", hit.overlay);
+            return;
+        };
+
+        self.interaction.hovered_id = Some(hit.overlay);
+
+        if let Some(primary_pointer) = hovered.primary_pointer {
+            if hit.pointer < primary_pointer {
+                hovered.primary_pointer = Some(hit.pointer);
+                hit.primary = true;
+            }
+        } else {
+            hovered.primary_pointer = Some(hit.pointer);
+            hit.primary = true;
+        }
+        hovered.backend.on_hover(app, &hit);
+
+        if self.now.scroll.abs() > 0.1 {
+            hovered.backend.on_scroll(app, &hit, self.now.scroll);
+        }
+
+        if self.now.click && !self.before.click {
+            self.interaction.clicked_id = Some(hit.overlay);
+            hovered.backend.on_pointer(app, &hit, true);
+        } else if !self.now.click && self.before.click {
+            if let Some(clicked_id) = self.interaction.clicked_id.take() {
+                if let Some(clicked) = overlays.mut_by_id(clicked_id) {
+                    clicked.backend.on_pointer(app, &hit, false);
+                }
+            } else {
+                hovered.backend.on_pointer(app, &hit, false);
+            }
+        }
+    }
+
+    fn get_nearest_hit<O>(&mut self, overlays: &mut OverlayContainer<O>) -> Option<PointerHit>
+    where
+        O: Default,
+    {
+        let mut hits = array_vec!([RayHit; 8]);
+
+        for overlay in overlays.iter() {
+            if !overlay.state.want_visible {
+                continue;
+            }
+
+            if let Some(hit) = self.ray_test(overlay.state.id, &overlay.state.transform) {
+                hits.try_push(hit);
+            }
+        }
+
+        hits.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap());
+
+        for hit in hits.iter() {
+            let uv = overlays
+                .get_by_id(hit.overlay)
+                .unwrap() // this is safe
+                .state
+                .transform
+                .inverse()
+                .transform_point3a(hit.hit_pos)
+                .truncate();
+
+            if uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0 {
+                continue;
+            }
+
+            return Some(PointerHit {
+                pointer: self.idx,
+                overlay: hit.overlay,
+                mode: self.interaction.mode,
+                primary: false,
+                uv,
+                dist: hit.dist,
+            });
+        }
+
+        None
+    }
+
+    fn start_grab<O>(&mut self, overlay: &mut OverlayData<O>)
+    where
+        O: Default,
+    {
+        let offset = self
+            .pose
+            .inverse()
+            .transform_point3a(overlay.state.transform.translation);
+
+        self.interaction.grabbed = Some(GrabData {
+            offset,
+            grabbed_id: overlay.state.id,
+        });
+    }
+    fn handle_grabbed<O>(&mut self, overlay: &mut OverlayData<O>, offset: Vec3A)
+    where
+        O: Default,
+    {
+        if self.now.grab {
+            overlay.state.transform.translation = self.pose.transform_point3a(offset);
+
+            if self.now.click && !self.before.click {
+                warn!("todo: click-while-grabbed");
+            }
+
+            match self.interaction.mode {
+                PointerMode::Left => {
+                    overlay.state.transform.translation.y += self.now.scroll * 0.01;
+                }
+                _ => {
+                    overlay.state.transform.matrix3 = overlay
+                        .state
+                        .transform
+                        .matrix3
+                        .mul_scalar(1.0 + 0.01 * self.now.scroll);
+                }
+            }
+        } else {
+            overlay.state.spawn_point = overlay.state.transform.translation;
+            self.interaction.grabbed = None;
+        }
+    }
+    fn ray_test(&self, overlay: usize, plane: &Affine3A) -> Option<RayHit> {
+        let plane_normal = plane.transform_vector3a(Vec3A::NEG_Z);
+        let ray_dir = self.pose.transform_vector3a(Vec3A::NEG_Z);
+
+        let d = plane.translation.dot(-plane_normal);
+        let dist = -(d + self.pose.translation.dot(plane_normal)) / ray_dir.dot(plane_normal);
+
+        let hit_pos = self.pose.translation + ray_dir * dist;
+
+        Some(RayHit {
+            overlay,
+            hit_pos,
+            dist,
+        })
+    }
+}

src/backend/mod.rs

@@ -1,3 +1,5 @@
 pub mod common;
+pub mod input;
 pub mod openvr;
 pub mod openxr;
+pub mod overlay;

src/backend/openvr/input.rs

@@ -1,4 +1,4 @@
-use std::array;
+use std::{array, io::Write, path::Path};
 
 use glam::Affine3A;
 use ovr_overlay::{
@@ -11,7 +11,7 @@ use ovr_overlay::{
     TrackedDeviceIndex,
 };
 
-use crate::backend::common::{
+use crate::backend::input::{
     InputState, InteractionState, Pointer, PointerState, TrackedDevice, TrackedDeviceRole,
 };
 
@@ -96,7 +96,8 @@ impl InputState<OpenVrInputState, OpenVrHandState> {
             .collect::<Result<_, &'static str>>()?;
 
         let hands: [Pointer<OpenVrHandState>; 2] = array::from_fn(|i| Pointer::<OpenVrHandState> {
-            hand: i,
+            idx: i,
+            hand: i as u8,
             now: PointerState::default(),
             before: PointerState::default(),
             pose: Affine3A::IDENTITY,
@@ -286,3 +287,30 @@ fn copy_from_hmd(in_mat: &HmdMatrix34_t, out_mat: &mut Affine3A) {
     out_mat.w_axis[1] = in_mat.m[1][3];
     out_mat.w_axis[2] = in_mat.m[2][3];
 }
+
+pub fn action_manifest_path() -> &'static Path {
+    let action_path = "/tmp/wlxoverlay-s/actions.json";
+    std::fs::create_dir_all("/tmp/wlxoverlay-s").unwrap();
+
+    std::fs::File::create(action_path)
+        .unwrap()
+        .write_all(include_bytes!("../../res/actions.json"))
+        .unwrap();
+
+    std::fs::File::create("/tmp/wlxoverlay-s/actions_binding_knuckles.json")
+        .unwrap()
+        .write_all(include_bytes!("../../res/actions_binding_knuckles.json"))
+        .unwrap();
+
+    std::fs::File::create("/tmp/wlxoverlay-s/actions_binding_vive.json")
+        .unwrap()
+        .write_all(include_bytes!("../../res/actions_binding_vive.json"))
+        .unwrap();
+
+    std::fs::File::create("/tmp/wlxoverlay-s/actions_binding_oculus.json")
+        .unwrap()
+        .write_all(include_bytes!("../../res/actions_binding_oculus.json"))
+        .unwrap();
+
+    Path::new(action_path)
+}

src/backend/openvr/mod.rs

@@ -1,4 +1,5 @@
 use std::{
+    collections::VecDeque,
     path::Path,
     time::{Duration, Instant},
 };
@@ -8,45 +9,73 @@ use ovr_overlay::{
     sys::{ETrackedDeviceProperty, EVRApplicationType, EVREventType},
     TrackedDeviceIndex,
 };
+use vulkano::{
+    device::{physical::PhysicalDevice, DeviceExtensions},
+    instance::InstanceExtensions,
+    Handle, VulkanObject,
+};
 
-use super::common::InputState;
+use crate::state::AppState;
+
+use self::{input::action_manifest_path, overlay::OpenVrOverlayData};
+
+use super::{
+    common::{OverlayContainer, TaskType},
+    input::InputState,
+};
 
 pub mod input;
 pub mod overlay;
 
-fn openvr_run() {
+pub fn openvr_run() {
     let app_type = EVRApplicationType::VRApplication_Overlay;
     let Ok(context) = ovr_overlay::Context::init(app_type) else {
             error!("Failed to initialize OpenVR");
             return;
         };
 
-    let mut overlay = context.overlay_mngr();
-    let mut settings = context.settings_mngr();
-    let mut input = context.input_mngr();
-    let mut system = context.system_mngr();
-    let mut compositor = context.compositor_mngr();
+    let mut overlay_mngr = context.overlay_mngr();
+    //let mut settings_mngr = context.settings_mngr();
+    let mut input_mngr = context.input_mngr();
+    let mut system_mngr = context.system_mngr();
+    let mut compositor_mngr = context.compositor_mngr();
 
-    let Ok(_) = input.set_action_manifest(Path::new("resources/actions.json")) else {
-        error!("Failed to set action manifest");
+    let device_extensions_fn = |device: &PhysicalDevice| {
+        let names = compositor_mngr.get_vulkan_device_extensions_required(device.handle().as_raw());
+        let ext = DeviceExtensions::from_iter(names.iter().map(|s| s.as_str()));
+        ext
+    };
+
+    let mut compositor_mngr = context.compositor_mngr();
+    let instance_extensions = {
+        let names = compositor_mngr.get_vulkan_instance_extensions_required();
+        InstanceExtensions::from_iter(names.iter().map(|s| s.as_str()))
+    };
+
+    let mut state = AppState::new(instance_extensions, device_extensions_fn);
+    let mut overlays = OverlayContainer::<OpenVrOverlayData>::new(&mut state);
+
+    if let Err(e) = input_mngr.set_action_manifest(action_manifest_path()) {
+        error!("Failed to set action manifest: {}", e.description());
         return;
     };
 
-    let Ok(mut input_state) = InputState::new(&mut input) else {
+    let Ok(mut input) = InputState::new(&mut input_mngr) else {
         error!("Failed to initialize input");
         return;
     };
 
-    let Ok(refresh_rate) = system.get_tracked_device_property::<f32>(TrackedDeviceIndex::HMD, ETrackedDeviceProperty::Prop_DisplayFrequency_Float) else {
+    let Ok(refresh_rate) = system_mngr.get_tracked_device_property::<f32>(TrackedDeviceIndex::HMD, ETrackedDeviceProperty::Prop_DisplayFrequency_Float) else {
         error!("Failed to get display refresh rate");
         return;
     };
 
     let frame_time = (1000.0 / refresh_rate).floor() * 0.001;
     let mut next_device_update = Instant::now();
+    let mut due_tasks = VecDeque::with_capacity(4);
 
     loop {
-        while let Some(event) = system.poll_next_event() {
+        while let Some(event) = system_mngr.poll_next_event() {
             match event.event_type {
                 EVREventType::VREvent_Quit => {
                     info!("Received quit event, shutting down.");
@@ -61,35 +90,57 @@ fn openvr_run() {
             }
 
             if next_device_update <= Instant::now() {
-                input_state.update_devices(&mut system);
+                input.update_devices(&mut system_mngr);
                 next_device_update = Instant::now() + Duration::from_secs(30);
             }
 
-            input_state.pre_update();
-            input_state.update(&mut input, &mut system);
-            input_state.post_update();
+            state.tasks.retrieve_due(&mut due_tasks);
+            while let Some(task) = due_tasks.pop_front() {
+                match task {
+                    TaskType::Global(f) => f(&mut state),
+                    TaskType::Overlay(sel, f) => {
+                        if let Some(o) = overlays.mut_by_selector(&sel) {
+                            f(&mut state, &mut o.state);
+                        }
+                    }
+                }
+            }
 
-            // task scheduler
+            input.pre_update();
+            input.update(&mut input_mngr, &mut system_mngr);
+            input.post_update();
 
-            // after input
+            input
+                .pointers
+                .iter_mut()
+                .for_each(|p| p.interact(&mut overlays, &mut state));
 
-            // interactions
+            overlays
+                .iter_mut()
+                .for_each(|o| o.after_input(&mut overlay_mngr, &mut state));
 
-            // show overlays
+            log::debug!("Rendering frame");
+
+            overlays
+                .iter_mut()
+                .filter(|o| o.state.want_visible)
+                .for_each(|o| o.render(&mut state));
+
+            log::debug!("Rendering overlays");
+
+            overlays
+                .iter_mut()
+                .for_each(|o| o.after_render(&mut overlay_mngr, &state.graphics));
 
             // chaperone
 
-            // render overlays
-
-            // hide overlays
-
-            // close font handles
+            // close font handles?
 
             // playspace moved end frame
 
             let mut seconds_since_vsync = 0f32;
             std::thread::sleep(Duration::from_secs_f32(
-                if system.get_time_since_last_vsync(&mut seconds_since_vsync, &mut 0u64) {
+                if system_mngr.get_time_since_last_vsync(&mut seconds_since_vsync, &mut 0u64) {
                     frame_time - seconds_since_vsync
                 } else {
                     0.011

src/backend/openvr/overlay.rs

@@ -1,15 +1,207 @@
-use ovr_overlay::sys::VRVulkanTextureData_t;
+use glam::Vec4;
+use ovr_overlay::{
+    overlay::{OverlayHandle, OverlayManager},
+    sys::VRVulkanTextureData_t,
+};
+use vulkano::{
+    command_buffer::{
+        synced::{
+            SyncCommandBuffer, SyncCommandBufferBuilder, SyncCommandBufferBuilderExecuteCommands,
+        },
+        AutoCommandBufferBuilder, CommandBufferExecFuture,
+    },
+    image::{ImageAccess, ImageLayout},
+    sync::{future::NowFuture, ImageMemoryBarrier},
+    Handle, VulkanObject,
+};
 
-use crate::overlays::OverlayData;
+use crate::{backend::overlay::OverlayData, graphics::WlxGraphics, state::AppState};
 
-pub(super) struct OpenVrOverlayManager {
-    pub(super) overlays: Vec<OpenVrOverlay>,
-}
-
-pub(super) struct OpenVrOverlay {
+#[derive(Default)]
+pub(super) struct OpenVrOverlayData {
+    handle: Option<OverlayHandle>,
+    last_image: Option<u64>,
     pub(super) visible: bool,
-    pub(super) color: [f32; 4],
-    overlay: OverlayData,
-    handle: u32,
-    ovr_texture: VRVulkanTextureData_t,
+    pub(super) color: Vec4,
+    pub(super) curvature: f32,
+    pub(super) sort_order: u32,
+}
+
+impl OverlayData<OpenVrOverlayData> {
+    pub fn initialize(
+        &mut self,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> OverlayHandle {
+        let key = format!("wlx-{}", self.state.name);
+        let handle = match overlay.create_overlay(&key, &key) {
+            Ok(handle) => handle,
+            Err(e) => {
+                panic!("Failed to create overlay: {}", e);
+            }
+        };
+        log::debug!("{}: initialize", self.state.name);
+
+        self.data.handle = Some(handle);
+
+        self.init(app);
+
+        self.upload_width(overlay);
+        self.upload_color(overlay);
+        self.upload_curvature(overlay);
+
+        handle
+    }
+
+    pub fn after_input(&mut self, overlay: &mut OverlayManager, app: &mut AppState) {
+        if self.state.want_visible && !self.data.visible {
+            self.show(overlay, app);
+        } else if !self.state.want_visible && self.data.visible {
+            self.hide(overlay);
+        }
+    }
+
+    pub fn after_render(&mut self, overlay: &mut OverlayManager, graphics: &WlxGraphics) {
+        if self.data.visible {
+            self.upload_texture(overlay, graphics);
+        }
+    }
+
+    fn show(&mut self, overlay: &mut OverlayManager, app: &mut AppState) {
+        let handle = match self.data.handle {
+            Some(handle) => handle,
+            None => self.initialize(overlay, app),
+        };
+        log::debug!("{}: show", self.state.name);
+        if let Err(e) = overlay.set_visibility(handle, true) {
+            panic!("Failed to show overlay: {}", e);
+        }
+        self.data.visible = true;
+    }
+
+    fn hide(&mut self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            return;
+        };
+        log::debug!("{}: hide", self.state.name);
+        if let Err(e) = overlay.set_visibility(handle, false) {
+            panic!("Failed to hide overlay: {}", e);
+        }
+        self.data.visible = false;
+    }
+
+    fn upload_color(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_opacity(handle, self.data.color.w) {
+            panic!("Failed to set overlay opacity: {}", e);
+        }
+        if let Err(e) = overlay.set_tint(
+            handle,
+            ovr_overlay::ColorTint {
+                r: self.data.color.x,
+                g: self.data.color.y,
+                b: self.data.color.z,
+                a: 1.0,
+            },
+        ) {
+            panic!("Failed to set overlay tint: {}", e);
+        }
+    }
+
+    fn upload_width(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_width(handle, self.state.width) {
+            panic!("Failed to set overlay width: {}", e);
+        }
+    }
+
+    fn upload_curvature(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_curvature(handle, self.data.curvature) {
+            panic!("Failed to set overlay curvature: {}", e);
+        }
+    }
+
+    fn upload_sort_order(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_sort_order(handle, self.data.sort_order) {
+            panic!("Failed to set overlay z order: {}", e);
+        }
+    }
+
+    fn upload_texture(&mut self, overlay: &mut OverlayManager, graphics: &WlxGraphics) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+
+        let Some(view) = self.backend.view() else {
+            log::debug!("{}: Not rendered", self.state.name);
+            return;
+        };
+
+        let image = view.image().inner().image.clone();
+
+        let raw_image = image.handle().as_raw();
+
+        if let Some(last_image) = self.data.last_image {
+            if last_image == raw_image {
+                return;
+            }
+        }
+
+        let Some(format) = image.format() else {
+            panic!("{}: Image format is None", self.state.name);
+        };
+
+        let dimensions = image.dimensions();
+
+        let mut texture = VRVulkanTextureData_t {
+            m_nImage: raw_image,
+            m_nFormat: format as _,
+            m_nWidth: dimensions.width(),
+            m_nHeight: dimensions.height(),
+            m_nSampleCount: image.samples() as u32,
+            m_pDevice: graphics.device.handle().as_raw() as *mut _,
+            m_pPhysicalDevice: graphics.device.physical_device().handle().as_raw() as *mut _,
+            m_pInstance: graphics.instance.handle().as_raw() as *mut _,
+            m_pQueue: graphics.queue.handle().as_raw() as *mut _,
+            m_nQueueFamilyIndex: graphics.queue.queue_family_index(),
+        };
+
+        graphics
+            .transition_layout(
+                image.clone(),
+                ImageLayout::ColorAttachmentOptimal,
+                ImageLayout::TransferSrcOptimal,
+            )
+            .wait(None)
+            .unwrap();
+
+        log::info!("nImage: {}, nFormat: {:?}, nWidth: {}, nHeight: {}, nSampleCount: {}, nQueueFamilyIndex: {}", texture.m_nImage, format, texture.m_nWidth, texture.m_nHeight, texture.m_nSampleCount, texture.m_nQueueFamilyIndex);
+        if let Err(e) = overlay.set_image_vulkan(handle, &mut texture) {
+            panic!("Failed to set overlay texture: {}", e);
+        }
+
+        graphics
+            .transition_layout(
+                image,
+                ImageLayout::TransferSrcOptimal,
+                ImageLayout::ColorAttachmentOptimal,
+            )
+            .wait(None)
+            .unwrap();
+    }
 }

src/backend/overlay.rs

@@ -0,0 +1,170 @@
+use std::sync::{
+    atomic::{AtomicUsize, Ordering},
+    Arc,
+};
+
+use glam::{Affine3A, Quat, Vec3A};
+use vulkano::image::ImageViewAbstract;
+
+use crate::state::AppState;
+
+use super::input::{DummyInteractionHandler, InteractionHandler, PointerHit};
+
+static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(0);
+
+pub trait OverlayBackend: OverlayRenderer + InteractionHandler {}
+
+pub struct OverlayState {
+    pub id: usize,
+    pub name: Arc<str>,
+    pub width: f32,
+    pub size: (i32, i32),
+    pub want_visible: bool,
+    pub show_hide: bool,
+    pub grabbable: bool,
+    pub transform: Affine3A,
+    pub spawn_point: Vec3A,
+    pub spawn_rotation: Quat,
+    pub relative_to: RelativeTo,
+    pub primary_pointer: Option<usize>,
+    pub interaction_transform: Affine3A,
+}
+
+impl Default for OverlayState {
+    fn default() -> Self {
+        OverlayState {
+            id: AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed),
+            name: Arc::from(""),
+            width: 1.,
+            size: (0, 0),
+            want_visible: false,
+            show_hide: false,
+            grabbable: false,
+            relative_to: RelativeTo::None,
+            spawn_point: Vec3A::NEG_Z,
+            spawn_rotation: Quat::IDENTITY,
+            transform: Affine3A::IDENTITY,
+            primary_pointer: None,
+            interaction_transform: Affine3A::IDENTITY,
+        }
+    }
+}
+
+pub struct OverlayData<T>
+where
+    T: Default,
+{
+    pub state: OverlayState,
+    pub backend: Box<dyn OverlayBackend>,
+    pub primary_pointer: Option<usize>,
+    pub data: T,
+}
+
+impl<T> Default for OverlayData<T>
+where
+    T: Default,
+{
+    fn default() -> Self {
+        OverlayData {
+            state: Default::default(),
+            backend: Box::new(SplitOverlayBackend::default()),
+            primary_pointer: None,
+            data: Default::default(),
+        }
+    }
+}
+
+impl OverlayState {
+    pub fn reset(&mut self, _app: &mut AppState) {
+        todo!()
+    }
+}
+
+impl<T> OverlayData<T>
+where
+    T: Default,
+{
+    pub fn init(&mut self, app: &mut AppState) {
+        self.backend.init(app);
+    }
+    pub fn render(&mut self, app: &mut AppState) {
+        self.backend.render(app);
+    }
+    pub fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>> {
+        self.backend.view()
+    }
+}
+
+pub trait OverlayRenderer {
+    fn init(&mut self, app: &mut AppState);
+    fn pause(&mut self, app: &mut AppState);
+    fn resume(&mut self, app: &mut AppState);
+    fn render(&mut self, app: &mut AppState);
+    fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>>;
+}
+
+pub struct FallbackRenderer;
+
+impl OverlayRenderer for FallbackRenderer {
+    fn init(&mut self, _app: &mut AppState) {}
+    fn pause(&mut self, _app: &mut AppState) {}
+    fn resume(&mut self, _app: &mut AppState) {}
+    fn render(&mut self, _app: &mut AppState) {}
+    fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>> {
+        unimplemented!()
+    }
+}
+// Boilerplate and dummies
+
+pub enum RelativeTo {
+    None,
+    Head,
+    Hand(usize),
+}
+
+pub struct SplitOverlayBackend {
+    pub renderer: Box<dyn OverlayRenderer>,
+    pub interaction: Box<dyn InteractionHandler>,
+}
+
+impl Default for SplitOverlayBackend {
+    fn default() -> SplitOverlayBackend {
+        SplitOverlayBackend {
+            renderer: Box::new(FallbackRenderer),
+            interaction: Box::new(DummyInteractionHandler),
+        }
+    }
+}
+
+impl OverlayBackend for SplitOverlayBackend {}
+impl OverlayRenderer for SplitOverlayBackend {
+    fn init(&mut self, app: &mut AppState) {
+        self.renderer.init(app);
+    }
+    fn pause(&mut self, app: &mut AppState) {
+        self.renderer.pause(app);
+    }
+    fn resume(&mut self, app: &mut AppState) {
+        self.renderer.resume(app);
+    }
+    fn render(&mut self, app: &mut AppState) {
+        self.renderer.render(app);
+    }
+    fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>> {
+        self.renderer.view()
+    }
+}
+impl InteractionHandler for SplitOverlayBackend {
+    fn on_left(&mut self, app: &mut AppState, pointer: usize) {
+        self.interaction.on_left(app, pointer);
+    }
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) {
+        self.interaction.on_hover(app, hit);
+    }
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32) {
+        self.interaction.on_scroll(app, hit, delta);
+    }
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
+        self.interaction.on_pointer(app, hit, pressed);
+    }
+}

src/graphics.rs

@@ -1,10 +1,67 @@
-use std::{sync::Arc, slice::Iter, io::Cursor, error::Error};
+use std::{error::Error, io::Cursor, slice::Iter, sync::Arc};
 
-use log::{info,error};
-use vulkano::{format::Format, device::{physical::PhysicalDeviceType, QueueFlags, DeviceExtensions, Device, DeviceCreateInfo, Features, QueueCreateInfo, Queue}, Version, VulkanLibrary, instance::{Instance, InstanceCreateInfo}, memory::allocator::{StandardMemoryAllocator, AllocationCreateInfo, MemoryUsage}, command_buffer::{allocator::StandardCommandBufferAllocator, CommandBufferUsage, AutoCommandBufferBuilder, PrimaryAutoCommandBuffer, RenderingAttachmentInfo, RenderingInfo, PrimaryCommandBufferAbstract, CommandBufferExecFuture, SubpassContents, SecondaryAutoCommandBuffer, CommandBufferInheritanceInfo, CommandBufferInheritanceRenderPassType, CommandBufferInheritanceRenderingInfo}, descriptor_set::{allocator::StandardDescriptorSetAllocator, PersistentDescriptorSet, WriteDescriptorSet}, buffer::{Buffer, BufferCreateInfo, BufferUsage, BufferContents, Subbuffer, allocator::{SubbufferAllocator, SubbufferAllocatorCreateInfo}}, sampler::{Filter, SamplerCreateInfo, SamplerAddressMode, Sampler}, pipeline::{GraphicsPipeline, Pipeline, graphics::{render_pass::PipelineRenderingCreateInfo, vertex_input::Vertex, input_assembly::InputAssemblyState, viewport::{ViewportState, Viewport}, color_blend::{ColorBlendState, AttachmentBlend}}, PipelineBindPoint}, image::{ImageViewAbstract, ImageUsage, SwapchainImage, ImageDimensions, ImmutableImage, MipmapsCount, StorageImage, ImageError, SubresourceData, ImageCreateFlags, AttachmentImage}, swapchain::{Surface, Swapchain, SwapchainCreateInfo, CompositeAlpha}, shader::ShaderModule, render_pass::{StoreOp, LoadOp}, sync::future::NowFuture};
-use winit::{event_loop::EventLoop, window::{WindowBuilder, Window}};
+use ash::vk::SubmitInfo;
+use log::{debug, error, info};
+use smallvec::smallvec;
+use vulkano::{
+    buffer::{
+        allocator::{SubbufferAllocator, SubbufferAllocatorCreateInfo},
+        Buffer, BufferContents, BufferCreateInfo, BufferUsage, Subbuffer,
+    },
+    command_buffer::{
+        allocator::{
+            CommandBufferAllocator, CommandBufferBuilderAlloc, StandardCommandBufferAllocator,
+        },
+        sys::{CommandBufferBeginInfo, UnsafeCommandBufferBuilder},
+        AutoCommandBufferBuilder, CommandBufferExecFuture, CommandBufferInheritanceInfo,
+        CommandBufferInheritanceRenderPassType, CommandBufferInheritanceRenderingInfo,
+        CommandBufferLevel, CommandBufferUsage, PrimaryAutoCommandBuffer,
+        PrimaryCommandBufferAbstract, RenderingAttachmentInfo, RenderingInfo,
+        SecondaryAutoCommandBuffer, SubpassContents,
+    },
+    descriptor_set::{
+        allocator::StandardDescriptorSetAllocator, PersistentDescriptorSet, WriteDescriptorSet,
+    },
+    device::{
+        physical::{PhysicalDevice, PhysicalDeviceType},
+        Device, DeviceCreateInfo, DeviceExtensions, Features, Queue, QueueCreateInfo, QueueFlags,
+    },
+    format::Format,
+    image::{
+        sys::Image, AttachmentImage, ImageAccess, ImageCreateFlags, ImageDimensions, ImageError,
+        ImageLayout, ImageUsage, ImageViewAbstract, ImmutableImage, MipmapsCount, StorageImage,
+        SubresourceData, SwapchainImage,
+    },
+    instance::{Instance, InstanceCreateInfo, InstanceExtensions},
+    memory::allocator::{AllocationCreateInfo, MemoryUsage, StandardMemoryAllocator},
+    pipeline::{
+        graphics::{
+            color_blend::{AttachmentBlend, ColorBlendState},
+            input_assembly::InputAssemblyState,
+            render_pass::PipelineRenderingCreateInfo,
+            vertex_input::Vertex,
+            viewport::{Viewport, ViewportState},
+        },
+        GraphicsPipeline, Pipeline, PipelineBindPoint,
+    },
+    render_pass::{LoadOp, StoreOp},
+    sampler::{Filter, Sampler, SamplerAddressMode, SamplerCreateInfo},
+    shader::ShaderModule,
+    swapchain::{CompositeAlpha, Surface, Swapchain, SwapchainCreateInfo},
+    sync::{
+        fence::Fence, future::NowFuture, AccessFlags, DependencyInfo, ImageMemoryBarrier,
+        PipelineStages,
+    },
+    Version, VulkanLibrary, VulkanObject,
+};
 use vulkano_win::VkSurfaceBuild;
-use wlx_capture::frame::{DmabufFrame, DRM_FORMAT_ABGR8888, DRM_FORMAT_XBGR8888, DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB8888};
+use winit::{
+    event_loop::EventLoop,
+    window::{Window, WindowBuilder},
+};
+use wlx_capture::frame::{
+    DmabufFrame, DRM_FORMAT_ABGR8888, DRM_FORMAT_ARGB8888, DRM_FORMAT_XBGR8888, DRM_FORMAT_XRGB8888,
+};
 
 #[repr(C)]
 #[derive(BufferContents, Vertex, Copy, Clone, Debug)]
@@ -15,10 +72,10 @@ pub struct Vert2Uv {
     pub in_uv: [f32; 2],
 }
 
-pub const INDICES : [u16; 6] = [2, 1, 0, 1, 2, 3];
+pub const INDICES: [u16; 6] = [2, 1, 0, 1, 2, 3];
 
 pub struct WlxGraphics {
-    pub instance: Arc<Instance>, 
+    pub instance: Arc<Instance>,
     pub device: Arc<Device>,
     pub queue: Arc<Queue>,
 
@@ -28,18 +85,26 @@ pub struct WlxGraphics {
     pub command_buffer_allocator: Arc<StandardCommandBufferAllocator>,
     pub descriptor_set_allocator: Arc<StandardDescriptorSetAllocator>,
 
+    pub quad_verts: Subbuffer<[Vert2Uv]>,
     pub quad_indices: Subbuffer<[u16]>,
 }
 
 impl WlxGraphics {
-    pub fn new() -> (Arc<Self>, EventLoop<()>) {
+    pub fn new(
+        vk_instance_extensions: InstanceExtensions,
+        mut vk_device_extensions_fn: impl FnMut(&PhysicalDevice) -> DeviceExtensions,
+    ) -> (Arc<Self>, EventLoop<()>) {
         #[cfg(debug_assertions)]
         let layers = vec!["VK_LAYER_KHRONOS_validation".to_owned()];
         #[cfg(not(debug_assertions))]
         let layers = vec![];
 
         let library = VulkanLibrary::new().unwrap();
-        let required_extensions = vulkano_win::required_extensions(&library);
+        let library_extensions = vulkano_win::required_extensions(&library);
+        let required_extensions = library_extensions.union(&vk_instance_extensions);
+
+        debug!("Instance exts for app: {:?}", &required_extensions);
+        debug!("Instance exts for runtime: {:?}", &vk_instance_extensions);
 
         let instance = Instance::new(
             library,
@@ -61,13 +126,14 @@ impl WlxGraphics {
             ..DeviceExtensions::empty()
         };
 
+        debug!("Device exts for app: {:?}", &device_extensions);
+
         // TODO headless
         let event_loop = EventLoop::new();
         let surface = WindowBuilder::new()
             .build_vk_surface(&event_loop, instance.clone())
             .unwrap();
 
-
         let (physical_device, queue_family_index) = instance
             .enumerate_physical_devices()
             .unwrap()
@@ -75,7 +141,14 @@ impl WlxGraphics {
                 p.api_version() >= Version::V1_3 || p.supported_extensions().khr_dynamic_rendering
             })
             .filter(|p| {
-                p.supported_extensions().contains(&device_extensions)
+                let runtime_extensions = vk_device_extensions_fn(p);
+                debug!(
+                    "Device exts for {}: {:?}",
+                    p.properties().device_name,
+                    &runtime_extensions
+                );
+                let my_extensions = runtime_extensions.union(&device_extensions);
+                p.supported_extensions().contains(&my_extensions)
             })
             .filter_map(|p| {
                 p.queue_family_properties()
@@ -87,20 +160,18 @@ impl WlxGraphics {
                     })
                     .map(|i| (p, i as u32))
             })
-            .min_by_key(|(p, _)| {
-                match p.properties().device_type {
-                    PhysicalDeviceType::DiscreteGpu => 0,
-                    PhysicalDeviceType::IntegratedGpu => 1,
-                    PhysicalDeviceType::VirtualGpu => 2,
-                    PhysicalDeviceType::Cpu => 3,
-                    PhysicalDeviceType::Other => 4,
-                    _ => 5,
-                }
+            .min_by_key(|(p, _)| match p.properties().device_type {
+                PhysicalDeviceType::DiscreteGpu => 0,
+                PhysicalDeviceType::IntegratedGpu => 1,
+                PhysicalDeviceType::VirtualGpu => 2,
+                PhysicalDeviceType::Cpu => 3,
+                PhysicalDeviceType::Other => 4,
+                _ => 5,
             })
             .expect("no suitable physical device found");
 
         info!(
-            "Nice {} you have there.",
+            "Using vkPhysicalDevice: {}",
             physical_device.properties().device_name,
         );
 
@@ -128,8 +199,44 @@ impl WlxGraphics {
         let queue = queues.next().unwrap();
 
         let memory_allocator = Arc::new(StandardMemoryAllocator::new_default(device.clone()));
-        let command_buffer_allocator = Arc::new(StandardCommandBufferAllocator::new(device.clone(), Default::default()));
-        let descriptor_set_allocator = Arc::new(StandardDescriptorSetAllocator::new(device.clone()));
+        let command_buffer_allocator = Arc::new(StandardCommandBufferAllocator::new(
+            device.clone(),
+            Default::default(),
+        ));
+        let descriptor_set_allocator =
+            Arc::new(StandardDescriptorSetAllocator::new(device.clone()));
+
+        let vertices = [
+            Vert2Uv {
+                in_pos: [0., 0.],
+                in_uv: [0., 0.],
+            },
+            Vert2Uv {
+                in_pos: [0., 1.],
+                in_uv: [0., 1.],
+            },
+            Vert2Uv {
+                in_pos: [1., 0.],
+                in_uv: [1., 0.],
+            },
+            Vert2Uv {
+                in_pos: [1., 1.],
+                in_uv: [1., 1.],
+            },
+        ];
+        let quad_verts = Buffer::from_iter(
+            &memory_allocator,
+            BufferCreateInfo {
+                usage: BufferUsage::VERTEX_BUFFER,
+                ..Default::default()
+            },
+            AllocationCreateInfo {
+                usage: MemoryUsage::Upload,
+                ..Default::default()
+            },
+            vertices.into_iter(),
+        )
+        .unwrap();
 
         let quad_indices = Buffer::from_iter(
             &memory_allocator,
@@ -142,7 +249,8 @@ impl WlxGraphics {
                 ..Default::default()
             },
             INDICES.iter().cloned(),
-        ).unwrap();
+        )
+        .unwrap();
 
         let me = Self {
             instance,
@@ -153,21 +261,30 @@ impl WlxGraphics {
             command_buffer_allocator,
             descriptor_set_allocator,
             quad_indices,
+            quad_verts,
         };
 
         (Arc::new(me), event_loop)
     }
 
-    pub fn create_swapchain(&self, format: Option<Format>) -> (Arc<Swapchain>, Vec<Arc<SwapchainImage>>) {
+    pub fn create_swapchain(
+        &self,
+        format: Option<Format>,
+    ) -> (Arc<Swapchain>, Vec<Arc<SwapchainImage>>) {
         let (min_image_count, composite_alpha, image_format) = if let Some(format) = format {
             (1, CompositeAlpha::Opaque, format)
         } else {
-            let surface_capabilities = self.device
+            let surface_capabilities = self
+                .device
                 .physical_device()
                 .surface_capabilities(&self.surface, Default::default())
                 .unwrap();
-            
-            let composite_alpha = surface_capabilities.supported_composite_alpha.into_iter().next().unwrap();
+
+            let composite_alpha = surface_capabilities
+                .supported_composite_alpha
+                .into_iter()
+                .next()
+                .unwrap();
 
             let image_format = Some(
                 self.device
@@ -176,9 +293,18 @@ impl WlxGraphics {
                     .unwrap()[0]
                     .0,
             );
-            (surface_capabilities.min_image_count, composite_alpha, image_format.unwrap())
+            (
+                surface_capabilities.min_image_count,
+                composite_alpha,
+                image_format.unwrap(),
+            )
         };
-        let window = self.surface.object().unwrap().downcast_ref::<Window>().unwrap();
+        let window = self
+            .surface
+            .object()
+            .unwrap()
+            .downcast_ref::<Window>()
+            .unwrap();
         let swapchain = Swapchain::new(
             self.device.clone(),
             self.surface.clone(),
@@ -196,7 +322,15 @@ impl WlxGraphics {
         swapchain
     }
 
-    pub fn upload_verts(&self, width: f32, height: f32, x: f32, y: f32, w: f32, h: f32) -> Subbuffer<[Vert2Uv]> {
+    pub fn upload_verts(
+        &self,
+        width: f32,
+        height: f32,
+        x: f32,
+        y: f32,
+        w: f32,
+        h: f32,
+    ) -> Subbuffer<[Vert2Uv]> {
         let rw = width;
         let rh = height;
 
@@ -207,16 +341,30 @@ impl WlxGraphics {
         let y1 = h / rh + y0;
 
         let vertices = [
-            Vert2Uv { in_pos: [x0, y0], in_uv: [0.0, 0.0] },
-            Vert2Uv { in_pos: [x0, y1], in_uv: [0.0, 1.0] },
-            Vert2Uv { in_pos: [x1, y0], in_uv: [1.0, 0.0] },
-            Vert2Uv { in_pos: [x1, y1], in_uv: [1.0, 1.0] },
+            Vert2Uv {
+                in_pos: [x0, y0],
+                in_uv: [0.0, 0.0],
+            },
+            Vert2Uv {
+                in_pos: [x0, y1],
+                in_uv: [0.0, 1.0],
+            },
+            Vert2Uv {
+                in_pos: [x1, y0],
+                in_uv: [1.0, 0.0],
+            },
+            Vert2Uv {
+                in_pos: [x1, y1],
+                in_uv: [1.0, 1.0],
+            },
         ];
         self.upload_buffer(BufferUsage::VERTEX_BUFFER, vertices.iter())
     }
 
     pub fn upload_buffer<T>(&self, usage: BufferUsage, contents: Iter<'_, T>) -> Subbuffer<[T]>
-    where T: BufferContents + Clone {
+    where
+        T: BufferContents + Clone,
+    {
         Buffer::from_iter(
             &self.memory_allocator,
             BufferCreateInfo {
@@ -228,8 +376,9 @@ impl WlxGraphics {
                 ..Default::default()
             },
             contents.cloned(),
-        ).unwrap()
-    } 
+        )
+        .unwrap()
+    }
 
     pub fn dmabuf_texture(&self, frame: DmabufFrame) -> Result<Arc<StorageImage>, ImageError> {
         let dimensions = ImageDimensions::Dim2d {
@@ -246,7 +395,8 @@ impl WlxGraphics {
             _ => panic!("Unsupported dmabuf format {:x}", frame.format.fourcc),
         };
 
-        let planes = frame.planes
+        let planes = frame
+            .planes
             .iter()
             .take(frame.num_planes)
             .filter_map(|plane| {
@@ -258,7 +408,8 @@ impl WlxGraphics {
                     offset: plane.offset as _,
                     row_pitch: plane.stride as _,
                 })
-            }).collect();
+            })
+            .collect();
 
         StorageImage::new_from_dma_buf_fd(
             &self.memory_allocator,
@@ -272,28 +423,112 @@ impl WlxGraphics {
             frame.format.modifier,
         )
     }
+
     pub fn render_texture(&self, width: u32, height: u32, format: Format) -> Arc<AttachmentImage> {
         let tex = AttachmentImage::with_usage(
-            &self.memory_allocator, 
-            [width, height], 
+            &self.memory_allocator,
+            [width, height],
             format,
             ImageUsage::SAMPLED | ImageUsage::TRANSFER_SRC | ImageUsage::COLOR_ATTACHMENT,
-        ).unwrap();
+        )
+        .unwrap();
 
         tex
     }
-    pub fn create_pipeline(self: &Arc<Self>, vert: Arc<ShaderModule>, frag: Arc<ShaderModule>, format: Format) -> Arc<WlxPipeline> {
+
+    pub fn create_pipeline(
+        self: &Arc<Self>,
+        vert: Arc<ShaderModule>,
+        frag: Arc<ShaderModule>,
+        format: Format,
+    ) -> Arc<WlxPipeline> {
         Arc::new(WlxPipeline::new(self.clone(), vert, frag, format))
     }
-    pub fn create_command_buffer(self: &Arc<Self>, usage: CommandBufferUsage) -> WlxCommandBuffer<PrimaryAutoCommandBuffer> {
+
+    pub fn create_command_buffer(
+        self: &Arc<Self>,
+        usage: CommandBufferUsage,
+    ) -> WlxCommandBuffer<PrimaryAutoCommandBuffer> {
         let command_buffer = AutoCommandBufferBuilder::primary(
             &self.command_buffer_allocator,
             self.queue.queue_family_index(),
             usage,
-        ).unwrap();
-        WlxCommandBuffer { graphics: self.clone(), command_buffer }
+        )
+        .unwrap();
+        WlxCommandBuffer {
+            graphics: self.clone(),
+            command_buffer,
+        }
     }
 
+    pub fn transition_layout(
+        &self,
+        image: Arc<Image>,
+        old_layout: ImageLayout,
+        new_layout: ImageLayout,
+    ) -> Fence {
+        let barrier = ImageMemoryBarrier {
+            src_stages: PipelineStages::ALL_TRANSFER,
+            src_access: AccessFlags::TRANSFER_WRITE,
+            dst_stages: PipelineStages::ALL_TRANSFER,
+            dst_access: AccessFlags::TRANSFER_READ,
+            old_layout,
+            new_layout,
+            subresource_range: image.subresource_range(),
+            ..ImageMemoryBarrier::image(image)
+        };
+
+        let builder_alloc = self
+            .command_buffer_allocator
+            .allocate(
+                self.queue.queue_family_index(),
+                CommandBufferLevel::Primary,
+                1,
+            )
+            .unwrap()
+            .next()
+            .unwrap();
+
+        let command_buffer = unsafe {
+            let mut builder = UnsafeCommandBufferBuilder::new(
+                &builder_alloc.inner(),
+                CommandBufferBeginInfo {
+                    usage: CommandBufferUsage::OneTimeSubmit,
+                    ..Default::default()
+                },
+            )
+            .unwrap();
+
+            builder.pipeline_barrier(&DependencyInfo {
+                image_memory_barriers: smallvec![barrier],
+                ..Default::default()
+            });
+            builder.build().unwrap()
+        };
+
+        let fence = vulkano::sync::fence::Fence::new(
+            self.device.clone(),
+            vulkano::sync::fence::FenceCreateInfo::default(),
+        )
+        .unwrap();
+
+        let fns = self.device.fns();
+        unsafe {
+            (fns.v1_0.queue_submit)(
+                self.queue.handle(),
+                1,
+                [SubmitInfo::builder()
+                    .command_buffers(&[command_buffer.handle()])
+                    .build()]
+                .as_ptr(),
+                fence.handle(),
+            )
+        }
+        .result()
+        .unwrap();
+
+        fence
+    }
 }
 
 pub struct WlxCommandBuffer<T> {
@@ -306,7 +541,9 @@ impl<T> WlxCommandBuffer<T> {
         &self.command_buffer
     }
 
-    pub fn inner_mut(&mut self) -> &mut AutoCommandBufferBuilder<T, Arc<StandardCommandBufferAllocator>> {
+    pub fn inner_mut(
+        &mut self,
+    ) -> &mut AutoCommandBufferBuilder<T, Arc<StandardCommandBufferAllocator>> {
         &mut self.command_buffer
     }
 
@@ -314,37 +551,55 @@ impl<T> WlxCommandBuffer<T> {
         self.command_buffer
     }
 
-    pub fn begin(mut self, render_target: Arc<dyn ImageViewAbstract>) -> Self 
-    {
+    pub fn begin(
+        mut self,
+        render_target: Arc<dyn ImageViewAbstract>,
+        want_layout: Option<ImageLayout>,
+    ) -> Self {
+        if let Some(want_layout) = want_layout {
+            let mut barrier =
+                ImageMemoryBarrier::image(render_target.image().inner().image.clone());
+            barrier.old_layout = ImageLayout::ColorAttachmentOptimal;
+            barrier.new_layout = want_layout;
+        }
+
         self.command_buffer
-                .begin_rendering(RenderingInfo {
-                    contents: SubpassContents::SecondaryCommandBuffers, 
-                    color_attachments: vec![Some(RenderingAttachmentInfo {
-                        load_op: LoadOp::Clear,
-                        store_op: StoreOp::Store,
-                        clear_value: Some([0.0, 0.0, 0.0, 0.0].into()),
-                        ..RenderingAttachmentInfo::image_view(
-                            render_target.clone(),
-                        )
-                    })],
-                    ..Default::default()
-                }).unwrap();
+            .begin_rendering(RenderingInfo {
+                contents: SubpassContents::SecondaryCommandBuffers,
+                color_attachments: vec![Some(RenderingAttachmentInfo {
+                    load_op: LoadOp::Clear,
+                    store_op: StoreOp::Store,
+                    clear_value: Some([0.0, 0.0, 0.0, 0.0].into()),
+                    ..RenderingAttachmentInfo::image_view(render_target.clone())
+                })],
+                ..Default::default()
+            })
+            .unwrap();
         self
     }
 
-    pub fn run_ref(&mut self, pass: &WlxPass) -> &mut Self
-    {
-        let _ = self.command_buffer.execute_commands(pass.command_buffer.clone()).unwrap();
+    pub fn run_ref(&mut self, pass: &WlxPass) -> &mut Self {
+        let _ = self
+            .command_buffer
+            .execute_commands(pass.command_buffer.clone())
+            .unwrap();
         self
     }
 
-    pub fn run(mut self, pass: &WlxPass) -> Self
-    {
-        let _ = self.command_buffer.execute_commands(pass.command_buffer.clone());
+    pub fn run(mut self, pass: &WlxPass) -> Self {
+        let _ = self
+            .command_buffer
+            .execute_commands(pass.command_buffer.clone());
         self
     }
 
-    pub fn texture2d(&mut self, width: u32, height: u32, format: Format, data: Vec<u8>) -> Arc<ImmutableImage> {
+    pub fn texture2d(
+        &mut self,
+        width: u32,
+        height: u32,
+        format: Format,
+        data: Vec<u8>,
+    ) -> Arc<ImmutableImage> {
         let dimensions = ImageDimensions::Dim2d {
             width,
             height,
@@ -374,7 +629,6 @@ impl<T> WlxCommandBuffer<T> {
         reader.next_frame(&mut image_data).unwrap();
         self.texture2d(width, height, Format::R8G8B8A8_UNORM, image_data)
     }
-
 }
 
 impl WlxCommandBuffer<PrimaryAutoCommandBuffer> {
@@ -414,26 +668,33 @@ pub struct WlxPipeline {
 }
 
 impl WlxPipeline {
-    fn new(graphics: Arc<WlxGraphics>, vert: Arc<ShaderModule>, frag: Arc<ShaderModule>, format: Format) -> Self {
+    fn new(
+        graphics: Arc<WlxGraphics>,
+        vert: Arc<ShaderModule>,
+        frag: Arc<ShaderModule>,
+        format: Format,
+    ) -> Self {
         let vep = vert.entry_point("main").unwrap();
         let fep = frag.entry_point("main").unwrap();
         let pipeline = GraphicsPipeline::start()
-        .render_pass(PipelineRenderingCreateInfo {
-            color_attachment_formats: vec![Some(format)],
-            ..Default::default()
-        })
-        .color_blend_state(ColorBlendState::default().blend(
-                AttachmentBlend::alpha()
-            ))
-        .vertex_input_state(Vert2Uv::per_vertex())
-        .input_assembly_state(InputAssemblyState::new())
-        .vertex_shader(vep, ())
-        .viewport_state(ViewportState::viewport_dynamic_scissor_irrelevant())
-        .fragment_shader(fep, ())
-        .build(graphics.device.clone())
-        .unwrap();
+            .render_pass(PipelineRenderingCreateInfo {
+                color_attachment_formats: vec![Some(format)],
+                ..Default::default()
+            })
+            .color_blend_state(ColorBlendState::default().blend(AttachmentBlend::alpha()))
+            .vertex_input_state(Vert2Uv::per_vertex())
+            .input_assembly_state(InputAssemblyState::new())
+            .vertex_shader(vep, ())
+            .viewport_state(ViewportState::viewport_dynamic_scissor_irrelevant())
+            .fragment_shader(fep, ())
+            .build(graphics.device.clone())
+            .unwrap();
 
-        Self { graphics, pipeline, format}
+        Self {
+            graphics,
+            pipeline,
+            format,
+        }
     }
 
     pub fn inner(&self) -> Arc<GraphicsPipeline> {
@@ -444,7 +705,12 @@ impl WlxPipeline {
         self.graphics.clone()
     }
 
-    pub fn uniform_sampler(&self, set: usize, texture: Arc<dyn ImageViewAbstract>, filter: Filter) -> Arc<PersistentDescriptorSet> {
+    pub fn uniform_sampler(
+        &self,
+        set: usize,
+        texture: Arc<dyn ImageViewAbstract>,
+        filter: Filter,
+    ) -> Arc<PersistentDescriptorSet> {
         let sampler = Sampler::new(
             self.graphics.device.clone(),
             SamplerCreateInfo {
@@ -455,7 +721,7 @@ impl WlxPipeline {
             },
         )
         .unwrap();
- 
+
         let layout = self.pipeline.layout().set_layouts().get(set).unwrap();
 
         PersistentDescriptorSet::new(
@@ -467,7 +733,9 @@ impl WlxPipeline {
     }
 
     pub fn uniform_buffer<T>(&self, set: usize, data: Vec<T>) -> Arc<PersistentDescriptorSet>
-    where T: BufferContents + Copy {
+    where
+        T: BufferContents + Copy,
+    {
         let uniform_buffer = SubbufferAllocator::new(
             self.graphics.memory_allocator.clone(),
             SubbufferAllocatorCreateInfo {
@@ -486,27 +754,44 @@ impl WlxPipeline {
         PersistentDescriptorSet::new(
             &self.graphics.descriptor_set_allocator,
             layout.clone(),
-            [WriteDescriptorSet::buffer(0, uniform_buffer_subbuffer)]
-        ).unwrap()
+            [WriteDescriptorSet::buffer(0, uniform_buffer_subbuffer)],
+        )
+        .unwrap()
     }
 
-    pub fn create_pass(self: &Arc<Self>, dimensions: [f32; 2], vertex_buffer: Subbuffer<[Vert2Uv]>, index_buffer: Subbuffer<[u16]>, descriptor_sets: Vec<Arc<PersistentDescriptorSet>>) -> WlxPass {
-        WlxPass::new(self.clone(), dimensions, vertex_buffer, index_buffer, descriptor_sets)
+    pub fn create_pass(
+        self: &Arc<Self>,
+        dimensions: [f32; 2],
+        vertex_buffer: Subbuffer<[Vert2Uv]>,
+        index_buffer: Subbuffer<[u16]>,
+        descriptor_sets: Vec<Arc<PersistentDescriptorSet>>,
+    ) -> WlxPass {
+        WlxPass::new(
+            self.clone(),
+            dimensions,
+            vertex_buffer,
+            index_buffer,
+            descriptor_sets,
+        )
     }
 }
 
-pub struct WlxPass
-{
+pub struct WlxPass {
     pipeline: Arc<WlxPipeline>,
     vertex_buffer: Subbuffer<[Vert2Uv]>,
     index_buffer: Subbuffer<[u16]>,
     descriptor_sets: Vec<Arc<PersistentDescriptorSet>>,
     pub command_buffer: Arc<SecondaryAutoCommandBuffer>,
-} 
+}
 
-impl WlxPass
-{
-    fn new(pipeline: Arc<WlxPipeline>, dimensions: [f32; 2], vertex_buffer: Subbuffer<[Vert2Uv]>, index_buffer: Subbuffer<[u16]>, descriptor_sets: Vec<Arc<PersistentDescriptorSet>>) -> Self {
+impl WlxPass {
+    fn new(
+        pipeline: Arc<WlxPipeline>,
+        dimensions: [f32; 2],
+        vertex_buffer: Subbuffer<[Vert2Uv]>,
+        index_buffer: Subbuffer<[u16]>,
+        descriptor_sets: Vec<Arc<PersistentDescriptorSet>>,
+    ) -> Self {
         let viewport = Viewport {
             origin: [0.0, 0.0],
             dimensions,
@@ -515,21 +800,23 @@ impl WlxPass
 
         let pipeline_inner = pipeline.inner().clone();
         let mut command_buffer = AutoCommandBufferBuilder::secondary(
-                &pipeline.graphics.command_buffer_allocator,
-                pipeline.graphics.queue.queue_family_index(),
-                CommandBufferUsage::MultipleSubmit,
-                CommandBufferInheritanceInfo { 
-                    render_pass: Some(CommandBufferInheritanceRenderPassType::BeginRendering(
-                        CommandBufferInheritanceRenderingInfo {
-                            color_attachment_formats: vec![Some(pipeline.format)],
-                            ..Default::default()
-                        })),
-                    ..Default::default()
-                }
-            )
-            .unwrap();
+            &pipeline.graphics.command_buffer_allocator,
+            pipeline.graphics.queue.queue_family_index(),
+            CommandBufferUsage::MultipleSubmit,
+            CommandBufferInheritanceInfo {
+                render_pass: Some(CommandBufferInheritanceRenderPassType::BeginRendering(
+                    CommandBufferInheritanceRenderingInfo {
+                        color_attachment_formats: vec![Some(pipeline.format)],
+                        ..Default::default()
+                    },
+                )),
+                ..Default::default()
+            },
+        )
+        .unwrap();
 
-            command_buffer.set_viewport(0, [viewport])
+        command_buffer
+            .set_viewport(0, [viewport])
             .bind_pipeline_graphics(pipeline_inner)
             .bind_descriptor_sets(
                 PipelineBindPoint::Graphics,
@@ -545,9 +832,10 @@ impl WlxPass
                     error!("Failed to draw: {}", source);
                 }
                 Err(err)
-            }).unwrap();
+            })
+            .unwrap();
 
-        Self { 
+        Self {
             pipeline,
             vertex_buffer,
             index_buffer,
@@ -556,4 +844,3 @@ impl WlxPass
         }
     }
 }
-

src/gui/font.rs

@@ -4,7 +4,7 @@ use fontconfig::{FontConfig, OwnedPattern};
 use freetype::{bitmap::PixelMode, face::LoadFlag, Face, Library};
 use idmap::IdMap;
 use log::debug;
-use vulkano::{format::Format, command_buffer::CommandBufferUsage, image::ImmutableImage};
+use vulkano::{command_buffer::CommandBufferUsage, format::Format, image::ImmutableImage};
 
 use crate::graphics::WlxGraphics;
 
@@ -23,9 +23,6 @@ struct FontCollection {
 
 struct Font {
     face: Face,
-    path: String,
-    index: isize,
-    size: isize,
     glyphs: IdMap<usize, Rc<Glyph>>,
 }
 
@@ -50,7 +47,12 @@ impl FontCache {
         }
     }
 
-    pub fn get_text_size(&mut self, text: &str, size: isize, graphics: Arc<WlxGraphics>) -> (f32, f32) {
+    pub fn get_text_size(
+        &mut self,
+        text: &str,
+        size: isize,
+        graphics: Arc<WlxGraphics>,
+    ) -> (f32, f32) {
         let sizef = size as f32;
 
         let height = sizef + ((text.lines().count() as f32) - 1f32) * (sizef * 1.5);
@@ -59,7 +61,10 @@ impl FontCache {
         for line in text.lines() {
             let w: f32 = line
                 .chars()
-                .map(|c| self.get_glyph_for_cp(c as usize, size, graphics.clone()).advance)
+                .map(|c| {
+                    self.get_glyph_for_cp(c as usize, size, graphics.clone())
+                        .advance
+                })
                 .sum();
 
             if w > max_w {
@@ -69,7 +74,12 @@ impl FontCache {
         (max_w, height)
     }
 
-    pub fn get_glyphs(&mut self, text: &str, size: isize, graphics: Arc<WlxGraphics>) -> Vec<Rc<Glyph>> {
+    pub fn get_glyphs(
+        &mut self,
+        text: &str,
+        size: isize,
+        graphics: Arc<WlxGraphics>,
+    ) -> Vec<Rc<Glyph>> {
         let mut glyphs = Vec::new();
         for line in text.lines() {
             for c in line.chars() {
@@ -143,13 +153,7 @@ impl FontCache {
             let mut glyphs = IdMap::new();
             glyphs.insert(0, zero_glyph);
 
-            let font = Font {
-                face,
-                path: path.to_string(),
-                size,
-                index: font_idx as _,
-                glyphs,
-            };
+            let font = Font { face, glyphs };
             coll.fonts.push(font);
 
             idx
@@ -159,7 +163,12 @@ impl FontCache {
         }
     }
 
-    fn get_glyph_for_cp(&mut self, cp: usize, size: isize, graphics: Arc<WlxGraphics>) -> Rc<Glyph> {
+    fn get_glyph_for_cp(
+        &mut self,
+        cp: usize,
+        size: isize,
+        graphics: Arc<WlxGraphics>,
+    ) -> Rc<Glyph> {
         let key = self.get_font_for_cp(cp, size);
 
         let font = &mut self.collections[size].fonts[key];

src/gui/mod.rs

@@ -4,16 +4,16 @@ use glam::{Vec2, Vec3};
 use vulkano::{
     command_buffer::{CommandBufferUsage, PrimaryAutoCommandBuffer},
     format::Format,
-    image::{view::ImageView, AttachmentImage},
+    image::{view::ImageView, AttachmentImage, ImageLayout, ImageViewAbstract},
     sampler::Filter,
 };
 
 use crate::{
-    graphics::{WlxCommandBuffer, WlxGraphics, WlxPass, WlxPipeline},
-    overlays::{
-        interactions::{InteractionHandler, PointerHit},
-        OverlayBackend, OverlayRenderer,
+    backend::{
+        input::{InteractionHandler, PointerHit},
+        overlay::{OverlayBackend, OverlayRenderer},
     },
+    graphics::{WlxCommandBuffer, WlxGraphics, WlxPass, WlxPipeline},
     shaders::{frag_color, frag_glyph, frag_sprite, vert_common},
     state::AppState,
 };
@@ -203,11 +203,8 @@ pub struct Canvas<D, S> {
     interact_stride: usize,
     interact_rows: usize,
 
-    tex_fg: Arc<AttachmentImage>,
     view_fg: Arc<ImageView<AttachmentImage>>,
-    tex_bg: Arc<AttachmentImage>,
     view_bg: Arc<ImageView<AttachmentImage>>,
-    tex_final: Arc<AttachmentImage>,
     view_final: Arc<ImageView<AttachmentImage>>,
 
     pass_fg: WlxPass,
@@ -284,11 +281,8 @@ impl<D, S> Canvas<D, S> {
             interact_map: vec![None; stride * rows],
             interact_stride: stride,
             interact_rows: rows,
-            tex_fg,
             view_fg,
-            tex_bg,
             view_bg,
-            tex_final,
             view_final,
             pass_fg,
             pass_bg,
@@ -323,7 +317,7 @@ impl<D, S> Canvas<D, S> {
             .canvas
             .graphics
             .create_command_buffer(CommandBufferUsage::OneTimeSubmit)
-            .begin(self.view_bg.clone());
+            .begin(self.view_bg.clone(), None);
         for c in self.controls.iter_mut() {
             if let Some(fun) = c.on_render_bg {
                 fun(c, &self.canvas, app, &mut cmd_buffer);
@@ -337,7 +331,7 @@ impl<D, S> Canvas<D, S> {
             .canvas
             .graphics
             .create_command_buffer(CommandBufferUsage::OneTimeSubmit)
-            .begin(self.view_fg.clone());
+            .begin(self.view_fg.clone(), None);
         for c in self.controls.iter_mut() {
             if let Some(fun) = c.on_render_fg {
                 fun(c, &self.canvas, app, &mut cmd_buffer);
@@ -352,32 +346,32 @@ impl<D, S> Canvas<D, S> {
 }
 
 impl<D, S> InteractionHandler for Canvas<D, S> {
-    fn on_left(&mut self, _app: &mut AppState, hand: usize) {
-        self.hover_controls[hand] = None;
+    fn on_left(&mut self, _app: &mut AppState, pointer: usize) {
+        self.hover_controls[pointer] = None;
     }
     fn on_hover(&mut self, _app: &mut AppState, hit: &PointerHit) {
         if let Some(i) = self.interactive_get_idx(hit.uv) {
-            self.hover_controls[hit.hand] = Some(i);
+            self.hover_controls[hit.pointer] = Some(i);
         } else {
-            self.hover_controls[hit.hand] = None;
+            self.hover_controls[hit.pointer] = None;
         }
     }
     fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
         let idx = if pressed {
             self.interactive_get_idx(hit.uv)
         } else {
-            self.pressed_controls[hit.hand]
+            self.pressed_controls[hit.pointer]
         };
 
         if let Some(idx) = idx {
             let c = &mut self.controls[idx];
             if pressed {
                 if let Some(ref mut f) = c.on_press {
-                    self.pressed_controls[hit.hand] = Some(idx);
+                    self.pressed_controls[hit.pointer] = Some(idx);
                     f(c, &mut self.canvas.data, app);
                 }
             } else if let Some(ref mut f) = c.on_release {
-                self.pressed_controls[hit.hand] = None;
+                self.pressed_controls[hit.pointer] = None;
                 f(c, &mut self.canvas.data, app);
             }
         }
@@ -410,7 +404,10 @@ impl<D, S> OverlayRenderer for Canvas<D, S> {
             .canvas
             .graphics
             .create_command_buffer(CommandBufferUsage::OneTimeSubmit)
-            .begin(self.view_final.clone());
+            .begin(
+                self.view_final.clone(),
+                Some(ImageLayout::TransferSrcOptimal),
+            );
 
         if dirty {
             self.render_fg(app);
@@ -437,8 +434,8 @@ impl<D, S> OverlayRenderer for Canvas<D, S> {
 
         let _ = cmd_buffer.end_render_and_execute();
     }
-    fn view(&mut self) -> Arc<dyn vulkano::image::ImageViewAbstract> {
-        self.view_final.clone()
+    fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>> {
+        Some(self.view_final.clone())
     }
 }
 

src/main.rs

@@ -4,43 +4,12 @@ mod graphics;
 mod gui;
 mod input;
 mod overlays;
-mod ovr;
 mod shaders;
 mod state;
 
-use std::collections::VecDeque;
-use std::sync::Arc;
-
-use crate::graphics::{Vert2Uv, WlxGraphics, INDICES};
-use crate::input::initialize_input;
-use crate::overlays::watch::create_watch;
-use crate::{
-    shaders::{frag_sprite, vert_common},
-    state::AppState,
-};
+use crate::backend::openvr::openvr_run;
 use env_logger::Env;
-use log::{info, warn};
-use vulkano::{
-    buffer::BufferUsage,
-    command_buffer::CommandBufferUsage,
-    image::{
-        view::{ImageView, ImageViewCreateInfo},
-        ImageAccess, ImageSubresourceRange, ImageViewType, SwapchainImage,
-    },
-    pipeline::graphics::viewport::Viewport,
-    sampler::Filter,
-    swapchain::{
-        acquire_next_image, AcquireError, SwapchainCreateInfo, SwapchainCreationError,
-        SwapchainPresentInfo,
-    },
-    sync::{self, FlushError, GpuFuture},
-};
-use winit::{
-    event::{Event, WindowEvent},
-    event_loop::ControlFlow,
-    window::Window,
-};
-use wlx_capture::{frame::WlxFrame, wayland::WlxClient, wlr::WlrDmabufCapture, WlxCapture};
+use log::info;
 
 fn main() {
     env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
@@ -50,222 +19,5 @@ fn main() {
         env!("CARGO_PKG_VERSION")
     );
 
-    let (graphics, event_loop) = WlxGraphics::new();
-    let (mut swapchain, images) = graphics.create_swapchain(None);
-
-    let mut app = AppState {
-        fc: crate::gui::font::FontCache::new(),
-        session: crate::state::AppSession::load(),
-        tasks: VecDeque::with_capacity(16),
-        graphics: graphics.clone(),
-        format: swapchain.image_format(),
-        input: initialize_input(),
-    };
-
-    let wl = WlxClient::new().unwrap();
-    let output_id = wl.outputs[0].id;
-    let mut capture = WlrDmabufCapture::new(wl, output_id).unwrap();
-    let rx = capture.init();
-
-    let vertices = [
-        Vert2Uv {
-            in_pos: [0., 0.],
-            in_uv: [0., 0.],
-        },
-        Vert2Uv {
-            in_pos: [0., 1.],
-            in_uv: [0., 1.],
-        },
-        Vert2Uv {
-            in_pos: [1., 0.],
-            in_uv: [1., 0.],
-        },
-        Vert2Uv {
-            in_pos: [1., 1.],
-            in_uv: [1., 1.],
-        },
-    ];
-
-    let vertex_buffer = graphics.upload_buffer(BufferUsage::VERTEX_BUFFER, vertices.iter());
-    let index_buffer = graphics.upload_buffer(BufferUsage::INDEX_BUFFER, INDICES.iter());
-
-    let vs = vert_common::load(graphics.device.clone()).unwrap();
-    let fs = frag_sprite::load(graphics.device.clone()).unwrap();
-
-    let uploads = graphics.create_command_buffer(CommandBufferUsage::OneTimeSubmit);
-
-    let mut watch = create_watch(&app, vec![]);
-    watch.init(&mut app);
-    watch.render(&mut app);
-
-    let pipeline1 = graphics.create_pipeline(vs.clone(), fs.clone(), swapchain.image_format());
-    let set1 = pipeline1.uniform_sampler(0, watch.view(), Filter::Nearest);
-
-    capture.request_new_frame();
-
-    let pipeline = graphics.create_pipeline(vs, fs, swapchain.image_format());
-    let set0;
-    loop {
-        if let Ok(frame) = rx.try_recv() {
-            match frame {
-                WlxFrame::Dmabuf(dmabuf_frame) => match graphics.dmabuf_texture(dmabuf_frame) {
-                    Ok(tex) => {
-                        let format = tex.format();
-                        let view = ImageView::new(
-                            tex,
-                            ImageViewCreateInfo {
-                                format: Some(format),
-                                view_type: ImageViewType::Dim2d,
-                                subresource_range: ImageSubresourceRange::from_parameters(
-                                    format, 1, 1,
-                                ),
-                                ..Default::default()
-                            },
-                        )
-                        .unwrap();
-                        set0 = pipeline.uniform_sampler(0, view, Filter::Nearest);
-                        break;
-                    }
-                    Err(e) => {
-                        warn!("Failed to create texture from dmabuf: {}", e);
-                    }
-                },
-                _ => {
-                    warn!("Received non-dmabuf frame");
-                }
-            }
-        }
-    }
-
-    //let set1 = graphics.uniform_buffer(1, vec![1.0, 1.0, 1.0, 1.0]);
-    let image_extent_f32 = [
-        swapchain.image_extent()[0] as f32,
-        swapchain.image_extent()[1] as f32,
-    ];
-    let image_extent2_f32 = [
-        swapchain.image_extent()[0] as f32 / 2.,
-        swapchain.image_extent()[1] as f32 / 2.,
-    ];
-    let pass = pipeline.create_pass(
-        image_extent_f32,
-        vertex_buffer.clone(),
-        index_buffer.clone(),
-        vec![set0],
-    );
-    let pass2 = pipeline1.create_pass(image_extent2_f32, vertex_buffer, index_buffer, vec![set1]);
-
-    let mut viewport = Viewport {
-        origin: [0.0, 0.0],
-        dimensions: [1024.0, 1024.0],
-        depth_range: 0.0..1.0,
-    };
-
-    let mut attachment_image_views = window_size_dependent_setup(&images, &mut viewport);
-
-    //let set1 = pipeline.uniform_buffer(1, vec![1.0, 0.0, 1.0, 1.0]);
-
-    let mut recreate_swapchain = false;
-    let mut previous_frame_end = //Some(sync::now(graphics.device.clone()).boxed());
-        Some(uploads.end_and_execute().boxed());
-
-    event_loop.run(move |event, _, control_flow| match event {
-        Event::WindowEvent {
-            event: WindowEvent::CloseRequested,
-            ..
-        } => {
-            *control_flow = ControlFlow::Exit;
-        }
-        Event::WindowEvent {
-            event: WindowEvent::Resized(_),
-            ..
-        } => {
-            recreate_swapchain = true;
-        }
-        Event::RedrawEventsCleared => {
-            previous_frame_end.as_mut().unwrap().cleanup_finished();
-
-            if recreate_swapchain {
-                let window = graphics
-                    .surface
-                    .object()
-                    .unwrap()
-                    .downcast_ref::<Window>()
-                    .unwrap();
-                let (new_swapchain, new_images) = match swapchain.recreate(SwapchainCreateInfo {
-                    image_extent: window.inner_size().into(),
-                    ..swapchain.create_info()
-                }) {
-                    Ok(r) => r,
-                    Err(SwapchainCreationError::ImageExtentNotSupported { .. }) => return,
-                    Err(e) => panic!("failed to recreate swapchain: {e}"),
-                };
-
-                swapchain = new_swapchain;
-                attachment_image_views = window_size_dependent_setup(&new_images, &mut viewport);
-                recreate_swapchain = false;
-            }
-
-            let (image_index, suboptimal, acquire_future) =
-                match acquire_next_image(swapchain.clone(), None) {
-                    Ok(r) => r,
-                    Err(AcquireError::OutOfDate) => {
-                        recreate_swapchain = true;
-                        return;
-                    }
-                    Err(e) => panic!("failed to acquire next image: {e}"),
-                };
-
-            if suboptimal {
-                recreate_swapchain = true;
-            }
-
-            let cmd = graphics
-                .create_command_buffer(CommandBufferUsage::OneTimeSubmit)
-                .begin(attachment_image_views[image_index as usize].clone())
-                .run(&pass)
-                .run(&pass2)
-                .end_render();
-
-            let future = previous_frame_end
-                .take()
-                .unwrap()
-                .join(acquire_future)
-                .then_execute(graphics.queue.clone(), cmd)
-                .unwrap()
-                .then_swapchain_present(
-                    graphics.queue.clone(),
-                    SwapchainPresentInfo::swapchain_image_index(swapchain.clone(), image_index),
-                )
-                .then_signal_fence_and_flush();
-
-            match future {
-                Ok(future) => {
-                    previous_frame_end = Some(future.boxed());
-                }
-                Err(FlushError::OutOfDate) => {
-                    recreate_swapchain = true;
-                    previous_frame_end = Some(sync::now(graphics.device.clone()).boxed());
-                }
-                Err(e) => {
-                    println!("failed to flush future: {e}");
-                    previous_frame_end = Some(sync::now(graphics.device.clone()).boxed());
-                }
-            }
-        }
-        _ => (),
-    });
-}
-
-/// This function is called once during initialization, then again whenever the window is resized.
-fn window_size_dependent_setup(
-    images: &[Arc<SwapchainImage>],
-    viewport: &mut Viewport,
-) -> Vec<Arc<ImageView<SwapchainImage>>> {
-    let dimensions = images[0].dimensions().width_height();
-    viewport.dimensions = [dimensions[0] as f32, dimensions[1] as f32];
-
-    images
-        .iter()
-        .map(|image| ImageView::new_default(image.clone()).unwrap())
-        .collect::<Vec<_>>()
+    openvr_run();
 }

src/overlays/interactions.rs

@@ -1,41 +0,0 @@
-use std::{collections::VecDeque, time::Instant};
-
-use glam::{Affine3A, Vec2, Vec3};
-
-use crate::state::AppState;
-
-pub const HAND_LEFT: usize = 0;
-pub const HAND_RIGHT: usize = 1;
-
-pub const POINTER_NORM: u16 = 0;
-pub const POINTER_SHIFT: u16 = 1;
-pub const POINTER_ALT: u16 = 2;
-
-pub trait InteractionHandler {
-    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit);
-    fn on_left(&mut self, app: &mut AppState, hand: usize);
-    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool);
-    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32);
-}
-
-// --- Dummies & plumbing below ---
-
-impl Default for PointerState {
-    fn default() -> Self {
-        Self {
-            click: false,
-            grab: false,
-            show_hide: false,
-            scroll: 0.,
-        }
-    }
-}
-
-pub struct DummyInteractionHandler;
-
-impl InteractionHandler for DummyInteractionHandler {
-    fn on_left(&mut self, _app: &mut AppState, _hand: usize) {}
-    fn on_hover(&mut self, _app: &mut AppState, _hit: &PointerHit) {}
-    fn on_pointer(&mut self, _app: &mut AppState, _hit: &PointerHit, _pressed: bool) {}
-    fn on_scroll(&mut self, _app: &mut AppState, _hit: &PointerHit, _delta: f32) {}
-}

src/overlays/keyboard.rs

@@ -10,23 +10,25 @@ use std::{
 };
 
 use crate::{
+    backend::overlay::{OverlayData, OverlayState},
     gui::{color_parse, CanvasBuilder, Control},
     input::{KeyModifier, VirtualKey, KEYS_TO_MODS},
     state::AppState,
 };
-use glam::{vec2, vec3};
+use glam::{vec2, vec3a};
 use log::error;
 use once_cell::sync::Lazy;
 use regex::Regex;
 use rodio::{Decoder, OutputStream, Source};
 use serde::{Deserialize, Serialize};
 
-use super::OverlayData;
-
 const PIXELS_PER_UNIT: f32 = 80.;
 const BUTTON_PADDING: f32 = 4.;
 
-pub fn create_keyboard(app: &AppState) -> OverlayData {
+pub fn create_keyboard<O>(app: &AppState) -> OverlayData<O>
+where
+    O: Default,
+{
     let size = vec2(
         LAYOUT.row_size * PIXELS_PER_UNIT,
         (LAYOUT.main_layout.len() as f32) * PIXELS_PER_UNIT,
@@ -106,12 +108,15 @@ pub fn create_keyboard(app: &AppState) -> OverlayData {
     let canvas = canvas.build();
 
     OverlayData {
-        name: Arc::from("Kbd"),
-        show_hide: true,
-        width: LAYOUT.row_size * 0.05,
-        size: (size.x as _, size.y as _),
-        grabbable: true,
-        spawn_point: vec3(0., -0.5, -1.),
+        state: OverlayState {
+            name: Arc::from("Kbd"),
+            show_hide: true,
+            width: LAYOUT.row_size * 0.05,
+            size: (size.x as _, size.y as _),
+            grabbable: true,
+            spawn_point: vec3a(0., -0.5, -1.),
+            ..Default::default()
+        },
         backend: Box::new(canvas),
         ..Default::default()
     }

src/overlays/mod.rs

@@ -1,145 +1,3 @@
-use std::sync::{
-    atomic::{AtomicUsize, Ordering},
-    Arc,
-};
-
-use glam::{Affine3A, Quat, Vec3};
-use vulkano::image::ImageViewAbstract;
-
-use crate::state::AppState;
-
-use self::interactions::{DummyInteractionHandler, InteractionHandler, PointerHit};
-
-pub mod interactions;
 pub mod keyboard;
+pub mod screen;
 pub mod watch;
-
-static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(0);
-
-pub enum RelativeTo {
-    None,
-    Head,
-    Hand(usize),
-}
-
-pub trait OverlayBackend: OverlayRenderer + InteractionHandler {}
-
-pub struct OverlayData {
-    pub id: usize,
-    pub name: Arc<str>,
-    pub width: f32,
-    pub size: (i32, i32),
-    pub want_visible: bool,
-    pub show_hide: bool,
-    pub grabbable: bool,
-    pub transform: Affine3A,
-    pub spawn_point: Vec3,
-    pub spawn_rotation: Quat,
-    pub relative_to: RelativeTo,
-    pub interaction_transform: Affine3A,
-    pub backend: Box<dyn OverlayBackend>,
-    pub primary_pointer: Option<usize>,
-}
-impl Default for OverlayData {
-    fn default() -> OverlayData {
-        OverlayData {
-            id: AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed),
-            name: Arc::from(""),
-            width: 1.,
-            size: (0, 0),
-            want_visible: false,
-            show_hide: false,
-            grabbable: false,
-            relative_to: RelativeTo::None,
-            spawn_point: Vec3::NEG_Z,
-            spawn_rotation: Quat::IDENTITY,
-            transform: Affine3A::IDENTITY,
-            interaction_transform: Affine3A::IDENTITY,
-            backend: Box::new(SplitOverlayBackend::default()),
-            primary_pointer: None,
-        }
-    }
-}
-
-impl OverlayData {
-    pub fn reset(&mut self, app: &mut AppState) {
-        todo!()
-    }
-    pub fn init(&mut self, app: &mut AppState) {
-        self.backend.init(app);
-    }
-    pub fn render(&mut self, app: &mut AppState) {
-        self.backend.render(app);
-    }
-    pub fn view(&mut self) -> Arc<dyn ImageViewAbstract> {
-        self.backend.view()
-    }
-}
-
-pub trait OverlayRenderer {
-    fn init(&mut self, app: &mut AppState);
-    fn pause(&mut self, app: &mut AppState);
-    fn resume(&mut self, app: &mut AppState);
-    fn render(&mut self, app: &mut AppState);
-    fn view(&mut self) -> Arc<dyn ImageViewAbstract>;
-}
-
-pub struct FallbackRenderer;
-
-impl OverlayRenderer for FallbackRenderer {
-    fn init(&mut self, _app: &mut AppState) {}
-    fn pause(&mut self, _app: &mut AppState) {}
-    fn resume(&mut self, _app: &mut AppState) {}
-    fn render(&mut self, _app: &mut AppState) {}
-    fn view(&mut self) -> Arc<dyn ImageViewAbstract> {
-        unimplemented!()
-    }
-}
-// Boilerplate and dummies
-
-pub struct SplitOverlayBackend {
-    pub renderer: Box<dyn OverlayRenderer>,
-    pub interaction: Box<dyn InteractionHandler>,
-}
-
-impl Default for SplitOverlayBackend {
-    fn default() -> SplitOverlayBackend {
-        SplitOverlayBackend {
-            renderer: Box::new(FallbackRenderer),
-            interaction: Box::new(DummyInteractionHandler),
-        }
-    }
-}
-
-impl OverlayBackend for SplitOverlayBackend {}
-impl OverlayRenderer for SplitOverlayBackend {
-    fn init(&mut self, app: &mut AppState) {
-        self.renderer.init(app);
-    }
-    fn pause(&mut self, app: &mut AppState) {
-        self.renderer.pause(app);
-    }
-    fn resume(&mut self, app: &mut AppState) {
-        self.renderer.resume(app);
-    }
-    fn render(&mut self, app: &mut AppState) {
-        self.renderer.render(app);
-    }
-    fn view(&mut self) -> Arc<dyn ImageViewAbstract> {
-        self.renderer.view()
-    }
-}
-impl InteractionHandler for SplitOverlayBackend {
-    fn on_left(&mut self, app: &mut AppState, hand: usize) {
-        self.interaction.on_left(app, hand);
-    }
-    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) {
-        self.interaction.on_hover(app, hit);
-    }
-    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32) {
-        self.interaction.on_scroll(app, hit, delta);
-    }
-    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
-        self.interaction.on_pointer(app, hit, pressed);
-    }
-}

src/overlays/screen.rs

@@ -1,20 +1,59 @@
+use log::{info, warn};
+use std::{
+    f32::consts::PI,
+    path::Path,
+    sync::{mpsc::Receiver, Arc},
+    time::{Duration, Instant},
+};
+use vulkano::{
+    command_buffer::CommandBufferUsage,
+    format::Format,
+    image::{view::ImageView, ImageAccess, ImageViewAbstract, ImmutableImage},
+    Handle, VulkanObject,
+};
+use wlx_capture::{
+    frame::WlxFrame,
+    pipewire::{pipewire_select_screen, PipewireCapture},
+    wayland::{Transform, WlxClient, WlxOutput},
+    wlr::WlrDmabufCapture,
+    WlxCapture,
+};
 
-pub struct ScreenInteractionData {
+use glam::{vec2, Affine2, Quat, Vec2, Vec3};
+
+use crate::{
+    backend::{
+        input::{InteractionHandler, PointerHit, PointerMode},
+        overlay::{OverlayData, OverlayRenderer, OverlayState, SplitOverlayBackend},
+    },
+    input::{MOUSE_LEFT, MOUSE_MIDDLE, MOUSE_RIGHT},
+    state::{AppSession, AppState},
+};
+
+pub struct ScreenInteractionHandler {
     next_scroll: Instant,
     next_move: Instant,
     mouse_transform: Affine2,
 }
-impl ScreenInteractionData {
+impl ScreenInteractionHandler {
     fn new(pos: Vec2, size: Vec2, transform: Transform) -> ScreenInteractionHandler {
         let transform = match transform {
-            Transform::_90 | Transform::Flipped90 =>
-                Affine2::from_cols(vec2(0., size.y), vec2(-size.x, 0.), vec2(pos.x + size.x, pos.y)),
-            Transform::_180 | Transform::Flipped180 => 
-                Affine2::from_cols(vec2(-size.x, 0.), vec2(0., -size.y), vec2(pos.x + size.x, pos.y + size.y)),
-            Transform::_270 | Transform::Flipped270 => 
-                Affine2::from_cols(vec2(0., -size.y), vec2(size.x, 0.), vec2(pos.x, pos.y + size.y)),
-            _ => 
-                Affine2::from_cols(vec2(size.x, 0.), vec2(0., size.y), pos),
+            Transform::_90 | Transform::Flipped90 => Affine2::from_cols(
+                vec2(0., size.y),
+                vec2(-size.x, 0.),
+                vec2(pos.x + size.x, pos.y),
+            ),
+            Transform::_180 | Transform::Flipped180 => Affine2::from_cols(
+                vec2(-size.x, 0.),
+                vec2(0., -size.y),
+                vec2(pos.x + size.x, pos.y + size.y),
+            ),
+            Transform::_270 | Transform::Flipped270 => Affine2::from_cols(
+                vec2(0., -size.y),
+                vec2(size.x, 0.),
+                vec2(pos.x, pos.y + size.y),
+            ),
+            _ => Affine2::from_cols(vec2(size.x, 0.), vec2(0., size.y), pos),
         };
 
         ScreenInteractionHandler {
@@ -25,7 +64,222 @@ impl ScreenInteractionData {
     }
 }
 
-struct ScreenInteractionHandler {
+impl InteractionHandler for ScreenInteractionHandler {
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) {
+        if self.next_move < Instant::now() {
+            let pos = self.mouse_transform.transform_point2(hit.uv);
+            app.input.mouse_move(pos);
+        }
+    }
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
+        let pos = self.mouse_transform.transform_point2(hit.uv);
+        app.input.mouse_move(pos);
 
+        let btn = match hit.mode {
+            PointerMode::Right => MOUSE_RIGHT,
+            PointerMode::Middle => MOUSE_MIDDLE,
+            _ => MOUSE_LEFT,
+        };
+
+        if pressed {
+            self.next_move = Instant::now() + Duration::from_millis(300);
+        }
+
+        app.input.send_button(btn, pressed);
+    }
+    fn on_scroll(&mut self, app: &mut AppState, _hit: &PointerHit, delta: f32) {
+        let millis = (1. - delta.abs()) * delta;
+        if let Some(next_scroll) = Instant::now().checked_add(Duration::from_millis(millis as _)) {
+            self.next_scroll = next_scroll;
+        }
+        app.input.wheel(if delta < 0. { -1 } else { 1 })
+    }
+    fn on_left(&mut self, _app: &mut AppState, _hand: usize) {}
 }
 
+pub struct ScreenRenderer {
+    capture: Box<dyn WlxCapture>,
+    resolution: (i32, i32),
+    receiver: Option<Receiver<WlxFrame>>,
+    view: Option<Arc<dyn ImageViewAbstract>>,
+}
+
+impl ScreenRenderer {
+    pub fn new_wlr(output: &WlxOutput) -> Option<ScreenRenderer> {
+        let Some(client) = WlxClient::new() else {
+            return None;
+        };
+        let Some(capture) = WlrDmabufCapture::new(client, output.id) else {
+            return None;
+        };
+        Some(ScreenRenderer {
+            capture: Box::new(capture),
+            resolution: output.size,
+            receiver: None,
+            view: None,
+        })
+    }
+
+    pub fn new_pw(
+        output: &WlxOutput,
+        token: Option<&str>,
+        _fallback: bool,
+    ) -> Option<ScreenRenderer> {
+        let name = output.name.clone();
+        let node_id = futures::executor::block_on(pipewire_select_screen(token)).ok()?;
+
+        let capture = PipewireCapture::new(name, node_id, 60);
+
+        Some(ScreenRenderer {
+            capture: Box::new(capture),
+            resolution: output.size,
+            receiver: None,
+            view: None,
+        })
+    }
+
+    pub fn new_xshm() -> ScreenRenderer {
+        todo!()
+    }
+}
+
+impl OverlayRenderer for ScreenRenderer {
+    fn init(&mut self, app: &mut AppState) {
+        self.receiver = Some(self.capture.init());
+        let mut cmd = app
+            .graphics
+            .create_command_buffer(CommandBufferUsage::OneTimeSubmit);
+        let default_image = cmd.texture2d(1, 1, Format::R8G8B8A8_UNORM, vec![255, 0, 255, 255]);
+        let _ = cmd.end_and_execute();
+    }
+    fn render(&mut self, app: &mut AppState) {
+        let Some(receiver) = self.receiver.as_mut() else {
+            log::error!("No receiver");
+            return;
+        };
+
+        for frame in receiver.try_iter() {
+            match frame {
+                WlxFrame::Dmabuf(frame) => {
+                    if let Ok(new) = app.graphics.dmabuf_texture(frame) {
+                        if let Some(current) = self.view.as_ref() {
+                            if current.image().inner().image.handle().as_raw()
+                                == new.inner().image.handle().as_raw()
+                            {
+                                return;
+                            }
+                        }
+                        self.view = Some(ImageView::new_default(new).unwrap());
+                    }
+                }
+                WlxFrame::MemFd(frame) => {
+                    todo!()
+                }
+                WlxFrame::MemPtr(frame) => {
+                    todo!()
+                }
+                _ => {}
+            };
+        }
+        self.capture.request_new_frame();
+    }
+    fn pause(&mut self, _app: &mut AppState) {
+        self.capture.pause();
+    }
+    fn resume(&mut self, _app: &mut AppState) {
+        self.capture.resume();
+    }
+    fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>> {
+        self.view.as_ref().and_then(|v| Some(v.clone()))
+    }
+}
+
+fn try_create_screen<O>(wl: &WlxClient, idx: usize, session: &AppSession) -> Option<OverlayData<O>>
+where
+    O: Default,
+{
+    let output = &wl.outputs[idx];
+    info!(
+        "{}: Res {}x{} Size {:?} Pos {:?}",
+        output.name, output.size.0, output.size.1, output.logical_size, output.logical_pos,
+    );
+
+    let size = (output.size.0, output.size.1);
+    let mut capture: Option<ScreenRenderer> = None;
+
+    if session.capture_method == "auto" && wl.maybe_wlr_dmabuf_mgr.is_some() {
+        info!("{}: Using Wlr DMA-Buf", &output.name);
+        capture = ScreenRenderer::new_wlr(output);
+    }
+
+    if capture.is_none() {
+        info!("{}: Using Pipewire capture", &output.name);
+        let file_name = format!("{}.token", &output.name);
+        let full_path = Path::new(&session.config_path).join(file_name);
+        let token = std::fs::read_to_string(full_path).ok();
+
+        capture = ScreenRenderer::new_pw(
+            output,
+            token.as_deref(),
+            session.capture_method == "pw_fallback",
+        );
+    }
+    if let Some(capture) = capture {
+        let backend = Box::new(SplitOverlayBackend {
+            renderer: Box::new(capture),
+            interaction: Box::new(ScreenInteractionHandler::new(
+                vec2(output.logical_pos.0 as f32, output.logical_pos.1 as f32),
+                vec2(output.logical_size.0 as f32, output.logical_size.1 as f32),
+                output.transform,
+            )),
+        });
+
+        let axis = Vec3::new(0., 0., 1.);
+
+        let angle = match output.transform {
+            Transform::_90 | Transform::Flipped90 => PI / 2.,
+            Transform::_180 | Transform::Flipped180 => PI,
+            Transform::_270 | Transform::Flipped270 => -PI / 2.,
+            _ => 0.,
+        };
+
+        Some(OverlayData {
+            state: OverlayState {
+                name: output.name.clone(),
+                size,
+                want_visible: idx == 0,
+                show_hide: true,
+                grabbable: true,
+                spawn_rotation: Quat::from_axis_angle(axis, angle),
+                ..Default::default()
+            },
+            backend,
+            ..Default::default()
+        })
+    } else {
+        warn!("{}: Will not be used", &output.name);
+        None
+    }
+}
+
+pub fn get_screens_wayland<O>(session: &AppSession) -> Vec<OverlayData<O>>
+where
+    O: Default,
+{
+    let mut overlays = vec![];
+    let wl = WlxClient::new().unwrap();
+
+    for idx in 0..wl.outputs.len() {
+        if let Some(overlay) = try_create_screen(&wl, idx, &session) {
+            overlays.push(overlay);
+        }
+    }
+    overlays
+}
+
+pub fn get_screens_x11<O>() -> Vec<OverlayData<O>>
+where
+    O: Default,
+{
+    todo!()
+}

src/overlays/watch.rs

@@ -1,19 +1,20 @@
 use std::{sync::Arc, time::Instant};
 
 use chrono::Local;
-use glam::{Quat, Vec3};
 
 use crate::{
+    backend::{
+        common::{OverlaySelector, TaskType},
+        overlay::{OverlayData, OverlayState, RelativeTo},
+    },
     gui::{color_parse, CanvasBuilder},
     state::AppState,
 };
 
-use super::{OverlayData, RelativeTo};
-
-pub const WATCH_DEFAULT_POS: Vec3 = Vec3::new(0., 0., 0.15);
-pub const WATCH_DEFAULT_ROT: Quat = Quat::from_xyzw(0.7071066, 0., 0.7071066, 0.0007963);
-
-pub fn create_watch(state: &AppState, screens: Vec<(usize, Arc<str>)>) -> OverlayData {
+pub fn create_watch<O>(state: &AppState, screens: &[OverlayData<O>]) -> OverlayData<O>
+where
+    O: Default,
+{
     let mut canvas = CanvasBuilder::new(400, 200, state.graphics.clone(), state.format, ());
     let empty_str: Arc<str> = Arc::from("");
 
@@ -93,22 +94,19 @@ pub fn create_watch(state: &AppState, screens: Vec<(usize, Arc<str>)>) -> Overla
                 .as_millis()
                 < 2000
             {
-                app.tasks.push_back(Box::new(|_app, o| {
-                    for overlay in o {
-                        if &*overlay.name == "Kbd" {
-                            overlay.want_visible = !overlay.want_visible;
-                            return;
-                        }
-                    }
-                }));
+                app.tasks.enqueue(TaskType::Overlay(
+                    OverlaySelector::Name("Kbd".into()),
+                    Box::new(|_app, o| {
+                        o.want_visible = !o.want_visible;
+                    }),
+                ));
             } else {
-                app.tasks.push_back(Box::new(|app, o| {
-                    for overlay in o {
-                        if &*overlay.name == "Kbd" {
-                            overlay.reset(app);
-                        }
-                    }
-                }));
+                app.tasks.enqueue(TaskType::Overlay(
+                    OverlaySelector::Name("Kbd".into()),
+                    Box::new(|app, o| {
+                        o.reset(app);
+                    }),
+                ));
             }
         }
     });
@@ -116,11 +114,17 @@ pub fn create_watch(state: &AppState, screens: Vec<(usize, Arc<str>)>) -> Overla
 
     canvas.bg_color = color_parse("#405060");
 
-    for (scr_idx, scr_name) in screens.into_iter() {
-        let button = canvas.button(button_x + 2., 162., button_width - 4., 36., scr_name);
+    for screen in screens.into_iter() {
+        let button = canvas.button(
+            button_x + 2.,
+            162.,
+            button_width - 4.,
+            36.,
+            screen.state.name.clone(),
+        );
         button.state = Some(WatchButtonState {
             pressed_at: Instant::now(),
-            scr_idx,
+            scr_idx: screen.state.id,
         });
 
         button.on_press = Some(|control, _data, _app| {
@@ -136,13 +140,19 @@ pub fn create_watch(state: &AppState, screens: Vec<(usize, Arc<str>)>) -> Overla
                     .as_millis()
                     < 2000
                 {
-                    app.tasks.push_back(Box::new(move |_app, o| {
-                        o[scr_idx].want_visible = !o[scr_idx].want_visible;
-                    }));
+                    app.tasks.enqueue(TaskType::Overlay(
+                        OverlaySelector::Id(scr_idx),
+                        Box::new(|_app, o| {
+                            o.want_visible = !o.want_visible;
+                        }),
+                    ));
                 } else {
-                    app.tasks.push_back(Box::new(move |app, o| {
-                        o[scr_idx].reset(app);
-                    }));
+                    app.tasks.enqueue(TaskType::Overlay(
+                        OverlaySelector::Id(scr_idx),
+                        Box::new(|app, o| {
+                            o.reset(app);
+                        }),
+                    ));
                 }
             }
         });
@@ -152,14 +162,17 @@ pub fn create_watch(state: &AppState, screens: Vec<(usize, Arc<str>)>) -> Overla
     let relative_to = RelativeTo::Hand(state.session.watch_hand);
 
     OverlayData {
-        name: "Watch".into(),
-        size: (400, 200),
-        width: 0.065,
+        state: OverlayState {
+            name: "Watch".into(),
+            size: (400, 200),
+            width: 0.065,
+            want_visible: true,
+            spawn_point: state.session.watch_pos.into(),
+            spawn_rotation: state.session.watch_rot,
+            relative_to,
+            ..Default::default()
+        },
         backend: Box::new(canvas.build()),
-        want_visible: true,
-        relative_to,
-        spawn_point: state.session.watch_pos,
-        spawn_rotation: state.session.watch_rot,
         ..Default::default()
     }
 }

src/ovr.rs

@@ -1,64 +0,0 @@
-use std::{path::Path, sync::Arc};
-
-use vulkano::{
-    image::{
-        sys::{Image, RawImage},
-        ImageViewAbstract,
-    },
-    Handle, VulkanObject,
-};
-
-use crate::graphics::WlxGraphics;
-
-pub struct OpenVrState {
-    pub context: ovr_overlay::Context,
-}
-
-
-pub struct OvrTextureData {
-    image_handle: u64,
-    device: u64,
-    physical: u64,
-    instance: u64,
-    queue: u64,
-    queue_family_index: u32,
-    width: u32,
-    height: u32,
-    format: u32,
-    sample_count: u32,
-}
-
-impl OvrTextureData {
-    pub fn new(graphics: Arc<WlxGraphics>, view: Arc<dyn ImageViewAbstract>) -> OvrTextureData {
-        let image = view.image();
-
-        let device = graphics.device.handle().as_raw();
-        let physical = graphics.device.physical_device().handle().as_raw();
-        let instance = graphics.instance.handle().as_raw();
-        let queue = graphics.queue.handle().as_raw();
-        let queue_family_index = graphics.queue.queue_family_index();
-
-        let (width, height) = {
-            let dim = image.dimensions();
-            (dim.width() as u32, dim.height() as u32)
-        };
-
-        let sample_count = image.samples() as u32;
-        let format = image.format() as u32;
-
-        let image_handle = image.inner().image.handle().as_raw();
-
-        OvrTextureData {
-            image_handle,
-            device,
-            physical,
-            instance,
-            queue,
-            queue_family_index,
-            width,
-            height,
-            format,
-            sample_count,
-        }
-    }
-}

src/res/actions_binding_knuckles.json

@@ -1,6 +1,6 @@
 {
    "action_manifest_version" : 0,
-   "app_key" : "galister.wlxoverlay",
+   "app_key" : "galister.wlxoverlay-s",
    "bindings" : {
       "/actions/default" : {
          "haptics" : [

src/res/actions_binding_oculus.json

@@ -1,6 +1,6 @@
 {
    "action_manifest_version" : 0,
-   "app_key" : "galister.wlxoverlay",
+   "app_key" : "galister.wlxoverlay-s",
    "bindings" : {
       "/actions/default" : {
          "haptics" : [

src/res/actions_binding_vive.json

@@ -1,6 +1,6 @@
 {
    "action_manifest_version" : 0,
-   "app_key" : "galister.wlxoverlay",
+   "app_key" : "galister.wlxoverlay-s",
    "bindings" : {
       "/actions/default": {
          "haptics" : [

src/state.rs

@@ -1,27 +1,50 @@
-use std::{collections::VecDeque, env::VarError, path::Path, sync::Arc};
+use std::{env::VarError, path::Path, sync::Arc};
 
 use glam::{Quat, Vec3};
 use log::warn;
+use vulkano::{
+    device::{physical::PhysicalDevice, DeviceExtensions},
+    format::Format,
+    instance::InstanceExtensions,
+};
 
 use crate::{
-    graphics::WlxGraphics, gui::font::FontCache, input::InputProvider, overlays::OverlayData,
+    backend::common::TaskContainer, graphics::WlxGraphics, gui::font::FontCache,
+    input::InputProvider,
 };
 
 pub const WATCH_DEFAULT_POS: Vec3 = Vec3::new(0., 0., 0.15);
 pub const WATCH_DEFAULT_ROT: Quat = Quat::from_xyzw(0.7071066, 0., 0.7071066, 0.0007963);
 
-pub type Task = Box<dyn FnOnce(&mut AppState, &mut [OverlayData]) + Send>;
-
 pub struct AppState {
     pub fc: FontCache,
     //pub input: InputState,
     pub session: AppSession,
-    pub tasks: VecDeque<Task>,
+    pub tasks: TaskContainer,
     pub graphics: Arc<WlxGraphics>,
     pub format: vulkano::format::Format,
     pub input: Box<dyn InputProvider>,
 }
 
+impl AppState {
+    pub fn new(
+        vk_instance_extensions: InstanceExtensions,
+        vk_device_extensions_fn: impl FnMut(&PhysicalDevice) -> DeviceExtensions,
+    ) -> Self {
+        let (graphics, _event_loop) =
+            WlxGraphics::new(vk_instance_extensions, vk_device_extensions_fn);
+
+        AppState {
+            fc: FontCache::new(),
+            session: AppSession::load(),
+            tasks: TaskContainer::new(),
+            graphics: graphics.clone(),
+            format: Format::R8G8B8A8_UNORM,
+            input: crate::input::initialize_input(),
+        }
+    }
+}
+
 pub struct AppSession {
     pub config_path: String,
 
@@ -50,7 +73,7 @@ pub struct AppSession {
 }
 
 impl AppSession {
-    pub fn load() -> AppSession {
+    pub fn load() -> Self {
         let config_path = std::env::var("XDG_CONFIG_HOME")
             .or_else(|_| std::env::var("HOME").map(|home| format!("{}/.config", home)))
             .or_else(|_| {