nothing works

src/backend/common.rs

@@ -0,0 +1,152 @@
+use std::{collections::VecDeque, time::Instant};
+
+use glam::{Affine3A, Vec2, Vec3, Vec3A};
+use ovr_overlay::TrackedDeviceIndex;
+
+pub struct InputState<TState, THand> {
+    pub hmd: Affine3A,
+    pub pointers: [Pointer<THand>; 2],
+    pub devices: Vec<TrackedDevice>,
+    pub(super) data: TState,
+}
+
+impl<TState, THand> InputState<TState, THand> {
+    pub fn pre_update(&mut self) {
+        self.pointers[0].before = self.pointers[0].now;
+        self.pointers[1].before = self.pointers[1].now;
+    }
+
+    pub fn post_update(&mut self) {
+        for hand in &mut self.pointers {
+            if hand.now.click_modifier_right {
+                hand.interaction.mode = PointerMode::Right;
+                continue;
+            }
+
+            if hand.now.click_modifier_middle {
+                hand.interaction.mode = PointerMode::Middle;
+                continue;
+            }
+
+            let hmd_up = self.hmd.transform_vector3a(Vec3A::Y);
+            let dot =
+                hmd_up.dot(hand.pose.transform_vector3a(Vec3A::X)) * (1.0 - 2.0 * hand.hand as f32);
+
+            hand.interaction.mode = if dot < -0.85 {
+                PointerMode::Right
+            } else if dot > 0.7 {
+                PointerMode::Middle
+            } else {
+                PointerMode::Left
+            };
+
+            let middle_click_orientation = false;
+            let right_click_orientation = false;
+            match hand.interaction.mode {
+                PointerMode::Middle => {
+                    if !middle_click_orientation {
+                        hand.interaction.mode = PointerMode::Left;
+                    }
+                }
+                PointerMode::Right => {
+                    if !right_click_orientation {
+                        hand.interaction.mode = PointerMode::Left;
+                    }
+                }
+                _ => {}
+            };
+        }
+    }
+}
+
+pub struct Pointer<THand> {
+    pub hand: usize,
+    pub pose: Affine3A,
+    pub now: PointerState,
+    pub before: PointerState,
+    pub(super) interaction: InteractionState,
+    pub(super) data: THand,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct PointerState {
+    pub scroll: f32,
+    pub click: bool,
+    pub grab: bool,
+    pub alt_click: bool,
+    pub show_hide: bool,
+    pub space_drag: bool,
+    pub click_modifier_right: bool,
+    pub click_modifier_middle: bool,
+}
+
+pub struct InteractionState {
+    pub mode: PointerMode,
+    pub grabbed: Option<GrabData>,
+    pub clicked_id: Option<usize>,
+    pub hovered_id: Option<usize>,
+    pub release_actions: VecDeque<Box<dyn Fn()>>,
+    pub next_push: Instant,
+}
+
+impl Default for InteractionState {
+    fn default() -> Self {
+        Self {
+            mode: PointerMode::Left,
+            grabbed: None,
+            clicked_id: None,
+            hovered_id: None,
+            release_actions: VecDeque::new(),
+            next_push: Instant::now(),
+        }
+    }
+}
+
+pub struct PointerHit {
+    pub hand: usize,
+    pub mode: PointerMode,
+    pub primary: bool,
+    pub uv: Vec2,
+    pub dist: f32,
+}
+
+struct RayHit {
+    idx: usize,
+    ray_pos: Vec3,
+    hit_pos: Vec3,
+    uv: Vec2,
+    dist: f32,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct GrabData {
+    pub offset: Vec3,
+    pub grabbed_id: usize,
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, Default)]
+pub enum PointerMode {
+    #[default]
+    Left,
+    Right,
+    Middle,
+}
+
+pub struct TrackedDevice {
+    pub index: TrackedDeviceIndex,
+    pub valid: bool,
+    pub soc: Option<f32>,
+    pub charging: bool,
+    pub role: TrackedDeviceRole,
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TrackedDeviceRole {
+    None,
+    Hmd,
+    LeftHand,
+    RightHand,
+    Tracker,
+}

src/backend/mod.rs

@@ -0,0 +1,3 @@
+pub mod common;
+pub mod openvr;
+pub mod openxr;

src/backend/openvr/input.rs

@@ -0,0 +1,288 @@
+use std::array;
+
+use glam::Affine3A;
+use ovr_overlay::{
+    input::{ActionHandle, ActionSetHandle, ActiveActionSet, InputManager, InputValueHandle},
+    sys::{
+        k_unMaxTrackedDeviceCount, ETrackedControllerRole, ETrackedDeviceClass,
+        ETrackedDeviceProperty, ETrackingUniverseOrigin, HmdMatrix34_t,
+    },
+    system::SystemManager,
+    TrackedDeviceIndex,
+};
+
+use crate::backend::common::{
+    InputState, InteractionState, Pointer, PointerState, TrackedDevice, TrackedDeviceRole,
+};
+
+macro_rules! result_str {
+    ( $e:expr ) => {
+        match $e {
+            Ok(x) => Ok(x),
+            Err(y) => Err(y.description()),
+        }
+    };
+}
+
+const SET_DEFAULT: &str = "/actions/default";
+const INPUT_SOURCES: [&str; 2] = ["/user/hand/left", "/user/hand/right"];
+const PATH_POSES: [&str; 2] = [
+    "/actions/default/in/LeftHand",
+    "/actions/default/in/RightHand",
+];
+const PATH_HAPTICS: [&str; 2] = [
+    "/actions/default/out/HapticsLeft",
+    "/actions/default/out/HapticsRight",
+];
+
+const PATH_CLICK: &str = "/actions/default/in/Click";
+const PATH_GRAB: &str = "/actions/default/in/Grab";
+const PATH_SCROLL: &str = "/actions/default/in/Scroll";
+const PATH_ALT_CLICK: &str = "/actions/default/in/AltClick";
+const PATH_SHOW_HIDE: &str = "/actions/default/in/ShowHide";
+const PATH_SPACE_DRAG: &str = "/actions/default/in/SpaceDrag";
+const PATH_CLICK_MODIFIER_RIGHT: &str = "/actions/default/in/ClickModifierRight";
+const PATH_CLICK_MODIFIER_MIDDLE: &str = "/actions/default/in/ClickModifierMiddle";
+
+const INPUT_ANY: InputValueHandle = InputValueHandle(ovr_overlay::sys::k_ulInvalidInputValueHandle);
+
+pub(super) struct OpenVrInputState {
+    set_hnd: ActionSetHandle,
+    click_hnd: ActionHandle,
+    grab_hnd: ActionHandle,
+    scroll_hnd: ActionHandle,
+    alt_click_hnd: ActionHandle,
+    show_hide_hnd: ActionHandle,
+    space_drag_hnd: ActionHandle,
+    click_modifier_right_hnd: ActionHandle,
+    click_modifier_middle_hnd: ActionHandle,
+}
+
+pub(super) struct OpenVrHandState {
+    has_pose: bool,
+    input_hnd: InputValueHandle,
+    pose_hnd: ActionHandle,
+    haptics_hnd: ActionHandle,
+}
+
+impl InputState<OpenVrInputState, OpenVrHandState> {
+    pub fn new(input: &mut InputManager) -> Result<Self, &'static str> {
+        let set_hnd = result_str!(input.get_action_set_handle(SET_DEFAULT))?;
+
+        let click_hnd = result_str!(input.get_action_handle(PATH_CLICK))?;
+        let grab_hnd = result_str!(input.get_action_handle(PATH_GRAB))?;
+        let scroll_hnd = result_str!(input.get_action_handle(PATH_SCROLL))?;
+        let alt_click_hnd = result_str!(input.get_action_handle(PATH_ALT_CLICK))?;
+        let show_hide_hnd = result_str!(input.get_action_handle(PATH_SHOW_HIDE))?;
+        let space_drag_hnd = result_str!(input.get_action_handle(PATH_SPACE_DRAG))?;
+        let click_modifier_right_hnd =
+            result_str!(input.get_action_handle(PATH_CLICK_MODIFIER_RIGHT))?;
+        let click_modifier_middle_hnd =
+            result_str!(input.get_action_handle(PATH_CLICK_MODIFIER_MIDDLE))?;
+
+        let input_hnd: Vec<InputValueHandle> = INPUT_SOURCES
+            .iter()
+            .map(|path| Ok(result_str!(input.get_input_source_handle(path))?))
+            .collect::<Result<_, &'static str>>()?;
+
+        let pose_hnd: Vec<ActionHandle> = PATH_POSES
+            .iter()
+            .map(|path| Ok(result_str!(input.get_action_handle(path))?))
+            .collect::<Result<_, &'static str>>()?;
+
+        let haptics_hnd: Vec<ActionHandle> = PATH_HAPTICS
+            .iter()
+            .map(|path| Ok(result_str!(input.get_action_handle(path))?))
+            .collect::<Result<_, &'static str>>()?;
+
+        let hands: [Pointer<OpenVrHandState>; 2] = array::from_fn(|i| Pointer::<OpenVrHandState> {
+            hand: i,
+            now: PointerState::default(),
+            before: PointerState::default(),
+            pose: Affine3A::IDENTITY,
+            interaction: InteractionState::default(),
+            data: OpenVrHandState {
+                has_pose: false,
+                input_hnd: input_hnd[i],
+                pose_hnd: pose_hnd[i],
+                haptics_hnd: haptics_hnd[i],
+            },
+        });
+
+        Ok(InputState {
+            hmd: Affine3A::IDENTITY,
+            pointers: hands,
+            devices: vec![],
+            data: OpenVrInputState {
+                set_hnd,
+                click_hnd,
+                grab_hnd,
+                scroll_hnd,
+                alt_click_hnd,
+                show_hide_hnd,
+                space_drag_hnd,
+                click_modifier_right_hnd,
+                click_modifier_middle_hnd,
+            },
+        })
+    }
+
+    pub fn update(&mut self, input: &mut InputManager, system: &mut SystemManager) {
+        let aas = ActiveActionSet {
+            0: ovr_overlay::sys::VRActiveActionSet_t {
+                ulActionSet: self.data.set_hnd.0,
+                ulRestrictedToDevice: 0,
+                ulSecondaryActionSet: 0,
+                unPadding: 0,
+                nPriority: 0,
+            },
+        };
+
+        let _ = input.update_actions(&mut [aas]);
+
+        let universe = ETrackingUniverseOrigin::TrackingUniverseStanding;
+
+        for i in 0..2 {
+            let hand = &mut self.pointers[i];
+
+            hand.data.has_pose = false;
+
+            let _ = input
+                .get_pose_action_data_relative_to_now(
+                    hand.data.pose_hnd,
+                    universe.clone(),
+                    0.005,
+                    INPUT_ANY,
+                )
+                .and_then(|pose| {
+                    copy_from_hmd(&pose.0.pose.mDeviceToAbsoluteTracking, &mut hand.pose);
+                    hand.data.has_pose = true;
+                    Ok(())
+                });
+
+            hand.now.click = input
+                .get_digital_action_data(self.data.click_hnd, hand.data.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            hand.now.grab = input
+                .get_digital_action_data(self.data.grab_hnd, hand.data.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            hand.now.alt_click = input
+                .get_digital_action_data(self.data.alt_click_hnd, hand.data.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            hand.now.show_hide = input
+                .get_digital_action_data(self.data.show_hide_hnd, hand.data.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            hand.now.space_drag = input
+                .get_digital_action_data(self.data.space_drag_hnd, hand.data.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            hand.now.click_modifier_right = input
+                .get_digital_action_data(self.data.click_modifier_right_hnd, hand.data.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            hand.now.click_modifier_middle = input
+                .get_digital_action_data(self.data.click_modifier_middle_hnd, hand.data.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            hand.now.scroll = input
+                .get_analog_action_data(self.data.scroll_hnd, hand.data.input_hnd)
+                .map(|x| x.0.x)
+                .unwrap_or(0.0);
+        }
+
+        let devices = system.get_device_to_absolute_tracking_pose(universe, 0.005);
+        copy_from_hmd(&devices[0].mDeviceToAbsoluteTracking, &mut self.hmd);
+    }
+
+    pub fn update_devices(&mut self, system: &mut SystemManager) {
+        self.devices.clear();
+        for i in 0..k_unMaxTrackedDeviceCount {
+            let index = TrackedDeviceIndex(i);
+            let maybe_role = match system.get_tracked_device_class(index) {
+                ETrackedDeviceClass::TrackedDeviceClass_HMD => Some(TrackedDeviceRole::Hmd),
+                ETrackedDeviceClass::TrackedDeviceClass_Controller => {
+                    let sys_role = system.get_controller_role_for_tracked_device_index(index);
+                    match sys_role {
+                        ETrackedControllerRole::TrackedControllerRole_LeftHand => {
+                            Some(TrackedDeviceRole::LeftHand)
+                        }
+                        ETrackedControllerRole::TrackedControllerRole_RightHand => {
+                            Some(TrackedDeviceRole::RightHand)
+                        }
+                        _ => None,
+                    }
+                }
+                ETrackedDeviceClass::TrackedDeviceClass_GenericTracker => {
+                    Some(TrackedDeviceRole::Tracker)
+                }
+                _ => None,
+            };
+
+            if let Some(role) = maybe_role {
+                if let Some(device) = get_tracked_device(system, index, role) {
+                    self.devices.push(device);
+                }
+            }
+        }
+        self.devices.sort_by(|a, b| {
+            (a.role as u8)
+                .cmp(&(b.role as u8))
+                .then(a.index.0.cmp(&b.index.0))
+        });
+    }
+}
+
+fn get_tracked_device(
+    system: &mut SystemManager,
+    index: TrackedDeviceIndex,
+    role: TrackedDeviceRole,
+) -> Option<TrackedDevice> {
+    let soc = system
+        .get_tracked_device_property(
+            index,
+            ETrackedDeviceProperty::Prop_DeviceBatteryPercentage_Float,
+        )
+        .ok();
+
+    let charging = if soc.is_some() {
+        system
+            .get_tracked_device_property(index, ETrackedDeviceProperty::Prop_DeviceIsCharging_Bool)
+            .unwrap_or(false)
+    } else {
+        false
+    };
+
+    Some(TrackedDevice {
+        valid: true,
+        index,
+        soc,
+        charging,
+        role,
+    })
+}
+
+fn copy_from_hmd(in_mat: &HmdMatrix34_t, out_mat: &mut Affine3A) {
+    out_mat.x_axis[0] = in_mat.m[0][0];
+    out_mat.x_axis[1] = in_mat.m[1][0];
+    out_mat.x_axis[2] = in_mat.m[2][0];
+    out_mat.y_axis[0] = in_mat.m[0][1];
+    out_mat.y_axis[1] = in_mat.m[1][1];
+    out_mat.y_axis[2] = in_mat.m[2][1];
+    out_mat.z_axis[0] = in_mat.m[0][2];
+    out_mat.z_axis[1] = in_mat.m[1][2];
+    out_mat.z_axis[2] = in_mat.m[2][2];
+    out_mat.w_axis[0] = in_mat.m[0][3];
+    out_mat.w_axis[1] = in_mat.m[1][3];
+    out_mat.w_axis[2] = in_mat.m[2][3];
+}

src/backend/openvr/mod.rs

@@ -0,0 +1,100 @@
+use std::{
+    path::Path,
+    time::{Duration, Instant},
+};
+
+use log::{error, info};
+use ovr_overlay::{
+    sys::{ETrackedDeviceProperty, EVRApplicationType, EVREventType},
+    TrackedDeviceIndex,
+};
+
+use super::common::InputState;
+
+pub mod input;
+pub mod overlay;
+
+fn openvr_run() {
+    let app_type = EVRApplicationType::VRApplication_Overlay;
+    let Ok(context) = ovr_overlay::Context::init(app_type) else {
+            error!("Failed to initialize OpenVR");
+            return;
+        };
+
+    let mut overlay = context.overlay_mngr();
+    let mut settings = context.settings_mngr();
+    let mut input = context.input_mngr();
+    let mut system = context.system_mngr();
+    let mut compositor = context.compositor_mngr();
+
+    let Ok(_) = input.set_action_manifest(Path::new("resources/actions.json")) else {
+        error!("Failed to set action manifest");
+        return;
+    };
+
+    let Ok(mut input_state) = InputState::new(&mut input) else {
+        error!("Failed to initialize input");
+        return;
+    };
+
+    let Ok(refresh_rate) = system.get_tracked_device_property::<f32>(TrackedDeviceIndex::HMD, ETrackedDeviceProperty::Prop_DisplayFrequency_Float) else {
+        error!("Failed to get display refresh rate");
+        return;
+    };
+
+    let frame_time = (1000.0 / refresh_rate).floor() * 0.001;
+    let mut next_device_update = Instant::now();
+
+    loop {
+        while let Some(event) = system.poll_next_event() {
+            match event.event_type {
+                EVREventType::VREvent_Quit => {
+                    info!("Received quit event, shutting down.");
+                    return;
+                }
+                EVREventType::VREvent_TrackedDeviceActivated
+                | EVREventType::VREvent_TrackedDeviceDeactivated
+                | EVREventType::VREvent_TrackedDeviceUpdated => {
+                    next_device_update = Instant::now();
+                }
+                _ => {}
+            }
+
+            if next_device_update <= Instant::now() {
+                input_state.update_devices(&mut system);
+                next_device_update = Instant::now() + Duration::from_secs(30);
+            }
+
+            input_state.pre_update();
+            input_state.update(&mut input, &mut system);
+            input_state.post_update();
+
+            // task scheduler
+
+            // after input
+
+            // interactions
+
+            // show overlays
+
+            // chaperone
+
+            // render overlays
+
+            // hide overlays
+
+            // close font handles
+
+            // playspace moved end frame
+
+            let mut seconds_since_vsync = 0f32;
+            std::thread::sleep(Duration::from_secs_f32(
+                if system.get_time_since_last_vsync(&mut seconds_since_vsync, &mut 0u64) {
+                    frame_time - seconds_since_vsync
+                } else {
+                    0.011
+                },
+            ));
+        }
+    }
+}

src/backend/openvr/overlay.rs

@@ -0,0 +1,15 @@
+use ovr_overlay::sys::VRVulkanTextureData_t;
+
+use crate::overlays::OverlayData;
+
+pub(super) struct OpenVrOverlayManager {
+    pub(super) overlays: Vec<OpenVrOverlay>,
+}
+
+pub(super) struct OpenVrOverlay {
+    pub(super) visible: bool,
+    pub(super) color: [f32; 4],
+    overlay: OverlayData,
+    handle: u32,
+    ovr_texture: VRVulkanTextureData_t,
+}

src/graphics.rs

@@ -0,0 +1,559 @@
+use std::{sync::Arc, slice::Iter, io::Cursor, error::Error};
+
+use log::{info,error};
+use vulkano::{format::Format, device::{physical::PhysicalDeviceType, QueueFlags, DeviceExtensions, Device, DeviceCreateInfo, Features, QueueCreateInfo, Queue}, Version, VulkanLibrary, instance::{Instance, InstanceCreateInfo}, memory::allocator::{StandardMemoryAllocator, AllocationCreateInfo, MemoryUsage}, command_buffer::{allocator::StandardCommandBufferAllocator, CommandBufferUsage, AutoCommandBufferBuilder, PrimaryAutoCommandBuffer, RenderingAttachmentInfo, RenderingInfo, PrimaryCommandBufferAbstract, CommandBufferExecFuture, SubpassContents, SecondaryAutoCommandBuffer, CommandBufferInheritanceInfo, CommandBufferInheritanceRenderPassType, CommandBufferInheritanceRenderingInfo}, descriptor_set::{allocator::StandardDescriptorSetAllocator, PersistentDescriptorSet, WriteDescriptorSet}, buffer::{Buffer, BufferCreateInfo, BufferUsage, BufferContents, Subbuffer, allocator::{SubbufferAllocator, SubbufferAllocatorCreateInfo}}, sampler::{Filter, SamplerCreateInfo, SamplerAddressMode, Sampler}, pipeline::{GraphicsPipeline, Pipeline, graphics::{render_pass::PipelineRenderingCreateInfo, vertex_input::Vertex, input_assembly::InputAssemblyState, viewport::{ViewportState, Viewport}, color_blend::{ColorBlendState, AttachmentBlend}}, PipelineBindPoint}, image::{ImageViewAbstract, ImageUsage, SwapchainImage, ImageDimensions, ImmutableImage, MipmapsCount, StorageImage, ImageError, SubresourceData, ImageCreateFlags, AttachmentImage}, swapchain::{Surface, Swapchain, SwapchainCreateInfo, CompositeAlpha}, shader::ShaderModule, render_pass::{StoreOp, LoadOp}, sync::future::NowFuture};
+use winit::{event_loop::EventLoop, window::{WindowBuilder, Window}};
+use vulkano_win::VkSurfaceBuild;
+use wlx_capture::frame::{DmabufFrame, DRM_FORMAT_ABGR8888, DRM_FORMAT_XBGR8888, DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB8888};
+
+#[repr(C)]
+#[derive(BufferContents, Vertex, Copy, Clone, Debug)]
+pub struct Vert2Uv {
+    #[format(R32G32_SFLOAT)]
+    pub in_pos: [f32; 2],
+    #[format(R32G32_SFLOAT)]
+    pub in_uv: [f32; 2],
+}
+
+pub const INDICES : [u16; 6] = [2, 1, 0, 1, 2, 3];
+
+pub struct WlxGraphics {
+    pub instance: Arc<Instance>, 
+    pub device: Arc<Device>,
+    pub queue: Arc<Queue>,
+
+    pub surface: Arc<Surface>,
+
+    pub memory_allocator: Arc<StandardMemoryAllocator>,
+    pub command_buffer_allocator: Arc<StandardCommandBufferAllocator>,
+    pub descriptor_set_allocator: Arc<StandardDescriptorSetAllocator>,
+
+    pub quad_indices: Subbuffer<[u16]>,
+}
+
+impl WlxGraphics {
+    pub fn new() -> (Arc<Self>, EventLoop<()>) {
+        #[cfg(debug_assertions)]
+        let layers = vec!["VK_LAYER_KHRONOS_validation".to_owned()];
+        #[cfg(not(debug_assertions))]
+        let layers = vec![];
+
+        let library = VulkanLibrary::new().unwrap();
+        let required_extensions = vulkano_win::required_extensions(&library);
+
+        let instance = Instance::new(
+            library,
+            InstanceCreateInfo {
+                enabled_extensions: required_extensions,
+                enabled_layers: layers,
+                enumerate_portability: true,
+                ..Default::default()
+            },
+        )
+        .unwrap();
+
+        let mut device_extensions = DeviceExtensions {
+            khr_swapchain: true,
+            khr_external_memory: true,
+            khr_external_memory_fd: true,
+            ext_external_memory_dma_buf: true,
+            ext_image_drm_format_modifier: true,
+            ..DeviceExtensions::empty()
+        };
+
+        // TODO headless
+        let event_loop = EventLoop::new();
+        let surface = WindowBuilder::new()
+            .build_vk_surface(&event_loop, instance.clone())
+            .unwrap();
+
+
+        let (physical_device, queue_family_index) = instance
+            .enumerate_physical_devices()
+            .unwrap()
+            .filter(|p| {
+                p.api_version() >= Version::V1_3 || p.supported_extensions().khr_dynamic_rendering
+            })
+            .filter(|p| {
+                p.supported_extensions().contains(&device_extensions)
+            })
+            .filter_map(|p| {
+                p.queue_family_properties()
+                    .iter()
+                    .enumerate()
+                    .position(|(i, q)| {
+                        q.queue_flags.intersects(QueueFlags::GRAPHICS)
+                            && p.surface_support(i as u32, &surface).unwrap_or(false)
+                    })
+                    .map(|i| (p, i as u32))
+            })
+            .min_by_key(|(p, _)| {
+                match p.properties().device_type {
+                    PhysicalDeviceType::DiscreteGpu => 0,
+                    PhysicalDeviceType::IntegratedGpu => 1,
+                    PhysicalDeviceType::VirtualGpu => 2,
+                    PhysicalDeviceType::Cpu => 3,
+                    PhysicalDeviceType::Other => 4,
+                    _ => 5,
+                }
+            })
+            .expect("no suitable physical device found");
+
+        info!(
+            "Nice {} you have there.",
+            physical_device.properties().device_name,
+        );
+
+        if physical_device.api_version() < Version::V1_3 {
+            device_extensions.khr_dynamic_rendering = true;
+        }
+
+        let (device, mut queues) = Device::new(
+            physical_device,
+            DeviceCreateInfo {
+                enabled_extensions: device_extensions,
+                enabled_features: Features {
+                    dynamic_rendering: true,
+                    ..Features::empty()
+                },
+                queue_create_infos: vec![QueueCreateInfo {
+                    queue_family_index,
+                    ..Default::default()
+                }],
+                ..Default::default()
+            },
+        )
+        .unwrap();
+
+        let queue = queues.next().unwrap();
+
+        let memory_allocator = Arc::new(StandardMemoryAllocator::new_default(device.clone()));
+        let command_buffer_allocator = Arc::new(StandardCommandBufferAllocator::new(device.clone(), Default::default()));
+        let descriptor_set_allocator = Arc::new(StandardDescriptorSetAllocator::new(device.clone()));
+
+        let quad_indices = Buffer::from_iter(
+            &memory_allocator,
+            BufferCreateInfo {
+                usage: BufferUsage::INDEX_BUFFER,
+                ..Default::default()
+            },
+            AllocationCreateInfo {
+                usage: MemoryUsage::Upload,
+                ..Default::default()
+            },
+            INDICES.iter().cloned(),
+        ).unwrap();
+
+        let me = Self {
+            instance,
+            device,
+            queue,
+            surface,
+            memory_allocator,
+            command_buffer_allocator,
+            descriptor_set_allocator,
+            quad_indices,
+        };
+
+        (Arc::new(me), event_loop)
+    }
+
+    pub fn create_swapchain(&self, format: Option<Format>) -> (Arc<Swapchain>, Vec<Arc<SwapchainImage>>) {
+        let (min_image_count, composite_alpha, image_format) = if let Some(format) = format {
+            (1, CompositeAlpha::Opaque, format)
+        } else {
+            let surface_capabilities = self.device
+                .physical_device()
+                .surface_capabilities(&self.surface, Default::default())
+                .unwrap();
+            
+            let composite_alpha = surface_capabilities.supported_composite_alpha.into_iter().next().unwrap();
+
+            let image_format = Some(
+                self.device
+                    .physical_device()
+                    .surface_formats(&self.surface, Default::default())
+                    .unwrap()[0]
+                    .0,
+            );
+            (surface_capabilities.min_image_count, composite_alpha, image_format.unwrap())
+        };
+        let window = self.surface.object().unwrap().downcast_ref::<Window>().unwrap();
+        let swapchain = Swapchain::new(
+            self.device.clone(),
+            self.surface.clone(),
+            SwapchainCreateInfo {
+                min_image_count,
+                image_format: Some(image_format),
+                image_extent: window.inner_size().into(),
+                image_usage: ImageUsage::COLOR_ATTACHMENT,
+                composite_alpha,
+                ..Default::default()
+            },
+        )
+        .unwrap();
+
+        swapchain
+    }
+
+    pub fn upload_verts(&self, width: f32, height: f32, x: f32, y: f32, w: f32, h: f32) -> Subbuffer<[Vert2Uv]> {
+        let rw = width;
+        let rh = height;
+
+        let x0 = x / rw;
+        let y0 = y / rh;
+
+        let x1 = w / rw + x0;
+        let y1 = h / rh + y0;
+
+        let vertices = [
+            Vert2Uv { in_pos: [x0, y0], in_uv: [0.0, 0.0] },
+            Vert2Uv { in_pos: [x0, y1], in_uv: [0.0, 1.0] },
+            Vert2Uv { in_pos: [x1, y0], in_uv: [1.0, 0.0] },
+            Vert2Uv { in_pos: [x1, y1], in_uv: [1.0, 1.0] },
+        ];
+        self.upload_buffer(BufferUsage::VERTEX_BUFFER, vertices.iter())
+    }
+
+    pub fn upload_buffer<T>(&self, usage: BufferUsage, contents: Iter<'_, T>) -> Subbuffer<[T]>
+    where T: BufferContents + Clone {
+        Buffer::from_iter(
+            &self.memory_allocator,
+            BufferCreateInfo {
+                usage,
+                ..Default::default()
+            },
+            AllocationCreateInfo {
+                usage: MemoryUsage::Upload,
+                ..Default::default()
+            },
+            contents.cloned(),
+        ).unwrap()
+    } 
+
+    pub fn dmabuf_texture(&self, frame: DmabufFrame) -> Result<Arc<StorageImage>, ImageError> {
+        let dimensions = ImageDimensions::Dim2d {
+            width: frame.format.width,
+            height: frame.format.height,
+            array_layers: 1,
+        };
+
+        let format = match frame.format.fourcc {
+            DRM_FORMAT_ABGR8888 => Format::R8G8B8A8_UNORM,
+            DRM_FORMAT_XBGR8888 => Format::R8G8B8A8_UNORM,
+            DRM_FORMAT_ARGB8888 => Format::B8G8R8A8_UNORM,
+            DRM_FORMAT_XRGB8888 => Format::B8G8R8A8_UNORM,
+            _ => panic!("Unsupported dmabuf format {:x}", frame.format.fourcc),
+        };
+
+        let planes = frame.planes
+            .iter()
+            .take(frame.num_planes)
+            .filter_map(|plane| {
+                let Some(fd) = plane.fd else {
+                    return None;
+                };
+                Some(SubresourceData {
+                    fd,
+                    offset: plane.offset as _,
+                    row_pitch: plane.stride as _,
+                })
+            }).collect();
+
+        StorageImage::new_from_dma_buf_fd(
+            &self.memory_allocator,
+            self.device.clone(),
+            dimensions,
+            format,
+            ImageUsage::SAMPLED | ImageUsage::TRANSFER_SRC,
+            ImageCreateFlags::empty(),
+            [self.queue.queue_family_index()],
+            planes,
+            frame.format.modifier,
+        )
+    }
+    pub fn render_texture(&self, width: u32, height: u32, format: Format) -> Arc<AttachmentImage> {
+        let tex = AttachmentImage::with_usage(
+            &self.memory_allocator, 
+            [width, height], 
+            format,
+            ImageUsage::SAMPLED | ImageUsage::TRANSFER_SRC | ImageUsage::COLOR_ATTACHMENT,
+        ).unwrap();
+
+        tex
+    }
+    pub fn create_pipeline(self: &Arc<Self>, vert: Arc<ShaderModule>, frag: Arc<ShaderModule>, format: Format) -> Arc<WlxPipeline> {
+        Arc::new(WlxPipeline::new(self.clone(), vert, frag, format))
+    }
+    pub fn create_command_buffer(self: &Arc<Self>, usage: CommandBufferUsage) -> WlxCommandBuffer<PrimaryAutoCommandBuffer> {
+        let command_buffer = AutoCommandBufferBuilder::primary(
+            &self.command_buffer_allocator,
+            self.queue.queue_family_index(),
+            usage,
+        ).unwrap();
+        WlxCommandBuffer { graphics: self.clone(), command_buffer }
+    }
+
+}
+
+pub struct WlxCommandBuffer<T> {
+    graphics: Arc<WlxGraphics>,
+    command_buffer: AutoCommandBufferBuilder<T, Arc<StandardCommandBufferAllocator>>,
+}
+
+impl<T> WlxCommandBuffer<T> {
+    pub fn inner(&self) -> &AutoCommandBufferBuilder<T, Arc<StandardCommandBufferAllocator>> {
+        &self.command_buffer
+    }
+
+    pub fn inner_mut(&mut self) -> &mut AutoCommandBufferBuilder<T, Arc<StandardCommandBufferAllocator>> {
+        &mut self.command_buffer
+    }
+
+    pub fn to_inner(self) -> AutoCommandBufferBuilder<T, Arc<StandardCommandBufferAllocator>> {
+        self.command_buffer
+    }
+
+    pub fn begin(mut self, render_target: Arc<dyn ImageViewAbstract>) -> Self 
+    {
+        self.command_buffer
+                .begin_rendering(RenderingInfo {
+                    contents: SubpassContents::SecondaryCommandBuffers, 
+                    color_attachments: vec![Some(RenderingAttachmentInfo {
+                        load_op: LoadOp::Clear,
+                        store_op: StoreOp::Store,
+                        clear_value: Some([0.0, 0.0, 0.0, 0.0].into()),
+                        ..RenderingAttachmentInfo::image_view(
+                            render_target.clone(),
+                        )
+                    })],
+                    ..Default::default()
+                }).unwrap();
+        self
+    }
+
+    pub fn run_ref(&mut self, pass: &WlxPass) -> &mut Self
+    {
+        let _ = self.command_buffer.execute_commands(pass.command_buffer.clone()).unwrap();
+        self
+    }
+
+    pub fn run(mut self, pass: &WlxPass) -> Self
+    {
+        let _ = self.command_buffer.execute_commands(pass.command_buffer.clone());
+        self
+    }
+
+    pub fn texture2d(&mut self, width: u32, height: u32, format: Format, data: Vec<u8>) -> Arc<ImmutableImage> {
+        let dimensions = ImageDimensions::Dim2d {
+            width,
+            height,
+            array_layers: 1,
+        };
+
+        ImmutableImage::from_iter(
+            &self.graphics.memory_allocator,
+            data,
+            dimensions,
+            MipmapsCount::One,
+            format,
+            &mut self.command_buffer,
+        )
+        .unwrap()
+    }
+
+    pub fn texture2d_png(&mut self, bytes: Vec<u8>) -> Arc<ImmutableImage> {
+        let cursor = Cursor::new(bytes);
+        let decoder = png::Decoder::new(cursor);
+        let mut reader = decoder.read_info().unwrap();
+        let info = reader.info();
+        let width = info.width;
+        let height = info.height;
+        let mut image_data = Vec::new();
+        image_data.resize((info.width * info.height * 4) as usize, 0);
+        reader.next_frame(&mut image_data).unwrap();
+        self.texture2d(width, height, Format::R8G8B8A8_UNORM, image_data)
+    }
+
+}
+
+impl WlxCommandBuffer<PrimaryAutoCommandBuffer> {
+    pub fn end_render_and_continue(&mut self) {
+        self.command_buffer.end_rendering().unwrap();
+    }
+
+    pub fn end_render(self) -> PrimaryAutoCommandBuffer {
+        let mut buf = self.command_buffer;
+        buf.end_rendering().unwrap();
+
+        buf.build().unwrap()
+    }
+
+    pub fn end(self) -> PrimaryAutoCommandBuffer {
+        self.command_buffer.build().unwrap()
+    }
+
+    pub fn end_render_and_execute(self) -> CommandBufferExecFuture<NowFuture> {
+        let mut buf = self.command_buffer;
+        buf.end_rendering().unwrap();
+        let buf = buf.build().unwrap();
+        buf.execute(self.graphics.queue.clone()).unwrap()
+    }
+
+    pub fn end_and_execute(self) -> CommandBufferExecFuture<NowFuture> {
+        let buf = self.command_buffer;
+        let buf = buf.build().unwrap();
+        buf.execute(self.graphics.queue.clone()).unwrap()
+    }
+}
+
+pub struct WlxPipeline {
+    pub graphics: Arc<WlxGraphics>,
+    pub pipeline: Arc<GraphicsPipeline>,
+    pub format: Format,
+}
+
+impl WlxPipeline {
+    fn new(graphics: Arc<WlxGraphics>, vert: Arc<ShaderModule>, frag: Arc<ShaderModule>, format: Format) -> Self {
+        let vep = vert.entry_point("main").unwrap();
+        let fep = frag.entry_point("main").unwrap();
+        let pipeline = GraphicsPipeline::start()
+        .render_pass(PipelineRenderingCreateInfo {
+            color_attachment_formats: vec![Some(format)],
+            ..Default::default()
+        })
+        .color_blend_state(ColorBlendState::default().blend(
+                AttachmentBlend::alpha()
+            ))
+        .vertex_input_state(Vert2Uv::per_vertex())
+        .input_assembly_state(InputAssemblyState::new())
+        .vertex_shader(vep, ())
+        .viewport_state(ViewportState::viewport_dynamic_scissor_irrelevant())
+        .fragment_shader(fep, ())
+        .build(graphics.device.clone())
+        .unwrap();
+
+        Self { graphics, pipeline, format}
+    }
+
+    pub fn inner(&self) -> Arc<GraphicsPipeline> {
+        self.pipeline.clone()
+    }
+
+    pub fn graphics(&self) -> Arc<WlxGraphics> {
+        self.graphics.clone()
+    }
+
+    pub fn uniform_sampler(&self, set: usize, texture: Arc<dyn ImageViewAbstract>, filter: Filter) -> Arc<PersistentDescriptorSet> {
+        let sampler = Sampler::new(
+            self.graphics.device.clone(),
+            SamplerCreateInfo {
+                mag_filter: filter,
+                min_filter: filter,
+                address_mode: [SamplerAddressMode::Repeat; 3],
+                ..Default::default()
+            },
+        )
+        .unwrap();
+ 
+        let layout = self.pipeline.layout().set_layouts().get(set).unwrap();
+
+        PersistentDescriptorSet::new(
+            &self.graphics.descriptor_set_allocator,
+            layout.clone(),
+            [WriteDescriptorSet::image_view_sampler(0, texture, sampler)],
+        )
+        .unwrap()
+    }
+
+    pub fn uniform_buffer<T>(&self, set: usize, data: Vec<T>) -> Arc<PersistentDescriptorSet>
+    where T: BufferContents + Copy {
+        let uniform_buffer = SubbufferAllocator::new(
+            self.graphics.memory_allocator.clone(),
+            SubbufferAllocatorCreateInfo {
+                buffer_usage: BufferUsage::UNIFORM_BUFFER,
+                ..Default::default()
+            },
+        );
+
+        let uniform_buffer_subbuffer = {
+            let subbuffer = uniform_buffer.allocate_slice(data.len() as _).unwrap();
+            subbuffer.write().unwrap().copy_from_slice(data.as_slice());
+            subbuffer
+        };
+
+        let layout = self.pipeline.layout().set_layouts().get(set).unwrap();
+        PersistentDescriptorSet::new(
+            &self.graphics.descriptor_set_allocator,
+            layout.clone(),
+            [WriteDescriptorSet::buffer(0, uniform_buffer_subbuffer)]
+        ).unwrap()
+    }
+
+    pub fn create_pass(self: &Arc<Self>, dimensions: [f32; 2], vertex_buffer: Subbuffer<[Vert2Uv]>, index_buffer: Subbuffer<[u16]>, descriptor_sets: Vec<Arc<PersistentDescriptorSet>>) -> WlxPass {
+        WlxPass::new(self.clone(), dimensions, vertex_buffer, index_buffer, descriptor_sets)
+    }
+}
+
+pub struct WlxPass
+{
+    pipeline: Arc<WlxPipeline>,
+    vertex_buffer: Subbuffer<[Vert2Uv]>,
+    index_buffer: Subbuffer<[u16]>,
+    descriptor_sets: Vec<Arc<PersistentDescriptorSet>>,
+    pub command_buffer: Arc<SecondaryAutoCommandBuffer>,
+} 
+
+impl WlxPass
+{
+    fn new(pipeline: Arc<WlxPipeline>, dimensions: [f32; 2], vertex_buffer: Subbuffer<[Vert2Uv]>, index_buffer: Subbuffer<[u16]>, descriptor_sets: Vec<Arc<PersistentDescriptorSet>>) -> Self {
+        let viewport = Viewport {
+            origin: [0.0, 0.0],
+            dimensions,
+            depth_range: 0.0..1.0,
+        };
+
+        let pipeline_inner = pipeline.inner().clone();
+        let mut command_buffer = AutoCommandBufferBuilder::secondary(
+                &pipeline.graphics.command_buffer_allocator,
+                pipeline.graphics.queue.queue_family_index(),
+                CommandBufferUsage::MultipleSubmit,
+                CommandBufferInheritanceInfo { 
+                    render_pass: Some(CommandBufferInheritanceRenderPassType::BeginRendering(
+                        CommandBufferInheritanceRenderingInfo {
+                            color_attachment_formats: vec![Some(pipeline.format)],
+                            ..Default::default()
+                        })),
+                    ..Default::default()
+                }
+            )
+            .unwrap();
+
+            command_buffer.set_viewport(0, [viewport])
+            .bind_pipeline_graphics(pipeline_inner)
+            .bind_descriptor_sets(
+                PipelineBindPoint::Graphics,
+                pipeline.inner().layout().clone(),
+                0,
+                descriptor_sets.clone(),
+            )
+            .bind_vertex_buffers(0, vertex_buffer.clone())
+            .bind_index_buffer(index_buffer.clone())
+            .draw_indexed(index_buffer.len() as u32, 1, 0, 0, 0)
+            .or_else(|err| {
+                if let Some(source) = err.source() {
+                    error!("Failed to draw: {}", source);
+                }
+                Err(err)
+            }).unwrap();
+
+        Self { 
+            pipeline,
+            vertex_buffer,
+            index_buffer,
+            descriptor_sets,
+            command_buffer: Arc::new(command_buffer.build().unwrap()),
+        }
+    }
+}
+

src/gui/font.rs

@@ -0,0 +1,211 @@
+use std::{rc::Rc, str::FromStr, sync::Arc};
+
+use fontconfig::{FontConfig, OwnedPattern};
+use freetype::{bitmap::PixelMode, face::LoadFlag, Face, Library};
+use idmap::IdMap;
+use log::debug;
+use vulkano::{format::Format, command_buffer::CommandBufferUsage, image::ImmutableImage};
+
+use crate::graphics::WlxGraphics;
+
+const PRIMARY_FONT: &str = "LiberationSans";
+
+pub struct FontCache {
+    fc: FontConfig,
+    ft: Library,
+    collections: IdMap<isize, FontCollection>,
+}
+
+struct FontCollection {
+    fonts: Vec<Font>,
+    cp_map: IdMap<usize, usize>,
+}
+
+struct Font {
+    face: Face,
+    path: String,
+    index: isize,
+    size: isize,
+    glyphs: IdMap<usize, Rc<Glyph>>,
+}
+
+pub struct Glyph {
+    pub tex: Option<Arc<ImmutableImage>>,
+    pub top: f32,
+    pub left: f32,
+    pub width: f32,
+    pub height: f32,
+    pub advance: f32,
+}
+
+impl FontCache {
+    pub fn new() -> Self {
+        let ft = Library::init().expect("Failed to initialize freetype");
+        let fc = FontConfig::default();
+
+        FontCache {
+            fc,
+            ft,
+            collections: IdMap::new(),
+        }
+    }
+
+    pub fn get_text_size(&mut self, text: &str, size: isize, graphics: Arc<WlxGraphics>) -> (f32, f32) {
+        let sizef = size as f32;
+
+        let height = sizef + ((text.lines().count() as f32) - 1f32) * (sizef * 1.5);
+
+        let mut max_w = sizef * 0.33;
+        for line in text.lines() {
+            let w: f32 = line
+                .chars()
+                .map(|c| self.get_glyph_for_cp(c as usize, size, graphics.clone()).advance)
+                .sum();
+
+            if w > max_w {
+                max_w = w;
+            }
+        }
+        (max_w, height)
+    }
+
+    pub fn get_glyphs(&mut self, text: &str, size: isize, graphics: Arc<WlxGraphics>) -> Vec<Rc<Glyph>> {
+        let mut glyphs = Vec::new();
+        for line in text.lines() {
+            for c in line.chars() {
+                glyphs.push(self.get_glyph_for_cp(c as usize, size, graphics.clone()));
+            }
+        }
+        glyphs
+    }
+
+    fn get_font_for_cp(&mut self, cp: usize, size: isize) -> usize {
+        if !self.collections.contains_key(size) {
+            self.collections.insert(
+                size,
+                FontCollection {
+                    fonts: Vec::new(),
+                    cp_map: IdMap::new(),
+                },
+            );
+        }
+        let coll = self.collections.get_mut(size).unwrap();
+
+        if let Some(font) = coll.cp_map.get(cp) {
+            return *font;
+        }
+
+        let pattern_str = format!("{PRIMARY_FONT}-{size}:style=bold:charset={cp:04x}");
+
+        let mut pattern =
+            OwnedPattern::from_str(&pattern_str).expect("Failed to create fontconfig pattern");
+        self.fc
+            .substitute(&mut pattern, fontconfig::MatchKind::Pattern);
+        pattern.default_substitute();
+
+        let pattern = pattern.font_match(&mut self.fc);
+
+        if let Some(path) = pattern.filename() {
+            debug!(
+                "Loading font: {} {}pt",
+                pattern.name().unwrap_or(path),
+                size
+            );
+
+            let font_idx = pattern.face_index().unwrap_or(0);
+
+            let face = self
+                .ft
+                .new_face(path, font_idx as _)
+                .expect("Failed to load font face");
+            face.set_char_size(size << 6, size << 6, 96, 96)
+                .expect("Failed to set font size");
+
+            let idx = coll.fonts.len();
+            for cp in 0..0xFFFF {
+                if coll.cp_map.contains_key(cp) {
+                    continue;
+                }
+                let g = face.get_char_index(cp);
+                if g > 0 {
+                    coll.cp_map.insert(cp, idx);
+                }
+            }
+
+            let zero_glyph = Rc::new(Glyph {
+                tex: None,
+                top: 0.,
+                left: 0.,
+                width: 0.,
+                height: 0.,
+                advance: size as f32 / 3.,
+            });
+            let mut glyphs = IdMap::new();
+            glyphs.insert(0, zero_glyph);
+
+            let font = Font {
+                face,
+                path: path.to_string(),
+                size,
+                index: font_idx as _,
+                glyphs,
+            };
+            coll.fonts.push(font);
+
+            idx
+        } else {
+            coll.cp_map.insert(cp, 0);
+            0
+        }
+    }
+
+    fn get_glyph_for_cp(&mut self, cp: usize, size: isize, graphics: Arc<WlxGraphics>) -> Rc<Glyph> {
+        let key = self.get_font_for_cp(cp, size);
+
+        let font = &mut self.collections[size].fonts[key];
+
+        if let Some(glyph) = font.glyphs.get(cp) {
+            return glyph.clone();
+        }
+
+        if font.face.load_char(cp, LoadFlag::DEFAULT).is_err() {
+            return font.glyphs[0].clone();
+        }
+
+        let glyph = font.face.glyph();
+        if glyph.render_glyph(freetype::RenderMode::Normal).is_err() {
+            return font.glyphs[0].clone();
+        }
+
+        let bmp = glyph.bitmap();
+        let buf = bmp.buffer().to_vec();
+        if buf.len() == 0 {
+            return font.glyphs[0].clone();
+        }
+
+        let metrics = glyph.metrics();
+
+        let format = match bmp.pixel_mode() {
+            Ok(PixelMode::Gray) => Format::R8_UNORM,
+            Ok(PixelMode::Gray2) => Format::R16_SFLOAT,
+            Ok(PixelMode::Gray4) => Format::R32_SFLOAT,
+            _ => return font.glyphs[0].clone(),
+        };
+
+        let mut cmd_buffer = graphics.create_command_buffer(CommandBufferUsage::OneTimeSubmit);
+        let texture = cmd_buffer.texture2d(bmp.width() as _, bmp.rows() as _, format, buf);
+        let _ = cmd_buffer.end_and_execute();
+
+        let g = Glyph {
+            tex: Some(texture),
+            top: (metrics.horiBearingY >> 6i64) as _,
+            left: (metrics.horiBearingX >> 6i64) as _,
+            advance: (metrics.horiAdvance >> 6i64) as _,
+            width: bmp.width() as _,
+            height: bmp.rows() as _,
+        };
+
+        font.glyphs.insert(cp, Rc::new(g));
+        font.glyphs[cp].clone()
+    }
+}

src/gui/mod.rs

@@ -0,0 +1,669 @@
+use std::sync::Arc;
+
+use glam::{Vec2, Vec3};
+use vulkano::{
+    command_buffer::{CommandBufferUsage, PrimaryAutoCommandBuffer},
+    format::Format,
+    image::{view::ImageView, AttachmentImage},
+    sampler::Filter,
+};
+
+use crate::{
+    graphics::{WlxCommandBuffer, WlxGraphics, WlxPass, WlxPipeline},
+    overlays::{
+        interactions::{InteractionHandler, PointerHit},
+        OverlayBackend, OverlayRenderer,
+    },
+    shaders::{frag_color, frag_glyph, frag_sprite, vert_common},
+    state::AppState,
+};
+
+pub mod font;
+
+const RES_DIVIDER: usize = 4;
+
+struct Rect {
+    x: f32,
+    y: f32,
+    w: f32,
+    h: f32,
+}
+
+// Parses a color from a HTML hex string
+pub fn color_parse(html_hex: &str) -> Vec3 {
+    let mut color = Vec3::ZERO;
+    color.x = u8::from_str_radix(&html_hex[1..3], 16).unwrap() as f32 / 255.;
+    color.y = u8::from_str_radix(&html_hex[3..5], 16).unwrap() as f32 / 255.;
+    color.z = u8::from_str_radix(&html_hex[5..7], 16).unwrap() as f32 / 255.;
+    color
+}
+
+pub struct CanvasBuilder<D, S> {
+    canvas: Canvas<D, S>,
+
+    pub fg_color: Vec3,
+    pub bg_color: Vec3,
+    pub font_size: isize,
+}
+
+impl<D, S> CanvasBuilder<D, S> {
+    pub fn new(
+        width: usize,
+        height: usize,
+        graphics: Arc<WlxGraphics>,
+        format: Format,
+        data: D,
+    ) -> Self {
+        Self {
+            canvas: Canvas::new(width, height, graphics, format, data),
+            bg_color: Vec3::ZERO,
+            fg_color: Vec3::ONE,
+            font_size: 16,
+        }
+    }
+
+    pub fn build(self) -> Canvas<D, S> {
+        self.canvas
+    }
+
+    // Creates a panel with bg_color inherited from the canvas
+    pub fn panel(&mut self, x: f32, y: f32, w: f32, h: f32) -> &mut Control<D, S> {
+        let idx = self.canvas.controls.len();
+        self.canvas.controls.push(Control {
+            rect: Rect { x, y, w, h },
+            bg_color: self.bg_color,
+            on_render_bg: Some(Control::render_rect),
+            ..Control::new()
+        });
+        &mut self.canvas.controls[idx]
+    }
+
+    // Creates a label with fg_color, font_size inherited from the canvas
+    pub fn label(&mut self, x: f32, y: f32, w: f32, h: f32, text: Arc<str>) -> &mut Control<D, S> {
+        let idx = self.canvas.controls.len();
+        self.canvas.controls.push(Control {
+            rect: Rect { x, y, w, h },
+            text,
+            fg_color: self.fg_color,
+            size: self.font_size,
+            on_render_fg: Some(Control::render_text),
+            ..Control::new()
+        });
+        &mut self.canvas.controls[idx]
+    }
+
+    // Creates a label with fg_color, font_size inherited from the canvas
+    pub fn label_centered(
+        &mut self,
+        x: f32,
+        y: f32,
+        w: f32,
+        h: f32,
+        text: Arc<str>,
+    ) -> &mut Control<D, S> {
+        let idx = self.canvas.controls.len();
+        self.canvas.controls.push(Control {
+            rect: Rect { x, y, w, h },
+            text,
+            fg_color: self.fg_color,
+            size: self.font_size,
+            on_render_fg: Some(Control::render_text_centered),
+            ..Control::new()
+        });
+        &mut self.canvas.controls[idx]
+    }
+
+    // Creates a button with fg_color, bg_color, font_size inherited from the canvas
+    pub fn button(&mut self, x: f32, y: f32, w: f32, h: f32, text: Arc<str>) -> &mut Control<D, S> {
+        let idx = self.canvas.controls.len();
+
+        self.canvas.interactive_set_idx(x, y, w, h, idx);
+        self.canvas.controls.push(Control {
+            rect: Rect { x, y, w, h },
+            text,
+            fg_color: self.fg_color,
+            bg_color: self.bg_color,
+            size: self.font_size,
+            on_render_bg: Some(Control::render_rect),
+            on_render_fg: Some(Control::render_text_centered),
+            on_render_hl: Some(Control::render_highlight),
+            ..Control::new()
+        });
+
+        &mut self.canvas.controls[idx]
+    }
+
+    pub fn key_button(
+        &mut self,
+        x: f32,
+        y: f32,
+        w: f32,
+        h: f32,
+        label: &Vec<String>,
+    ) -> &mut Control<D, S> {
+        let idx = self.canvas.controls.len();
+        self.canvas.interactive_set_idx(x, y, w, h, idx);
+
+        self.canvas.controls.push(Control {
+            rect: Rect { x, y, w, h },
+            bg_color: self.bg_color,
+            on_render_bg: Some(Control::render_rect),
+            on_render_hl: Some(Control::render_highlight),
+            ..Control::new()
+        });
+
+        for (i, item) in label.iter().enumerate().take(label.len().min(2)) {
+            self.canvas.controls.push(Control {
+                rect: if i == 0 {
+                    Rect {
+                        x: x + 4.,
+                        y: y + (self.font_size as f32) + 4.,
+                        w,
+                        h,
+                    }
+                } else {
+                    Rect {
+                        x: x + w * 0.5,
+                        y: y + h - (self.font_size as f32) + 4.,
+                        w,
+                        h,
+                    }
+                },
+                text: Arc::from(item.as_str()),
+                fg_color: self.fg_color,
+                size: self.font_size,
+                on_render_fg: Some(Control::render_text),
+                ..Control::new()
+            });
+        }
+
+        &mut self.canvas.controls[idx]
+    }
+}
+
+pub struct CanvasData<D> {
+    pub data: D,
+    pub width: usize,
+    pub height: usize,
+
+    graphics: Arc<WlxGraphics>,
+
+    pipeline_color: Arc<WlxPipeline>,
+    pipeline_glyph: Arc<WlxPipeline>,
+}
+
+pub struct Canvas<D, S> {
+    controls: Vec<Control<D, S>>,
+    canvas: CanvasData<D>,
+
+    hover_controls: [Option<usize>; 2],
+    pressed_controls: [Option<usize>; 2],
+
+    interact_map: Vec<Option<u8>>,
+    interact_stride: usize,
+    interact_rows: usize,
+
+    tex_fg: Arc<AttachmentImage>,
+    view_fg: Arc<ImageView<AttachmentImage>>,
+    tex_bg: Arc<AttachmentImage>,
+    view_bg: Arc<ImageView<AttachmentImage>>,
+    tex_final: Arc<AttachmentImage>,
+    view_final: Arc<ImageView<AttachmentImage>>,
+
+    pass_fg: WlxPass,
+    pass_bg: WlxPass,
+}
+
+impl<D, S> Canvas<D, S> {
+    fn new(
+        width: usize,
+        height: usize,
+        graphics: Arc<WlxGraphics>,
+        format: Format,
+        data: D,
+    ) -> Self {
+        let pipeline_color = graphics.create_pipeline(
+            vert_common::load(graphics.device.clone()).unwrap(),
+            frag_color::load(graphics.device.clone()).unwrap(),
+            format,
+        );
+
+        let pipeline_glyph = graphics.create_pipeline(
+            vert_common::load(graphics.device.clone()).unwrap(),
+            frag_glyph::load(graphics.device.clone()).unwrap(),
+            format,
+        );
+
+        let vertex_buffer =
+            graphics.upload_verts(width as _, height as _, 0., 0., width as _, height as _);
+
+        let pipeline = graphics.create_pipeline(
+            vert_common::load(graphics.device.clone()).unwrap(),
+            frag_sprite::load(graphics.device.clone()).unwrap(),
+            format,
+        );
+
+        let tex_fg = graphics.render_texture(width as _, height as _, format);
+        let tex_bg = graphics.render_texture(width as _, height as _, format);
+        let tex_final = graphics.render_texture(width as _, height as _, format);
+
+        let view_fg = ImageView::new_default(tex_fg.clone()).unwrap();
+        let view_bg = ImageView::new_default(tex_bg.clone()).unwrap();
+        let view_final = ImageView::new_default(tex_final.clone()).unwrap();
+
+        let set_fg = pipeline.uniform_sampler(0, view_fg.clone(), Filter::Nearest);
+        let set_bg = pipeline.uniform_sampler(0, view_bg.clone(), Filter::Nearest);
+        let pass_fg = pipeline.create_pass(
+            [width as _, height as _],
+            vertex_buffer.clone(),
+            graphics.quad_indices.clone(),
+            vec![set_fg],
+        );
+        let pass_bg = pipeline.create_pass(
+            [width as _, height as _],
+            vertex_buffer.clone(),
+            graphics.quad_indices.clone(),
+            vec![set_bg],
+        );
+
+        let stride = width / RES_DIVIDER;
+        let rows = height / RES_DIVIDER;
+
+        Self {
+            canvas: CanvasData {
+                data,
+                width,
+                height,
+                graphics,
+                pipeline_color,
+                pipeline_glyph,
+            },
+            controls: Vec::new(),
+            hover_controls: [None, None],
+            pressed_controls: [None, None],
+            interact_map: vec![None; stride * rows],
+            interact_stride: stride,
+            interact_rows: rows,
+            tex_fg,
+            view_fg,
+            tex_bg,
+            view_bg,
+            tex_final,
+            view_final,
+            pass_fg,
+            pass_bg,
+        }
+    }
+
+    fn interactive_set_idx(&mut self, x: f32, y: f32, w: f32, h: f32, idx: usize) {
+        let (x, y, w, h) = (x as usize, y as usize, w as usize, h as usize);
+
+        let x_min = (x / RES_DIVIDER).max(0);
+        let y_min = (y / RES_DIVIDER).max(0);
+        let x_max = (x_min + (w / RES_DIVIDER)).min(self.interact_stride - 1);
+        let y_max = (y_min + (h / RES_DIVIDER)).min(self.interact_rows - 1);
+
+        for y in y_min..y_max {
+            for x in x_min..x_max {
+                self.interact_map[y * self.interact_stride + x] = Some(idx as u8);
+            }
+        }
+    }
+
+    fn interactive_get_idx(&self, uv: Vec2) -> Option<usize> {
+        let x = (uv.x * self.canvas.width as f32) as usize;
+        let y = (uv.y * self.canvas.height as f32) as usize;
+        let x = (x / RES_DIVIDER).max(0).min(self.interact_stride - 1);
+        let y = (y / RES_DIVIDER).max(0).min(self.interact_rows - 1);
+        self.interact_map[y * self.interact_stride + x].map(|x| x as usize)
+    }
+
+    fn render_bg(&mut self, app: &mut AppState) {
+        let mut cmd_buffer = self
+            .canvas
+            .graphics
+            .create_command_buffer(CommandBufferUsage::OneTimeSubmit)
+            .begin(self.view_bg.clone());
+        for c in self.controls.iter_mut() {
+            if let Some(fun) = c.on_render_bg {
+                fun(c, &self.canvas, app, &mut cmd_buffer);
+            }
+        }
+        let _ = cmd_buffer.end_render_and_execute();
+    }
+
+    fn render_fg(&mut self, app: &mut AppState) {
+        let mut cmd_buffer = self
+            .canvas
+            .graphics
+            .create_command_buffer(CommandBufferUsage::OneTimeSubmit)
+            .begin(self.view_fg.clone());
+        for c in self.controls.iter_mut() {
+            if let Some(fun) = c.on_render_fg {
+                fun(c, &self.canvas, app, &mut cmd_buffer);
+            }
+        }
+        let _ = cmd_buffer.end_render_and_execute();
+    }
+
+    pub fn render_view(&self) -> Arc<ImageView<AttachmentImage>> {
+        self.view_final.clone()
+    }
+}
+
+impl<D, S> InteractionHandler for Canvas<D, S> {
+    fn on_left(&mut self, _app: &mut AppState, hand: usize) {
+        self.hover_controls[hand] = None;
+    }
+    fn on_hover(&mut self, _app: &mut AppState, hit: &PointerHit) {
+        if let Some(i) = self.interactive_get_idx(hit.uv) {
+            self.hover_controls[hit.hand] = Some(i);
+        } else {
+            self.hover_controls[hit.hand] = None;
+        }
+    }
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
+        let idx = if pressed {
+            self.interactive_get_idx(hit.uv)
+        } else {
+            self.pressed_controls[hit.hand]
+        };
+
+        if let Some(idx) = idx {
+            let c = &mut self.controls[idx];
+            if pressed {
+                if let Some(ref mut f) = c.on_press {
+                    self.pressed_controls[hit.hand] = Some(idx);
+                    f(c, &mut self.canvas.data, app);
+                }
+            } else if let Some(ref mut f) = c.on_release {
+                self.pressed_controls[hit.hand] = None;
+                f(c, &mut self.canvas.data, app);
+            }
+        }
+    }
+    fn on_scroll(&mut self, _app: &mut AppState, _hit: &PointerHit, _delta: f32) {}
+}
+
+impl<D, S> OverlayRenderer for Canvas<D, S> {
+    fn init(&mut self, app: &mut AppState) {
+        self.render_bg(app);
+
+        self.render_fg(app);
+    }
+    fn pause(&mut self, _app: &mut AppState) {}
+    fn resume(&mut self, _app: &mut AppState) {}
+    fn render(&mut self, app: &mut AppState) {
+        let mut dirty = false;
+
+        for c in self.controls.iter_mut() {
+            if let Some(fun) = c.on_update {
+                fun(c, &mut self.canvas.data, app);
+            }
+            if c.dirty {
+                dirty = true;
+                c.dirty = false;
+            }
+        }
+
+        let mut cmd_buffer = self
+            .canvas
+            .graphics
+            .create_command_buffer(CommandBufferUsage::OneTimeSubmit)
+            .begin(self.view_final.clone());
+
+        if dirty {
+            self.render_fg(app);
+        }
+
+        // static background
+        cmd_buffer.run_ref(&self.pass_bg);
+
+        for (i, c) in self.controls.iter_mut().enumerate() {
+            if let Some(render) = c.on_render_hl {
+                if let Some(test) = c.test_highlight {
+                    if test(c, &mut self.canvas.data, app) {
+                        render(c, &self.canvas, app, &mut cmd_buffer, true);
+                    }
+                }
+                if self.hover_controls.contains(&Some(i)) {
+                    render(c, &self.canvas, app, &mut cmd_buffer, false);
+                }
+            }
+        }
+
+        // mostly static text
+        cmd_buffer.run_ref(&self.pass_fg);
+
+        let _ = cmd_buffer.end_render_and_execute();
+    }
+    fn view(&mut self) -> Arc<dyn vulkano::image::ImageViewAbstract> {
+        self.view_final.clone()
+    }
+}
+
+impl<D, S> OverlayBackend for Canvas<D, S> {}
+
+pub struct Control<D, S> {
+    pub state: Option<S>,
+    rect: Rect,
+    fg_color: Vec3,
+    bg_color: Vec3,
+    text: Arc<str>,
+    size: isize,
+    dirty: bool,
+    pass_hl: Option<(WlxPass, WlxPass)>,
+
+    pub on_update: Option<fn(&mut Self, &mut D, &mut AppState)>,
+    pub on_press: Option<fn(&mut Self, &mut D, &mut AppState)>,
+    pub on_release: Option<fn(&mut Self, &mut D, &mut AppState)>,
+    pub test_highlight: Option<fn(&Self, &mut D, &mut AppState) -> bool>,
+
+    on_render_bg: Option<
+        fn(&Self, &CanvasData<D>, &mut AppState, &mut WlxCommandBuffer<PrimaryAutoCommandBuffer>),
+    >,
+    on_render_hl: Option<
+        fn(
+            &Self,
+            &CanvasData<D>,
+            &mut AppState,
+            &mut WlxCommandBuffer<PrimaryAutoCommandBuffer>,
+            bool,
+        ),
+    >,
+    on_render_fg: Option<
+        fn(&Self, &CanvasData<D>, &mut AppState, &mut WlxCommandBuffer<PrimaryAutoCommandBuffer>),
+    >,
+}
+
+impl<D, S> Control<D, S> {
+    fn new() -> Self {
+        Self {
+            rect: Rect {
+                x: 0.,
+                y: 0.,
+                w: 0.,
+                h: 0.,
+            },
+            fg_color: Vec3::ONE,
+            bg_color: Vec3::ZERO,
+            text: Arc::from(""),
+            dirty: false,
+            size: 24,
+            state: None,
+            on_update: None,
+            on_render_bg: None,
+            on_render_hl: None,
+            on_render_fg: None,
+            test_highlight: None,
+            on_press: None,
+            on_release: None,
+            pass_hl: None,
+        }
+    }
+
+    #[inline(always)]
+    pub fn set_text(&mut self, text: &str) {
+        if *self.text == *text {
+            return;
+        }
+        self.text = text.into();
+        self.dirty = true;
+    }
+
+    #[inline(always)]
+    pub fn get_text(&self) -> &str {
+        &self.text
+    }
+
+    fn render_rect(
+        &self,
+        canvas: &CanvasData<D>,
+        _: &mut AppState,
+        cmd_buffer: &mut WlxCommandBuffer<PrimaryAutoCommandBuffer>,
+    ) {
+        let pass = {
+            let vertex_buffer = canvas.graphics.upload_verts(
+                canvas.width as _,
+                canvas.height as _,
+                self.rect.x,
+                self.rect.y,
+                self.rect.w,
+                self.rect.h,
+            );
+            let set0 = canvas.pipeline_color.uniform_buffer(
+                0,
+                vec![self.bg_color.x, self.bg_color.y, self.bg_color.z, 1.],
+            );
+            canvas.pipeline_color.create_pass(
+                [canvas.width as _, canvas.height as _],
+                vertex_buffer,
+                canvas.graphics.quad_indices.clone(),
+                vec![set0],
+            )
+        };
+
+        cmd_buffer.run_ref(&pass);
+    }
+
+    fn render_highlight(
+        &self,
+        canvas: &CanvasData<D>,
+        _: &mut AppState,
+        cmd_buffer: &mut WlxCommandBuffer<PrimaryAutoCommandBuffer>,
+        strong: bool,
+    ) {
+        let vertex_buffer = canvas.graphics.upload_verts(
+            canvas.width as _,
+            canvas.height as _,
+            self.rect.x,
+            self.rect.y,
+            self.rect.w,
+            self.rect.h,
+        );
+        let set0 = canvas.pipeline_color.uniform_buffer(
+            0,
+            vec![
+                self.bg_color.x,
+                self.bg_color.y,
+                self.bg_color.z,
+                if strong { 0.5 } else { 0.3 },
+            ],
+        );
+        let pass = canvas.pipeline_color.create_pass(
+            [canvas.width as _, canvas.height as _],
+            vertex_buffer.clone(),
+            canvas.graphics.quad_indices.clone(),
+            vec![set0],
+        );
+
+        cmd_buffer.run_ref(&pass);
+    }
+
+    fn render_text(
+        &self,
+        canvas: &CanvasData<D>,
+        app: &mut AppState,
+        cmd_buffer: &mut WlxCommandBuffer<PrimaryAutoCommandBuffer>,
+    ) {
+        let mut cur_y = self.rect.y;
+        for line in self.text.lines() {
+            let mut cur_x = self.rect.x;
+            for glyph in app.fc.get_glyphs(line, self.size, canvas.graphics.clone()) {
+                if let Some(tex) = glyph.tex.clone() {
+                    let vertex_buffer = canvas.graphics.upload_verts(
+                        canvas.width as _,
+                        canvas.height as _,
+                        cur_x + glyph.left,
+                        cur_y - glyph.top,
+                        glyph.width,
+                        glyph.height,
+                    );
+                    let set0 = canvas.pipeline_glyph.uniform_sampler(
+                        0,
+                        ImageView::new_default(tex).unwrap(),
+                        Filter::Nearest,
+                    );
+                    let set1 = canvas.pipeline_glyph.uniform_buffer(
+                        1,
+                        vec![self.fg_color.x, self.fg_color.y, self.fg_color.z, 1.],
+                    );
+                    let pass = canvas.pipeline_glyph.create_pass(
+                        [canvas.width as _, canvas.height as _],
+                        vertex_buffer,
+                        canvas.graphics.quad_indices.clone(),
+                        vec![set0, set1],
+                    );
+                    cmd_buffer.run_ref(&pass);
+                }
+                cur_x += glyph.advance;
+            }
+            cur_y += (self.size as f32) * 1.5;
+        }
+    }
+    fn render_text_centered(
+        &self,
+        canvas: &CanvasData<D>,
+        app: &mut AppState,
+        cmd_buffer: &mut WlxCommandBuffer<PrimaryAutoCommandBuffer>,
+    ) {
+        let (w, h) = app
+            .fc
+            .get_text_size(&self.text, self.size, canvas.graphics.clone());
+
+        let mut cur_y = self.rect.y + (self.rect.h) - (h * 0.5);
+        for line in self.text.lines() {
+            let mut cur_x = self.rect.x + (self.rect.w * 0.5) - (w * 0.5);
+            for glyph in app.fc.get_glyphs(line, self.size, canvas.graphics.clone()) {
+                if let Some(tex) = glyph.tex.clone() {
+                    let vertex_buffer = canvas.graphics.upload_verts(
+                        canvas.width as _,
+                        canvas.height as _,
+                        cur_x + glyph.left,
+                        cur_y - glyph.top,
+                        glyph.width,
+                        glyph.height,
+                    );
+                    let set0 = canvas.pipeline_glyph.uniform_sampler(
+                        0,
+                        ImageView::new_default(tex).unwrap(),
+                        Filter::Nearest,
+                    );
+                    let set1 = canvas.pipeline_glyph.uniform_buffer(
+                        1,
+                        vec![self.fg_color.x, self.fg_color.y, self.fg_color.z, 1.],
+                    );
+                    let pass = canvas.pipeline_glyph.create_pass(
+                        [canvas.width as _, canvas.height as _],
+                        vertex_buffer,
+                        canvas.graphics.quad_indices.clone(),
+                        vec![set0, set1],
+                    );
+                    cmd_buffer.run_ref(&pass);
+                }
+                cur_x += glyph.advance;
+            }
+            cur_y += (self.size as f32) * 1.5;
+        }
+    }
+}

src/input.rs

@@ -0,0 +1,431 @@
+use glam::Vec2;
+use idmap::{idmap, IdMap};
+use idmap_derive::IntegerId;
+use input_linux::{
+    AbsoluteAxis, AbsoluteInfo, AbsoluteInfoSetup, EventKind, InputId, Key, RelativeAxis,
+    UInputHandle,
+};
+use libc::{input_event, timeval};
+use log::{error, info};
+use once_cell::sync::Lazy;
+use std::fs::File;
+use std::mem::transmute;
+use strum::{EnumIter, EnumString, IntoEnumIterator};
+
+pub fn initialize_input() -> Box<dyn InputProvider> {
+    if let Some(uinput) = UInputProvider::try_new() {
+        info!("Initialized uinput.");
+        return Box::new(uinput);
+    }
+    error!("Could not create uinput provider. Keyboard/Mouse input will not work!");
+    error!("Check if you're in `input` group: `id -nG`");
+    Box::new(DummyProvider {})
+}
+
+pub trait InputProvider {
+    fn mouse_move(&mut self, pos: Vec2);
+    fn send_button(&self, button: u16, down: bool);
+    fn wheel(&self, delta: i32);
+    fn set_modifiers(&mut self, mods: u8);
+    fn send_key(&self, key: u16, down: bool);
+    fn set_desktop_extent(&mut self, extent: Vec2);
+    fn on_new_frame(&mut self);
+}
+
+pub struct UInputProvider {
+    handle: UInputHandle<File>,
+    desktop_extent: Vec2,
+    mouse_moved: bool,
+    cur_modifiers: u8,
+}
+
+pub struct DummyProvider;
+
+pub const MOUSE_LEFT: u16 = 0x110;
+pub const MOUSE_RIGHT: u16 = 0x111;
+pub const MOUSE_MIDDLE: u16 = 0x112;
+
+const MOUSE_EXTENT: f32 = 32768.;
+
+const EV_SYN: u16 = 0x0;
+const EV_KEY: u16 = 0x1;
+const EV_REL: u16 = 0x2;
+const EV_ABS: u16 = 0x3;
+
+impl UInputProvider {
+    fn try_new() -> Option<Self> {
+        if let Ok(file) = File::create("/dev/uinput") {
+            let handle = UInputHandle::new(file);
+
+            let id = InputId {
+                bustype: 0x03,
+                vendor: 0x4711,
+                product: 0x0819,
+                version: 5,
+            };
+
+            let name = b"WlxOverlay Keyboard-Mouse Hybrid Thing\0";
+
+            let abs_info = vec![
+                AbsoluteInfoSetup {
+                    axis: input_linux::AbsoluteAxis::X,
+                    info: AbsoluteInfo {
+                        value: 0,
+                        minimum: 0,
+                        maximum: MOUSE_EXTENT as _,
+                        fuzz: 0,
+                        flat: 0,
+                        resolution: 10,
+                    },
+                },
+                AbsoluteInfoSetup {
+                    axis: input_linux::AbsoluteAxis::Y,
+                    info: AbsoluteInfo {
+                        value: 0,
+                        minimum: 0,
+                        maximum: MOUSE_EXTENT as _,
+                        fuzz: 0,
+                        flat: 0,
+                        resolution: 10,
+                    },
+                },
+            ];
+
+            if handle.set_evbit(EventKind::Key).is_err() {
+                return None;
+            }
+            if handle.set_evbit(EventKind::Absolute).is_err() {
+                return None;
+            }
+            if handle.set_evbit(EventKind::Relative).is_err() {
+                return None;
+            }
+
+            for btn in MOUSE_LEFT..=MOUSE_MIDDLE {
+                let key: Key = unsafe { transmute(btn) };
+                if handle.set_keybit(key).is_err() {
+                    return None;
+                }
+            }
+
+            for key in VirtualKey::iter() {
+                let key: Key = unsafe { transmute(key as u16) };
+                if handle.set_keybit(key).is_err() {
+                    return None;
+                }
+            }
+
+            if handle.set_absbit(AbsoluteAxis::X).is_err() {
+                return None;
+            }
+            if handle.set_absbit(AbsoluteAxis::Y).is_err() {
+                return None;
+            }
+            if handle.set_relbit(RelativeAxis::Wheel).is_err() {
+                return None;
+            }
+
+            if handle.create(&id, name, 0, &abs_info).is_ok() {
+                return Some(UInputProvider {
+                    handle,
+                    desktop_extent: Vec2::ZERO,
+                    mouse_moved: false,
+                    cur_modifiers: 0,
+                });
+            }
+        }
+        None
+    }
+}
+
+impl InputProvider for UInputProvider {
+    fn mouse_move(&mut self, pos: Vec2) {
+        if self.mouse_moved {
+            return;
+        }
+        self.mouse_moved = true;
+
+        let pos = pos * (MOUSE_EXTENT / self.desktop_extent);
+
+        let time = get_time();
+        let events = [
+            new_event(time, EV_ABS, AbsoluteAxis::X as _, pos.x as i32),
+            new_event(time, EV_ABS, AbsoluteAxis::Y as _, pos.y as i32),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.handle.write(&events) {
+            error!("{}", res.to_string());
+        }
+    }
+    fn send_button(&self, button: u16, down: bool) {
+        let time = get_time();
+        let events = [
+            new_event(time, EV_KEY, button, down as _),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.handle.write(&events) {
+            error!("{}", res.to_string());
+        }
+    }
+    fn wheel(&self, delta: i32) {
+        let time = get_time();
+        let events = [
+            new_event(time, EV_REL, RelativeAxis::Wheel as _, delta),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.handle.write(&events) {
+            error!("{}", res.to_string());
+        }
+    }
+    fn set_modifiers(&mut self, modifiers: u8) {
+        let changed = self.cur_modifiers ^ modifiers;
+        for i in 0..7 {
+            let m = 1 << i;
+            if changed & m != 0 {
+                let vk = MODS_TO_KEYS.get(m).unwrap()[0] as u16;
+                self.send_key(vk, modifiers & m != 0);
+            }
+        }
+        self.cur_modifiers = modifiers;
+    }
+    fn send_key(&self, key: u16, down: bool) {
+        let time = get_time();
+        let events = [
+            new_event(time, EV_KEY, key - 8, down as _),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.handle.write(&events) {
+            error!("{}", res.to_string());
+        }
+    }
+    fn set_desktop_extent(&mut self, extent: Vec2) {
+        info!("Desktop extent: {:?}", extent);
+        self.desktop_extent = extent;
+    }
+    fn on_new_frame(&mut self) {
+        self.mouse_moved = false;
+    }
+}
+
+impl InputProvider for DummyProvider {
+    fn mouse_move(&mut self, _pos: Vec2) {}
+    fn send_button(&self, _button: u16, _down: bool) {}
+    fn wheel(&self, _delta: i32) {}
+    fn set_modifiers(&mut self, _modifiers: u8) {}
+    fn send_key(&self, _key: u16, _down: bool) {}
+    fn set_desktop_extent(&mut self, _extent: Vec2) {}
+    fn on_new_frame(&mut self) {}
+}
+
+#[inline]
+fn get_time() -> timeval {
+    let mut time = timeval {
+        tv_sec: 0,
+        tv_usec: 0,
+    };
+    unsafe { libc::gettimeofday(&mut time, std::ptr::null_mut()) };
+    time
+}
+
+#[inline]
+fn new_event(time: timeval, type_: u16, code: u16, value: i32) -> input_event {
+    input_event {
+        time,
+        type_,
+        code,
+        value,
+    }
+}
+
+pub type KeyModifier = u8;
+pub const SHIFT: KeyModifier = 0x01;
+pub const CAPS_LOCK: KeyModifier = 0x02;
+pub const CTRL: KeyModifier = 0x04;
+pub const ALT: KeyModifier = 0x08;
+pub const NUM_LOCK: KeyModifier = 0x10;
+pub const SUPER: KeyModifier = 0x40;
+pub const META: KeyModifier = 0x80;
+
+#[allow(non_camel_case_types)]
+#[derive(Debug, PartialEq, Clone, Copy, IntegerId, EnumString, EnumIter)]
+pub enum VirtualKey {
+    Escape = 9,
+    N1, // number row
+    N2,
+    N3,
+    N4,
+    N5,
+    N6,
+    N7,
+    N8,
+    N9,
+    N0,
+    Minus,
+    Plus,
+    BackSpace,
+    Tab,
+    Q,
+    W,
+    E,
+    R,
+    T,
+    Y,
+    U,
+    I,
+    O,
+    P,
+    Oem4, // [ {
+    Oem6, // ] }
+    Return,
+    LCtrl,
+    A,
+    S,
+    D,
+    F,
+    G,
+    H,
+    J,
+    K,
+    L,
+    Oem1, // ; :
+    Oem7, // ' "
+    Oem3, // ` ~
+    LShift,
+    Oem5, // \ |
+    Z,
+    X,
+    C,
+    V,
+    B,
+    N,
+    M,
+    Comma,  // , <
+    Period, // . >
+    Oem2,   // / ?
+    RShift,
+    KP_Multiply,
+    LAlt,
+    Space,
+    Caps,
+    F1,
+    F2,
+    F3,
+    F4,
+    F5,
+    F6,
+    F7,
+    F8,
+    F9,
+    F10,
+    NumLock,
+    Scroll,
+    KP_7, // KeyPad
+    KP_8,
+    KP_9,
+    KP_Subtract,
+    KP_4,
+    KP_5,
+    KP_6,
+    KP_Add,
+    KP_1,
+    KP_2,
+    KP_3,
+    KP_0,
+    KP_Decimal,
+    Oem102 = 94, // Optional key usually between LShift and Z
+    F11,
+    F12,
+    AbntC1,
+    Katakana,
+    Hiragana,
+    Henkan,
+    Kana,
+    Muhenkan,
+    KP_Enter = 104,
+    RCtrl,
+    KP_Divide,
+    Print,
+    Meta, // Right Alt aka AltGr
+    Home = 110,
+    Up,
+    Prior,
+    Left,
+    Right,
+    End,
+    Down,
+    Next,
+    Insert,
+    Delete,
+    XF86AudioMute = 121,
+    XF86AudioLowerVolume,
+    XF86AudioRaiseVolume,
+    Pause = 127,
+    AbntC2 = 129,
+    Hangul,
+    Hanja,
+    LSuper = 133,
+    RSuper,
+    Menu,
+    Help = 146,
+    XF86MenuKB,
+    XF86Sleep = 150,
+    XF86Xfer = 155,
+    XF86Launch1,
+    XF86Launch2,
+    XF86WWW,
+    XF86Mail = 163,
+    XF86Favorites,
+    XF86MyComputer,
+    XF86Back,
+    XF86Forward,
+    XF86AudioNext = 171,
+    XF86AudioPlay,
+    XF86AudioPrev,
+    XF86AudioStop,
+    XF86HomePage = 180,
+    XF86Reload,
+    F13 = 191,
+    F14,
+    F15,
+    F16,
+    F17,
+    F18,
+    F19,
+    F20,
+    F21,
+    F22,
+    F23,
+    F24,
+    Hyper = 207,
+    XF86Launch3,
+    XF86Launch4,
+    XF86LaunchB,
+    XF86Search = 225,
+}
+
+pub static KEYS_TO_MODS: Lazy<IdMap<VirtualKey, KeyModifier>> = Lazy::new(|| {
+    idmap! {
+        VirtualKey::LShift => SHIFT,
+        VirtualKey::RShift => SHIFT,
+        VirtualKey::Caps => CAPS_LOCK,
+        VirtualKey::LCtrl => CTRL,
+        VirtualKey::RCtrl => CTRL,
+        VirtualKey::LAlt => ALT,
+        VirtualKey::NumLock => NUM_LOCK,
+        VirtualKey::LSuper => SUPER,
+        VirtualKey::RSuper => SUPER,
+        VirtualKey::Meta => META,
+    }
+});
+
+pub static MODS_TO_KEYS: Lazy<IdMap<KeyModifier, Vec<VirtualKey>>> = Lazy::new(|| {
+    idmap! {
+        SHIFT => vec![VirtualKey::LShift, VirtualKey::RShift],
+        CAPS_LOCK => vec![VirtualKey::Caps],
+        CTRL => vec![VirtualKey::LCtrl, VirtualKey::RCtrl],
+        ALT => vec![VirtualKey::LAlt],
+        NUM_LOCK => vec![VirtualKey::NumLock],
+        SUPER => vec![VirtualKey::LSuper, VirtualKey::RSuper],
+        META => vec![VirtualKey::Meta],
+    }
+});

src/main.rs

@@ -0,0 +1,271 @@
+#[allow(dead_code)]
+mod backend;
+mod graphics;
+mod gui;
+mod input;
+mod overlays;
+mod ovr;
+mod shaders;
+mod state;
+
+use std::collections::VecDeque;
+use std::sync::Arc;
+
+use crate::graphics::{Vert2Uv, WlxGraphics, INDICES};
+use crate::input::initialize_input;
+use crate::overlays::watch::create_watch;
+use crate::{
+    shaders::{frag_sprite, vert_common},
+    state::AppState,
+};
+use env_logger::Env;
+use log::{info, warn};
+use vulkano::{
+    buffer::BufferUsage,
+    command_buffer::CommandBufferUsage,
+    image::{
+        view::{ImageView, ImageViewCreateInfo},
+        ImageAccess, ImageSubresourceRange, ImageViewType, SwapchainImage,
+    },
+    pipeline::graphics::viewport::Viewport,
+    sampler::Filter,
+    swapchain::{
+        acquire_next_image, AcquireError, SwapchainCreateInfo, SwapchainCreationError,
+        SwapchainPresentInfo,
+    },
+    sync::{self, FlushError, GpuFuture},
+};
+use winit::{
+    event::{Event, WindowEvent},
+    event_loop::ControlFlow,
+    window::Window,
+};
+use wlx_capture::{frame::WlxFrame, wayland::WlxClient, wlr::WlrDmabufCapture, WlxCapture};
+
+fn main() {
+    env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
+    info!(
+        "Welcome to {} version {}!",
+        env!("CARGO_PKG_NAME"),
+        env!("CARGO_PKG_VERSION")
+    );
+
+    let (graphics, event_loop) = WlxGraphics::new();
+    let (mut swapchain, images) = graphics.create_swapchain(None);
+
+    let mut app = AppState {
+        fc: crate::gui::font::FontCache::new(),
+        session: crate::state::AppSession::load(),
+        tasks: VecDeque::with_capacity(16),
+        graphics: graphics.clone(),
+        format: swapchain.image_format(),
+        input: initialize_input(),
+    };
+
+    let wl = WlxClient::new().unwrap();
+    let output_id = wl.outputs[0].id;
+    let mut capture = WlrDmabufCapture::new(wl, output_id).unwrap();
+    let rx = capture.init();
+
+    let vertices = [
+        Vert2Uv {
+            in_pos: [0., 0.],
+            in_uv: [0., 0.],
+        },
+        Vert2Uv {
+            in_pos: [0., 1.],
+            in_uv: [0., 1.],
+        },
+        Vert2Uv {
+            in_pos: [1., 0.],
+            in_uv: [1., 0.],
+        },
+        Vert2Uv {
+            in_pos: [1., 1.],
+            in_uv: [1., 1.],
+        },
+    ];
+
+    let vertex_buffer = graphics.upload_buffer(BufferUsage::VERTEX_BUFFER, vertices.iter());
+    let index_buffer = graphics.upload_buffer(BufferUsage::INDEX_BUFFER, INDICES.iter());
+
+    let vs = vert_common::load(graphics.device.clone()).unwrap();
+    let fs = frag_sprite::load(graphics.device.clone()).unwrap();
+
+    let uploads = graphics.create_command_buffer(CommandBufferUsage::OneTimeSubmit);
+
+    let mut watch = create_watch(&app, vec![]);
+    watch.init(&mut app);
+    watch.render(&mut app);
+
+    let pipeline1 = graphics.create_pipeline(vs.clone(), fs.clone(), swapchain.image_format());
+    let set1 = pipeline1.uniform_sampler(0, watch.view(), Filter::Nearest);
+
+    capture.request_new_frame();
+
+    let pipeline = graphics.create_pipeline(vs, fs, swapchain.image_format());
+    let set0;
+    loop {
+        if let Ok(frame) = rx.try_recv() {
+            match frame {
+                WlxFrame::Dmabuf(dmabuf_frame) => match graphics.dmabuf_texture(dmabuf_frame) {
+                    Ok(tex) => {
+                        let format = tex.format();
+                        let view = ImageView::new(
+                            tex,
+                            ImageViewCreateInfo {
+                                format: Some(format),
+                                view_type: ImageViewType::Dim2d,
+                                subresource_range: ImageSubresourceRange::from_parameters(
+                                    format, 1, 1,
+                                ),
+                                ..Default::default()
+                            },
+                        )
+                        .unwrap();
+                        set0 = pipeline.uniform_sampler(0, view, Filter::Nearest);
+                        break;
+                    }
+                    Err(e) => {
+                        warn!("Failed to create texture from dmabuf: {}", e);
+                    }
+                },
+                _ => {
+                    warn!("Received non-dmabuf frame");
+                }
+            }
+        }
+    }
+
+    //let set1 = graphics.uniform_buffer(1, vec![1.0, 1.0, 1.0, 1.0]);
+    let image_extent_f32 = [
+        swapchain.image_extent()[0] as f32,
+        swapchain.image_extent()[1] as f32,
+    ];
+    let image_extent2_f32 = [
+        swapchain.image_extent()[0] as f32 / 2.,
+        swapchain.image_extent()[1] as f32 / 2.,
+    ];
+    let pass = pipeline.create_pass(
+        image_extent_f32,
+        vertex_buffer.clone(),
+        index_buffer.clone(),
+        vec![set0],
+    );
+    let pass2 = pipeline1.create_pass(image_extent2_f32, vertex_buffer, index_buffer, vec![set1]);
+
+    let mut viewport = Viewport {
+        origin: [0.0, 0.0],
+        dimensions: [1024.0, 1024.0],
+        depth_range: 0.0..1.0,
+    };
+
+    let mut attachment_image_views = window_size_dependent_setup(&images, &mut viewport);
+
+    //let set1 = pipeline.uniform_buffer(1, vec![1.0, 0.0, 1.0, 1.0]);
+
+    let mut recreate_swapchain = false;
+    let mut previous_frame_end = //Some(sync::now(graphics.device.clone()).boxed());
+        Some(uploads.end_and_execute().boxed());
+
+    event_loop.run(move |event, _, control_flow| match event {
+        Event::WindowEvent {
+            event: WindowEvent::CloseRequested,
+            ..
+        } => {
+            *control_flow = ControlFlow::Exit;
+        }
+        Event::WindowEvent {
+            event: WindowEvent::Resized(_),
+            ..
+        } => {
+            recreate_swapchain = true;
+        }
+        Event::RedrawEventsCleared => {
+            previous_frame_end.as_mut().unwrap().cleanup_finished();
+
+            if recreate_swapchain {
+                let window = graphics
+                    .surface
+                    .object()
+                    .unwrap()
+                    .downcast_ref::<Window>()
+                    .unwrap();
+                let (new_swapchain, new_images) = match swapchain.recreate(SwapchainCreateInfo {
+                    image_extent: window.inner_size().into(),
+                    ..swapchain.create_info()
+                }) {
+                    Ok(r) => r,
+                    Err(SwapchainCreationError::ImageExtentNotSupported { .. }) => return,
+                    Err(e) => panic!("failed to recreate swapchain: {e}"),
+                };
+
+                swapchain = new_swapchain;
+                attachment_image_views = window_size_dependent_setup(&new_images, &mut viewport);
+                recreate_swapchain = false;
+            }
+
+            let (image_index, suboptimal, acquire_future) =
+                match acquire_next_image(swapchain.clone(), None) {
+                    Ok(r) => r,
+                    Err(AcquireError::OutOfDate) => {
+                        recreate_swapchain = true;
+                        return;
+                    }
+                    Err(e) => panic!("failed to acquire next image: {e}"),
+                };
+
+            if suboptimal {
+                recreate_swapchain = true;
+            }
+
+            let cmd = graphics
+                .create_command_buffer(CommandBufferUsage::OneTimeSubmit)
+                .begin(attachment_image_views[image_index as usize].clone())
+                .run(&pass)
+                .run(&pass2)
+                .end_render();
+
+            let future = previous_frame_end
+                .take()
+                .unwrap()
+                .join(acquire_future)
+                .then_execute(graphics.queue.clone(), cmd)
+                .unwrap()
+                .then_swapchain_present(
+                    graphics.queue.clone(),
+                    SwapchainPresentInfo::swapchain_image_index(swapchain.clone(), image_index),
+                )
+                .then_signal_fence_and_flush();
+
+            match future {
+                Ok(future) => {
+                    previous_frame_end = Some(future.boxed());
+                }
+                Err(FlushError::OutOfDate) => {
+                    recreate_swapchain = true;
+                    previous_frame_end = Some(sync::now(graphics.device.clone()).boxed());
+                }
+                Err(e) => {
+                    println!("failed to flush future: {e}");
+                    previous_frame_end = Some(sync::now(graphics.device.clone()).boxed());
+                }
+            }
+        }
+        _ => (),
+    });
+}
+
+/// This function is called once during initialization, then again whenever the window is resized.
+fn window_size_dependent_setup(
+    images: &[Arc<SwapchainImage>],
+    viewport: &mut Viewport,
+) -> Vec<Arc<ImageView<SwapchainImage>>> {
+    let dimensions = images[0].dimensions().width_height();
+    viewport.dimensions = [dimensions[0] as f32, dimensions[1] as f32];
+
+    images
+        .iter()
+        .map(|image| ImageView::new_default(image.clone()).unwrap())
+        .collect::<Vec<_>>()
+}

src/main.rs.bak2

@@ -0,0 +1,159 @@
+#[allow(dead_code)]
+#[allow(unused_imports)]
+
+mod graphics;
+//mod gui;
+//mod interactions;
+//mod overlay;
+//mod state;
+
+use core::slice;
+
+use ash::vk;
+use env_logger::Env;
+use glam::f32::Vec4;
+use log::{info, warn};
+use wlx_capture::{wlr::WlrDmabufCapture, wayland::WlxClient, WlxCapture, frame::WlxFrame};
+
+use crate::{graphics::{VkGraphics, VkDescriptor}};
+
+fn main() {
+    env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
+    info!("Welcome to {} version {}!", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
+
+//    let mut app = AppState {
+//        fc: gui::font::FontCache::new(),
+//        session: state::AppSession::load(),
+//    };
+
+    let wl = WlxClient::new().unwrap();
+    let output_id = wl.outputs[0].id;
+    let mut capture = WlrDmabufCapture::new(wl, output_id).unwrap();
+    let rx = capture.init();
+
+    let gfx = VkGraphics::new();
+    let vert = gfx.create_shader(include_bytes!("shaders/vert-common.spv"), None);
+    let frag_sprite = gfx.create_shader(
+        include_bytes!("shaders/frag-sprite.spv"), 
+        Some(vec![vk::DescriptorType::COMBINED_IMAGE_SAMPLER])
+    );
+    let frag_srgb = gfx.create_shader(
+        include_bytes!("shaders/frag-srgb.spv"),
+        Some(vec![vk::DescriptorType::COMBINED_IMAGE_SAMPLER])
+    );
+    let frag_glyph = gfx.create_shader(
+        include_bytes!("shaders/frag-glyph.spv"), 
+        Some(vec![vk::DescriptorType::COMBINED_IMAGE_SAMPLER, vk::DescriptorType::UNIFORM_BUFFER])
+    );
+    let frag_color = gfx.create_shader(
+        include_bytes!("shaders/frag-color.spv"), 
+        Some(vec![vk::DescriptorType::UNIFORM_BUFFER])
+    );
+
+    let pipeline = gfx.create_pipeline(vert, frag_color, gfx.swapchain_format);
+
+    let color_buf = gfx.create_buffer(vk::BufferUsageFlags::UNIFORM_BUFFER, &[Vec4::new(1.0, 0.0, 1.0, 1.0); 16]);
+
+    gfx.render_loop(|| {
+        let frame = gfx.create_frame();
+
+        unsafe { gfx.device.reset_command_pool(frame.command_pool, vk::CommandPoolResetFlags::RELEASE_RESOURCES) }.unwrap();
+
+        let (present_index, _) = unsafe {
+            gfx.swapchain_loader.acquire_next_image(
+                gfx.swapchain, 
+                std::u64::MAX, 
+                frame.present_semaphore, 
+                vk::Fence::null()
+            ).unwrap()
+        };
+        
+        let command_buffer_begin_info = vk::CommandBufferBeginInfo::builder()
+            .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT)
+            .build();
+
+        unsafe { gfx.device.begin_command_buffer(frame.command_buffer, &command_buffer_begin_info) }.unwrap();
+
+        let image = gfx.present_images[present_index as usize];
+
+        let image_memory_barrier = vk::ImageMemoryBarrier2::builder()
+        .src_stage_mask(vk::PipelineStageFlags2::TOP_OF_PIPE)
+        .dst_stage_mask(vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT)
+        .dst_access_mask(vk::AccessFlags2::COLOR_ATTACHMENT_WRITE)
+        .old_layout(vk::ImageLayout::UNDEFINED)
+        .new_layout(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL)
+        .image(image)
+        .subresource_range(vk::ImageSubresourceRange::builder().aspect_mask(vk::ImageAspectFlags::COLOR).level_count(1).layer_count(1).build())
+        .build();
+
+        unsafe { gfx.device.cmd_pipeline_barrier2(frame.command_buffer, &vk::DependencyInfo::builder().image_memory_barriers(slice::from_ref(&image_memory_barrier)).build()) };
+        
+        let color_attachment = vk::RenderingAttachmentInfo::builder()
+            .image_view(gfx.present_image_views[present_index as usize])
+            .image_layout(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL)
+            .load_op(vk::AttachmentLoadOp::CLEAR)
+            .store_op(vk::AttachmentStoreOp::STORE)
+            .clear_value(vk::ClearValue {
+                color: vk::ClearColorValue {
+                    float32: [0.0, 0.0, 0.0, 1.0]
+                }
+            });
+
+        let rendering_info = vk::RenderingInfo::builder()
+            .render_area(vk::Rect2D::builder().extent(vk::Extent2D::builder().width(1600).height(900).build()).build())
+            .layer_count(1)
+            .color_attachments(slice::from_ref(&color_attachment))
+            .build();
+
+        
+        unsafe { gfx.device.cmd_begin_rendering(frame.command_buffer, &rendering_info) };
+
+        let descriptor = VkDescriptor::Buffer(color_buf.clone());
+        pipeline.bind_descriptors(slice::from_ref(&descriptor));
+        pipeline.render(&frame, vk::Rect2D {
+            offset: vk::Offset2D { x: 0, y: 0 },
+            extent: vk::Extent2D { width: 1600, height: 900 }
+        });
+
+        unsafe { gfx.device.cmd_end_rendering(frame.command_buffer) };
+
+
+        let image_memory_barrier = vk::ImageMemoryBarrier2::builder()
+            .src_stage_mask(vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT)
+            .src_access_mask(vk::AccessFlags2::COLOR_ATTACHMENT_WRITE)
+            .dst_stage_mask(vk::PipelineStageFlags2::BOTTOM_OF_PIPE)
+            .old_layout(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL)
+            .new_layout(vk::ImageLayout::PRESENT_SRC_KHR)
+            .image(image)
+            .subresource_range(vk::ImageSubresourceRange::builder().aspect_mask(vk::ImageAspectFlags::COLOR).level_count(1).layer_count(1).build())
+            .build();
+
+        unsafe {
+            gfx.device.cmd_pipeline_barrier2(frame.command_buffer, &vk::DependencyInfo::builder().image_memory_barriers(slice::from_ref(&image_memory_barrier)).build());
+        };
+
+        unsafe { gfx.device.end_command_buffer(frame.command_buffer) }.unwrap();
+
+        let wait_semaphores = [frame.present_semaphore];
+        let wait_dst_stage_mask = [vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT];
+
+        let submit_info = vk::SubmitInfo::builder()
+            .wait_semaphores(&wait_semaphores)
+            .wait_dst_stage_mask(&wait_dst_stage_mask)
+            .command_buffers(slice::from_ref(&frame.command_buffer))
+            .signal_semaphores(slice::from_ref(&frame.render_semaphore));
+
+        unsafe { gfx.device.queue_submit(gfx.present_queue, slice::from_ref(&submit_info), frame.fence) }.unwrap();
+
+        let present_info = vk::PresentInfoKHR::builder()
+            .wait_semaphores(slice::from_ref(&frame.present_semaphore))
+            .swapchains(slice::from_ref(&gfx.swapchain))
+            .image_indices(slice::from_ref(&present_index))
+            .build();
+
+        unsafe { gfx.swapchain_loader.queue_present(gfx.present_queue, &present_info).unwrap() };
+    });
+
+    unsafe { gfx.device.device_wait_idle().unwrap() };
+}
+

src/overlays/interactions.rs

@@ -0,0 +1,41 @@
+use std::{collections::VecDeque, time::Instant};
+
+use glam::{Affine3A, Vec2, Vec3};
+
+use crate::state::AppState;
+
+pub const HAND_LEFT: usize = 0;
+pub const HAND_RIGHT: usize = 1;
+
+pub const POINTER_NORM: u16 = 0;
+pub const POINTER_SHIFT: u16 = 1;
+pub const POINTER_ALT: u16 = 2;
+
+pub trait InteractionHandler {
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit);
+    fn on_left(&mut self, app: &mut AppState, hand: usize);
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool);
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32);
+}
+
+// --- Dummies & plumbing below ---
+
+impl Default for PointerState {
+    fn default() -> Self {
+        Self {
+            click: false,
+            grab: false,
+            show_hide: false,
+            scroll: 0.,
+        }
+    }
+}
+
+pub struct DummyInteractionHandler;
+
+impl InteractionHandler for DummyInteractionHandler {
+    fn on_left(&mut self, _app: &mut AppState, _hand: usize) {}
+    fn on_hover(&mut self, _app: &mut AppState, _hit: &PointerHit) {}
+    fn on_pointer(&mut self, _app: &mut AppState, _hit: &PointerHit, _pressed: bool) {}
+    fn on_scroll(&mut self, _app: &mut AppState, _hit: &PointerHit, _delta: f32) {}
+}

src/overlays/keyboard.rs

@@ -0,0 +1,354 @@
+use std::{
+    collections::HashMap,
+    env::var,
+    fs,
+    io::Cursor,
+    path::PathBuf,
+    process::{Child, Command},
+    str::FromStr,
+    sync::Arc,
+};
+
+use crate::{
+    gui::{color_parse, CanvasBuilder, Control},
+    input::{KeyModifier, VirtualKey, KEYS_TO_MODS},
+    state::AppState,
+};
+use glam::{vec2, vec3};
+use log::error;
+use once_cell::sync::Lazy;
+use regex::Regex;
+use rodio::{Decoder, OutputStream, Source};
+use serde::{Deserialize, Serialize};
+
+use super::OverlayData;
+
+const PIXELS_PER_UNIT: f32 = 80.;
+const BUTTON_PADDING: f32 = 4.;
+
+pub fn create_keyboard(app: &AppState) -> OverlayData {
+    let size = vec2(
+        LAYOUT.row_size * PIXELS_PER_UNIT,
+        (LAYOUT.main_layout.len() as f32) * PIXELS_PER_UNIT,
+    );
+
+    let data = KeyboardData {
+        modifiers: 0,
+        processes: vec![],
+        audio_stream: None,
+    };
+
+    let mut canvas = CanvasBuilder::new(
+        size.x as _,
+        size.y as _,
+        app.graphics.clone(),
+        app.format,
+        data,
+    );
+
+    canvas.bg_color = color_parse("#101010");
+    canvas.panel(0., 0., size.x, size.y);
+
+    canvas.font_size = 18;
+    canvas.bg_color = color_parse("#202020");
+
+    let unit_size = size.x / LAYOUT.row_size;
+    let h = unit_size - 2. * BUTTON_PADDING;
+
+    for row in 0..LAYOUT.key_sizes.len() {
+        let y = unit_size * (row as f32) + BUTTON_PADDING;
+        let mut sum_size = 0f32;
+
+        for col in 0..LAYOUT.key_sizes[row].len() {
+            let my_size = LAYOUT.key_sizes[row][col];
+            let x = unit_size * sum_size + BUTTON_PADDING;
+            let w = unit_size * my_size - 2. * BUTTON_PADDING;
+
+            if let Some(key) = LAYOUT.main_layout[row][col].as_ref() {
+                let mut maybe_state: Option<KeyButtonData> = None;
+                if let Ok(vk) = VirtualKey::from_str(key) {
+                    if let Some(mods) = KEYS_TO_MODS.get(vk) {
+                        maybe_state = Some(KeyButtonData::Modifier {
+                            modifier: *mods,
+                            sticky: false,
+                            pressed: false,
+                        });
+                    } else {
+                        maybe_state = Some(KeyButtonData::Key { vk, pressed: false });
+                    }
+                } else if let Some(macro_verbs) = LAYOUT.macros.get(key) {
+                    maybe_state = Some(KeyButtonData::Macro {
+                        verbs: key_events_for_macro(macro_verbs),
+                    });
+                } else if let Some(exec_args) = LAYOUT.exec_commands.get(key) {
+                    maybe_state = Some(KeyButtonData::Exec {
+                        program: exec_args.first().unwrap().clone(),
+                        args: exec_args.iter().skip(1).cloned().collect(),
+                    });
+                } else {
+                    error!("Unknown key: {}", key);
+                }
+
+                if let Some(state) = maybe_state {
+                    let label = LAYOUT.label_for_key(key);
+                    let button = canvas.key_button(x, y, w, h, &label);
+                    button.state = Some(state);
+                    button.on_press = Some(key_press);
+                    button.on_release = Some(key_release);
+                    button.test_highlight = Some(test_highlight);
+                }
+            }
+
+            sum_size += my_size;
+        }
+    }
+
+    let canvas = canvas.build();
+
+    OverlayData {
+        name: Arc::from("Kbd"),
+        show_hide: true,
+        width: LAYOUT.row_size * 0.05,
+        size: (size.x as _, size.y as _),
+        grabbable: true,
+        spawn_point: vec3(0., -0.5, -1.),
+        backend: Box::new(canvas),
+        ..Default::default()
+    }
+}
+
+fn key_press(
+    control: &mut Control<KeyboardData, KeyButtonData>,
+    data: &mut KeyboardData,
+    app: &mut AppState,
+) {
+    match control.state.as_mut() {
+        Some(KeyButtonData::Key { vk, pressed }) => {
+            data.key_click();
+            app.input.send_key(*vk as _, true);
+            *pressed = true;
+        }
+        Some(KeyButtonData::Modifier {
+            modifier,
+            sticky,
+            pressed,
+        }) => {
+            *sticky = data.modifiers & *modifier == 0;
+            data.modifiers |= *modifier;
+            data.key_click();
+            app.input.set_modifiers(data.modifiers);
+            *pressed = true;
+        }
+        Some(KeyButtonData::Macro { verbs }) => {
+            data.key_click();
+            for (vk, press) in verbs {
+                app.input.send_key(*vk as _, *press);
+            }
+        }
+        Some(KeyButtonData::Exec { program, args }) => {
+            // Reap previous processes
+            data.processes
+                .retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_))));
+
+            data.key_click();
+            if let Ok(child) = Command::new(program).args(args).spawn() {
+                data.processes.push(child);
+            }
+        }
+        None => {}
+    }
+}
+
+fn key_release(
+    control: &mut Control<KeyboardData, KeyButtonData>,
+    data: &mut KeyboardData,
+    app: &mut AppState,
+) {
+    match control.state.as_mut() {
+        Some(KeyButtonData::Key { vk, pressed }) => {
+            app.input.send_key(*vk as _, false);
+            *pressed = false;
+        }
+        Some(KeyButtonData::Modifier {
+            modifier,
+            sticky,
+            pressed,
+        }) => {
+            if !*sticky {
+                data.modifiers &= !*modifier;
+                app.input.set_modifiers(data.modifiers);
+                *pressed = false;
+            }
+        }
+        _ => {}
+    }
+}
+
+fn test_highlight(
+    control: &Control<KeyboardData, KeyButtonData>,
+    _data: &mut KeyboardData,
+    _app: &mut AppState,
+) -> bool {
+    match control.state.as_ref() {
+        Some(KeyButtonData::Key { pressed, .. }) => *pressed,
+        Some(KeyButtonData::Modifier { pressed, .. }) => *pressed,
+        _ => false,
+    }
+}
+
+struct KeyboardData {
+    modifiers: KeyModifier,
+    processes: Vec<Child>,
+    audio_stream: Option<OutputStream>,
+}
+
+impl KeyboardData {
+    fn key_click(&mut self) {
+        let wav = include_bytes!("../res/421581.wav");
+        let cursor = Cursor::new(wav);
+        let source = Decoder::new_wav(cursor).unwrap();
+        self.audio_stream = None;
+        if let Ok((stream, handle)) = OutputStream::try_default() {
+            let _ = handle.play_raw(source.convert_samples());
+            self.audio_stream = Some(stream);
+        } else {
+            error!("Failed to play key click");
+        }
+    }
+}
+
+enum KeyButtonData {
+    Key {
+        vk: VirtualKey,
+        pressed: bool,
+    },
+    Modifier {
+        modifier: KeyModifier,
+        sticky: bool,
+        pressed: bool,
+    },
+    Macro {
+        verbs: Vec<(VirtualKey, bool)>,
+    },
+    Exec {
+        program: String,
+        args: Vec<String>,
+    },
+}
+
+static KEYBOARD_YAML: Lazy<PathBuf> = Lazy::new(|| {
+    let home = &var("HOME").unwrap();
+    [home, ".config/wlxoverlay/keyboard.yaml"].iter().collect() //TODO other paths
+});
+
+static LAYOUT: Lazy<Layout> = Lazy::new(Layout::load_from_disk);
+
+static MACRO_REGEX: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"^([A-Za-z0-1_-]+)(?: +(UP|DOWN))?$").unwrap());
+
+#[derive(Debug, Deserialize, Serialize)]
+struct Layout {
+    name: String,
+    row_size: f32,
+    key_sizes: Vec<Vec<f32>>,
+    main_layout: Vec<Vec<Option<String>>>,
+    exec_commands: HashMap<String, Vec<String>>,
+    macros: HashMap<String, Vec<String>>,
+    labels: HashMap<String, Vec<String>>,
+}
+
+impl Layout {
+    fn load_from_disk() -> Layout {
+        let mut yaml = fs::read_to_string(KEYBOARD_YAML.as_path()).ok();
+
+        if yaml.is_none() {
+            yaml = Some(include_str!("../res/keyboard.yaml").to_string());
+        }
+
+        let mut layout: Layout =
+            serde_yaml::from_str(&yaml.unwrap()).expect("Failed to parse keyboard.yaml");
+        layout.post_load();
+
+        layout
+    }
+
+    fn post_load(&mut self) {
+        for i in 0..self.key_sizes.len() {
+            let row = &self.key_sizes[i];
+            let width: f32 = row.iter().sum();
+            if (width - self.row_size).abs() > 0.001 {
+                panic!(
+                    "Row {} has a width of {}, but the row size is {}",
+                    i, width, self.row_size
+                );
+            }
+        }
+
+        for i in 0..self.main_layout.len() {
+            let row = &self.main_layout[i];
+            let width = row.len();
+            if width != self.key_sizes[i].len() {
+                panic!(
+                    "Row {} has {} keys, needs to have {} according to key_sizes",
+                    i,
+                    width,
+                    self.key_sizes[i].len()
+                );
+            }
+        }
+    }
+
+    fn label_for_key(&self, key: &str) -> Vec<String> {
+        if let Some(label) = self.labels.get(key) {
+            return label.clone();
+        }
+        if key.is_empty() {
+            return vec![];
+        }
+        if key.len() == 1 {
+            return vec![key.to_string().to_lowercase()];
+        }
+        let mut key = key;
+        if key.starts_with("KP_") {
+            key = &key[3..];
+        }
+        if key.contains('_') {
+            key = key.split('_').next().unwrap();
+        }
+        vec![format!(
+            "{}{}",
+            key.chars().next().unwrap().to_uppercase(),
+            &key[1..].to_lowercase()
+        )]
+    }
+}
+
+fn key_events_for_macro(macro_verbs: &Vec<String>) -> Vec<(VirtualKey, bool)> {
+    let mut key_events = vec![];
+    for verb in macro_verbs {
+        if let Some(caps) = MACRO_REGEX.captures(verb) {
+            if let Ok(virtual_key) = VirtualKey::from_str(&caps[1]) {
+                if let Some(state) = caps.get(2) {
+                    if state.as_str() == "UP" {
+                        key_events.push((virtual_key, false));
+                    } else if state.as_str() == "DOWN" {
+                        key_events.push((virtual_key, true));
+                    } else {
+                        error!(
+                            "Unknown key state in macro: {}, looking for UP or DOWN.",
+                            state.as_str()
+                        );
+                        return vec![];
+                    }
+                } else {
+                    key_events.push((virtual_key, true));
+                    key_events.push((virtual_key, false));
+                }
+            } else {
+                error!("Unknown virtual key: {}", &caps[1]);
+                return vec![];
+            }
+        }
+    }
+    key_events
+}

src/overlays/mod.rs

@@ -0,0 +1,145 @@
+use std::sync::{
+    atomic::{AtomicUsize, Ordering},
+    Arc,
+};
+
+use glam::{Affine3A, Quat, Vec3};
+use vulkano::image::ImageViewAbstract;
+
+use crate::state::AppState;
+
+use self::interactions::{DummyInteractionHandler, InteractionHandler, PointerHit};
+
+pub mod interactions;
+pub mod keyboard;
+pub mod watch;
+
+static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(0);
+
+pub enum RelativeTo {
+    None,
+    Head,
+    Hand(usize),
+}
+
+pub trait OverlayBackend: OverlayRenderer + InteractionHandler {}
+
+pub struct OverlayData {
+    pub id: usize,
+    pub name: Arc<str>,
+    pub width: f32,
+    pub size: (i32, i32),
+    pub want_visible: bool,
+    pub show_hide: bool,
+    pub grabbable: bool,
+    pub transform: Affine3A,
+    pub spawn_point: Vec3,
+    pub spawn_rotation: Quat,
+    pub relative_to: RelativeTo,
+    pub interaction_transform: Affine3A,
+    pub backend: Box<dyn OverlayBackend>,
+    pub primary_pointer: Option<usize>,
+}
+impl Default for OverlayData {
+    fn default() -> OverlayData {
+        OverlayData {
+            id: AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed),
+            name: Arc::from(""),
+            width: 1.,
+            size: (0, 0),
+            want_visible: false,
+            show_hide: false,
+            grabbable: false,
+            relative_to: RelativeTo::None,
+            spawn_point: Vec3::NEG_Z,
+            spawn_rotation: Quat::IDENTITY,
+            transform: Affine3A::IDENTITY,
+            interaction_transform: Affine3A::IDENTITY,
+            backend: Box::new(SplitOverlayBackend::default()),
+            primary_pointer: None,
+        }
+    }
+}
+
+impl OverlayData {
+    pub fn reset(&mut self, app: &mut AppState) {
+        todo!()
+    }
+    pub fn init(&mut self, app: &mut AppState) {
+        self.backend.init(app);
+    }
+    pub fn render(&mut self, app: &mut AppState) {
+        self.backend.render(app);
+    }
+    pub fn view(&mut self) -> Arc<dyn ImageViewAbstract> {
+        self.backend.view()
+    }
+}
+
+pub trait OverlayRenderer {
+    fn init(&mut self, app: &mut AppState);
+    fn pause(&mut self, app: &mut AppState);
+    fn resume(&mut self, app: &mut AppState);
+    fn render(&mut self, app: &mut AppState);
+    fn view(&mut self) -> Arc<dyn ImageViewAbstract>;
+}
+
+pub struct FallbackRenderer;
+
+impl OverlayRenderer for FallbackRenderer {
+    fn init(&mut self, _app: &mut AppState) {}
+    fn pause(&mut self, _app: &mut AppState) {}
+    fn resume(&mut self, _app: &mut AppState) {}
+    fn render(&mut self, _app: &mut AppState) {}
+    fn view(&mut self) -> Arc<dyn ImageViewAbstract> {
+        unimplemented!()
+    }
+}
+// Boilerplate and dummies
+
+pub struct SplitOverlayBackend {
+    pub renderer: Box<dyn OverlayRenderer>,
+    pub interaction: Box<dyn InteractionHandler>,
+}
+
+impl Default for SplitOverlayBackend {
+    fn default() -> SplitOverlayBackend {
+        SplitOverlayBackend {
+            renderer: Box::new(FallbackRenderer),
+            interaction: Box::new(DummyInteractionHandler),
+        }
+    }
+}
+
+impl OverlayBackend for SplitOverlayBackend {}
+impl OverlayRenderer for SplitOverlayBackend {
+    fn init(&mut self, app: &mut AppState) {
+        self.renderer.init(app);
+    }
+    fn pause(&mut self, app: &mut AppState) {
+        self.renderer.pause(app);
+    }
+    fn resume(&mut self, app: &mut AppState) {
+        self.renderer.resume(app);
+    }
+    fn render(&mut self, app: &mut AppState) {
+        self.renderer.render(app);
+    }
+    fn view(&mut self) -> Arc<dyn ImageViewAbstract> {
+        self.renderer.view()
+    }
+}
+impl InteractionHandler for SplitOverlayBackend {
+    fn on_left(&mut self, app: &mut AppState, hand: usize) {
+        self.interaction.on_left(app, hand);
+    }
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) {
+        self.interaction.on_hover(app, hit);
+    }
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32) {
+        self.interaction.on_scroll(app, hit, delta);
+    }
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
+        self.interaction.on_pointer(app, hit, pressed);
+    }
+}

src/overlays/screen.rs

@@ -0,0 +1,31 @@
+
+pub struct ScreenInteractionData {
+    next_scroll: Instant,
+    next_move: Instant,
+    mouse_transform: Affine2,
+}
+impl ScreenInteractionData {
+    fn new(pos: Vec2, size: Vec2, transform: Transform) -> ScreenInteractionHandler {
+        let transform = match transform {
+            Transform::_90 | Transform::Flipped90 =>
+                Affine2::from_cols(vec2(0., size.y), vec2(-size.x, 0.), vec2(pos.x + size.x, pos.y)),
+            Transform::_180 | Transform::Flipped180 => 
+                Affine2::from_cols(vec2(-size.x, 0.), vec2(0., -size.y), vec2(pos.x + size.x, pos.y + size.y)),
+            Transform::_270 | Transform::Flipped270 => 
+                Affine2::from_cols(vec2(0., -size.y), vec2(size.x, 0.), vec2(pos.x, pos.y + size.y)),
+            _ => 
+                Affine2::from_cols(vec2(size.x, 0.), vec2(0., size.y), pos),
+        };
+
+        ScreenInteractionHandler {
+            next_scroll: Instant::now(),
+            next_move: Instant::now(),
+            mouse_transform: transform,
+        }
+    }
+}
+
+struct ScreenInteractionHandler {
+
+}
+

src/overlays/watch.rs

@@ -0,0 +1,170 @@
+use std::{sync::Arc, time::Instant};
+
+use chrono::Local;
+use glam::{Quat, Vec3};
+
+use crate::{
+    gui::{color_parse, CanvasBuilder},
+    state::AppState,
+};
+
+use super::{OverlayData, RelativeTo};
+
+pub const WATCH_DEFAULT_POS: Vec3 = Vec3::new(0., 0., 0.15);
+pub const WATCH_DEFAULT_ROT: Quat = Quat::from_xyzw(0.7071066, 0., 0.7071066, 0.0007963);
+
+pub fn create_watch(state: &AppState, screens: Vec<(usize, Arc<str>)>) -> OverlayData {
+    let mut canvas = CanvasBuilder::new(400, 200, state.graphics.clone(), state.format, ());
+    let empty_str: Arc<str> = Arc::from("");
+
+    // Background
+    canvas.bg_color = color_parse("#353535");
+    canvas.panel(0., 0., 400., 200.);
+
+    // Time display
+    canvas.font_size = 46;
+    let clock = canvas.label(19., 100., 200., 50., empty_str.clone());
+    clock.on_update = Some(|control, _data, _app| {
+        let date = Local::now();
+        control.set_text(&format!("{}", &date.format("%H:%M")));
+    });
+
+    canvas.font_size = 14;
+    let date = canvas.label(20., 125., 200., 50., empty_str.clone());
+    date.on_update = Some(|control, _data, _app| {
+        let date = Local::now();
+        control.set_text(&format!("{}", &date.format("%x")));
+    });
+
+    let day_of_week = canvas.label(20., 150., 200., 50., empty_str);
+    day_of_week.on_update = Some(|control, _data, _app| {
+        let date = Local::now();
+        control.set_text(&format!("{}", &date.format("%A")));
+    });
+
+    // Volume controls
+    canvas.bg_color = color_parse("#222222");
+    canvas.fg_color = color_parse("#AAAAAA");
+    canvas.font_size = 14;
+
+    canvas.bg_color = color_parse("#303030");
+    canvas.fg_color = color_parse("#353535");
+
+    let vol_up = canvas.button(327., 116., 46., 32., "+".into());
+    vol_up.on_press = Some(|_control, _data, _app| {
+        println!("Volume up!"); //TODO
+    });
+
+    let vol_dn = canvas.button(327., 52., 46., 32., "-".into());
+    vol_dn.on_press = Some(|_control, _data, _app| {
+        println!("Volume down!"); //TODO
+    });
+
+    canvas.bg_color = color_parse("#303030");
+    canvas.fg_color = color_parse("#353535");
+
+    let settings = canvas.button(2., 162., 36., 36., "☰".into());
+    settings.on_press = Some(|_control, _data, _app| {
+        println!("Settings!"); //TODO
+    });
+
+    canvas.fg_color = color_parse("#CCBBAA");
+    canvas.bg_color = color_parse("#406050");
+    // Bottom row
+    let num_buttons = screens.len() + 1;
+    let button_width = 360. / num_buttons as f32;
+    let mut button_x = 40.;
+
+    let keyboard = canvas.button(button_x + 2., 162., button_width - 4., 36., "Kbd".into());
+    keyboard.state = Some(WatchButtonState {
+        pressed_at: Instant::now(),
+        scr_idx: 0,
+    });
+
+    keyboard.on_press = Some(|control, _data, _app| {
+        if let Some(state) = control.state.as_mut() {
+            state.pressed_at = Instant::now();
+        }
+    });
+    keyboard.on_release = Some(|control, _data, app| {
+        if let Some(state) = control.state.as_ref() {
+            if Instant::now()
+                .saturating_duration_since(state.pressed_at)
+                .as_millis()
+                < 2000
+            {
+                app.tasks.push_back(Box::new(|_app, o| {
+                    for overlay in o {
+                        if &*overlay.name == "Kbd" {
+                            overlay.want_visible = !overlay.want_visible;
+                            return;
+                        }
+                    }
+                }));
+            } else {
+                app.tasks.push_back(Box::new(|app, o| {
+                    for overlay in o {
+                        if &*overlay.name == "Kbd" {
+                            overlay.reset(app);
+                        }
+                    }
+                }));
+            }
+        }
+    });
+    button_x += button_width;
+
+    canvas.bg_color = color_parse("#405060");
+
+    for (scr_idx, scr_name) in screens.into_iter() {
+        let button = canvas.button(button_x + 2., 162., button_width - 4., 36., scr_name);
+        button.state = Some(WatchButtonState {
+            pressed_at: Instant::now(),
+            scr_idx,
+        });
+
+        button.on_press = Some(|control, _data, _app| {
+            if let Some(state) = control.state.as_mut() {
+                state.pressed_at = Instant::now();
+            }
+        });
+        button.on_release = Some(|control, _data, app| {
+            if let Some(state) = control.state.as_ref() {
+                let scr_idx = state.scr_idx;
+                if Instant::now()
+                    .saturating_duration_since(state.pressed_at)
+                    .as_millis()
+                    < 2000
+                {
+                    app.tasks.push_back(Box::new(move |_app, o| {
+                        o[scr_idx].want_visible = !o[scr_idx].want_visible;
+                    }));
+                } else {
+                    app.tasks.push_back(Box::new(move |app, o| {
+                        o[scr_idx].reset(app);
+                    }));
+                }
+            }
+        });
+        button_x += button_width;
+    }
+
+    let relative_to = RelativeTo::Hand(state.session.watch_hand);
+
+    OverlayData {
+        name: "Watch".into(),
+        size: (400, 200),
+        width: 0.065,
+        backend: Box::new(canvas.build()),
+        want_visible: true,
+        relative_to,
+        spawn_point: state.session.watch_pos,
+        spawn_rotation: state.session.watch_rot,
+        ..Default::default()
+    }
+}
+
+struct WatchButtonState {
+    pressed_at: Instant,
+    scr_idx: usize,
+}

src/ovr.rs

@@ -0,0 +1,64 @@
+use std::{path::Path, sync::Arc};
+
+use vulkano::{
+    image::{
+        sys::{Image, RawImage},
+        ImageViewAbstract,
+    },
+    Handle, VulkanObject,
+};
+
+use crate::graphics::WlxGraphics;
+
+pub struct OpenVrState {
+    pub context: ovr_overlay::Context,
+}
+
+
+pub struct OvrTextureData {
+    image_handle: u64,
+    device: u64,
+    physical: u64,
+    instance: u64,
+    queue: u64,
+    queue_family_index: u32,
+    width: u32,
+    height: u32,
+    format: u32,
+    sample_count: u32,
+}
+
+impl OvrTextureData {
+    pub fn new(graphics: Arc<WlxGraphics>, view: Arc<dyn ImageViewAbstract>) -> OvrTextureData {
+        let image = view.image();
+
+        let device = graphics.device.handle().as_raw();
+        let physical = graphics.device.physical_device().handle().as_raw();
+        let instance = graphics.instance.handle().as_raw();
+        let queue = graphics.queue.handle().as_raw();
+        let queue_family_index = graphics.queue.queue_family_index();
+
+        let (width, height) = {
+            let dim = image.dimensions();
+            (dim.width() as u32, dim.height() as u32)
+        };
+
+        let sample_count = image.samples() as u32;
+        let format = image.format() as u32;
+
+        let image_handle = image.inner().image.handle().as_raw();
+
+        OvrTextureData {
+            image_handle,
+            device,
+            physical,
+            instance,
+            queue,
+            queue_family_index,
+            width,
+            height,
+            format,
+            sample_count,
+        }
+    }
+}

src/res/actions.json

@@ -0,0 +1,83 @@
+{
+  "actions": [
+    {
+      "name": "/actions/default/in/Click",
+      "type": "boolean",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/Grab",
+      "type": "boolean",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/Scroll",
+      "type": "vector2",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/ShowHide",
+      "type": "boolean",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/AltClick",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/ClickModifierRight",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/ClickModifierMiddle",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/SpaceDrag",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/LeftHand",
+      "type": "pose",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/RightHand",
+      "type": "pose",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/out/HapticsLeft",
+      "type": "vibration"
+    },
+    {
+      "name": "/actions/default/out/HapticsRight",
+      "type": "vibration"
+    }
+  ],
+  "action_sets": [
+    {
+      "name": "/actions/default",
+      "usage": "leftright"
+    }
+  ],
+  "default_bindings": [
+    {
+      "controller_type": "knuckles",
+      "binding_url": "actions_binding_knuckles.json"
+    },
+    {
+      "controller_type": "oculus_touch",
+      "binding_url": "actions_binding_oculus.json"
+    },
+    {
+      "controller_type": "vive_controller",
+      "binding_url": "actions_binding_vive.json"
+    }
+  ],
+  "localization": []
+}

src/res/actions_binding_knuckles.json

@@ -0,0 +1,173 @@
+{
+   "action_manifest_version" : 0,
+   "app_key" : "galister.wlxoverlay",
+   "bindings" : {
+      "/actions/default" : {
+         "haptics" : [
+            {
+               "output" : "/actions/default/out/hapticsleft",
+               "path" : "/user/hand/left/output/haptic"
+            },
+            {
+               "output" : "/actions/default/out/hapticsright",
+               "path" : "/user/hand/right/output/haptic"
+            }
+         ],
+         "poses" : [
+            {
+               "output" : "/actions/default/in/lefthand",
+               "path" : "/user/hand/left/pose/tip"
+            },
+            {
+               "output" : "/actions/default/in/righthand",
+               "path" : "/user/hand/right/pose/tip"
+            }
+         ],
+         "sources" : [
+            {
+               "inputs" : {
+                  "double" : {
+                     "output" : "/actions/default/in/showhide"
+                  },
+                  "touch" : {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/b"
+            },
+            {
+               "path": "/user/hand/left/input/a",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/b",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/a",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  }
+               },
+               "mode" : "button",
+               "parameters" : {
+                  "click_activate_threshold" : "0.35",
+                  "click_deactivate_threshold" : "0.31"
+               },
+               "path" : "/user/hand/left/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  }
+               },
+               "mode" : "button",
+               "parameters" : {
+                  "click_activate_threshold" : "0.35",
+                  "click_deactivate_threshold" : "0.31"
+               },
+               "path" : "/user/hand/right/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/altclick"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/right/input/trackpad"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/altclick"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/trackpad"
+            },
+            {
+               "inputs" : {
+                  "grab" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode" : "grab",
+               "parameters" : {
+                  "value_hold_threshold" : "1.3",
+                  "value_release_threshold" : "1.1"
+               },
+               "path" : "/user/hand/left/input/grip"
+            },
+            {
+               "inputs" : {
+                  "grab" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode" : "grab",
+               "parameters" : {
+                  "value_hold_threshold" : "1.3",
+                  "value_release_threshold" : "1.1"
+               },
+               "path" : "/user/hand/right/input/grip"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/left/input/thumbstick"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/right/input/thumbstick"
+            }
+         ]
+      }
+   },
+   "category" : "steamvr_input",
+   "controller_type" : "knuckles",
+   "description" : "Ver1",
+   "interaction_profile" : "",
+   "name" : "WlxOverlay configuration for Index Controller",
+   "options" : {
+      "mirror_actions" : false,
+      "simulated_controller_type" : "none"
+   },
+   "simulated_actions" : []
+}

src/res/actions_binding_oculus.json

@@ -0,0 +1,139 @@
+{
+   "action_manifest_version" : 0,
+   "app_key" : "galister.wlxoverlay",
+   "bindings" : {
+      "/actions/default" : {
+         "haptics" : [
+            {
+               "output" : "/actions/default/out/hapticsleft",
+               "path" : "/user/hand/left/output/haptic"
+            },
+            {
+               "output" : "/actions/default/out/hapticsright",
+               "path" : "/user/hand/right/output/haptic"
+            }
+         ],
+         "poses" : [
+            {
+               "output" : "/actions/default/in/lefthand",
+               "path" : "/user/hand/left/pose/tip"
+            },
+            {
+               "output" : "/actions/default/in/righthand",
+               "path" : "/user/hand/right/pose/tip"
+            }
+         ],
+         "sources" : [
+            {
+               "inputs" : {
+                  "double" : {
+                     "output" : "/actions/default/in/showhide"
+                  },
+                  "touch" : {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/y"
+            },
+            {
+               "path": "/user/hand/left/input/x",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/b",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/a",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/left/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/right/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/left/input/grip"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/right/input/grip"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/left/input/joystick"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/right/input/joystick"
+            }
+         ]
+      }
+   },
+   "category" : "steamvr_input",
+   "controller_type" : "oculus_touch",
+   "description" : "Ver1",
+   "interaction_profile" : "",
+   "name" : "WlxOverlay configuration for Oculus Touch Controller",
+   "options" : {
+      "mirror_actions" : false,
+      "simulated_controller_type" : "none"
+   },
+   "simulated_actions" : []
+}

src/res/actions_binding_vive.json

@@ -0,0 +1,126 @@
+{
+   "action_manifest_version" : 0,
+   "app_key" : "galister.wlxoverlay",
+   "bindings" : {
+      "/actions/default": {
+         "haptics" : [
+            {
+               "output" : "/actions/default/out/hapticsleft",
+               "path" : "/user/hand/left/output/haptic"
+            },
+            {
+               "output" : "/actions/default/out/hapticsright",
+               "path" : "/user/hand/right/output/haptic"
+            }
+         ],
+         "sources": [
+            {
+               "path": "/user/hand/left/input/grip",
+               "mode": "button",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/grab"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/grip",
+               "mode": "button",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/grab"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/trigger",
+               "mode": "button",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/click"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/trigger",
+               "mode": "trigger",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/click"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/trackpad",
+               "mode": "scroll",
+               "parameters": {
+                  "scroll_mode": "smooth"
+               },
+               "inputs": {
+                  "scroll": {
+                     "output": "/actions/default/in/scroll"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/trackpad",
+               "mode": "scroll",
+               "inputs": {
+                  "scroll": {
+                     "output": "/actions/default/in/scroll"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/application_menu",
+               "mode": "button",
+               "inputs": {
+                  "double": {
+                     "output": "/actions/default/in/showhide"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/trackpad",
+               "mode": "dpad",
+               "parameters": {
+                  "sub_mode": "touch"
+               },
+               "inputs": {
+                  "west": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  },
+                  "east": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/trackpad",
+               "mode": "dpad",
+               "parameters": {
+                  "sub_mode": "touch"
+               },
+               "inputs": {
+                  "west": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  },
+                  "east": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            }
+         ]
+      }
+   },
+   "category" : "steamvr_input",
+   "controller_type" : "vive_controller",
+   "description" : "Ver1",
+   "interaction_profile" : "",
+   "name" : "WlxOverlay configuration for Vive Controller",
+   "options" : {
+      "mirror_actions" : false,
+      "simulated_controller_type" : "none"
+   },
+   "simulated_actions" : []
+}

src/res/keyboard.yaml

@@ -0,0 +1,127 @@
+---
+
+# This file contains all data needed to generate the keyboard.
+# You can create any layout, as long as:
+# - All keys are rectangular with 1 unit of height.
+# This means:
+# - We're limited to the flat & boring ANSI enter key.
+# - Numpad + and Enter might not look so great.
+
+# *** Important ***
+# The keyboard layout uses virtual key codes, so they are layout-independent.
+# For example, Q on a French layout actually results in A.
+# If you're using a non-english layout, chances are you only need to edit the label section below.
+
+# Not used for anything right now
+name: "en-us_full"
+
+# How many units of key size in each row? 1 = standard letter key size
+row_size: 23
+
+# Specifies the size of each key. The sum of any given row must equal RowSize
+key_sizes:
+    - [1.5,0.5, 1, 1, 1, 1,0.5,1, 1, 1, 1,0.5,1, 1, 1, 1, 0.5, 1, 1, 1, 4.5]
+    - [1,   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2,     0.5, 1, 1, 1, 0.5, 1, 1, 1, 1]
+    - [1.5,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.5,  0.5, 1, 1, 1, 0.5, 1, 1, 1, 1]
+    - [1.75,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2.25,   4,                 1, 1, 1, 1]
+    - [2.25,    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2.75,    1.5,    1, 1.5,    1, 1, 1, 1]
+    - [1.25, 1.25, 1.25, 6.25, 1.25, 1.25, 1.25, 1.25, 0.5, 1, 1, 1, 0.5, 2,    1, 1]
+
+# The main (blue) layout of the keyboard.
+# Accepted are:
+# - virtual keys, for a full list go to https://github.com/galister/X11Overlay/blob/master/Types/VirtualKey.cs
+# - ExecCommands (defined below)
+# - Macros (defined below)
+# - ~ (null) will leave an empty space with the corresponding size from key_sizes
+main_layout:
+    - ["Escape", ~, "F1", "F2", "F3", "F4", ~, "F5", "F6", "F7", "F8", ~, "F9", "F10", "F11", "F12", ~, "Print", "Scroll", "Pause", ~]
+    - ["Oem3", "N1", "N2", "N3", "N4", "N5", "N6", "N7", "N8", "N9", "N0", "Minus", "Plus", "BackSpace", ~, "Insert", "Home", "Prior", ~, "NumLock", "KP_Divide", "KP_Multiply", "KP_Subtract"]
+    - ["Tab", "Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P", "Oem4", "Oem6", "Oem5", ~, "Delete", "End", "Next", ~, "KP_7", "KP_8", "KP_9", "KP_Add"]
+    - ["XF86Favorites", "A", "S", "D", "F", "G", "H", "J", "K", "L", "Oem1", "Oem7", "Return", ~, "KP_4", "KP_5", "KP_6", ~]
+    - ["LShift", "Z", "X", "C", "V", "B", "N", "M", "Comma", "Period", "Oem2", "RShift", ~, "Up", ~, "KP_1", "KP_2", "KP_3", "KP_Enter"]
+    - ["LCtrl", "LSuper", "LAlt", "Space", "Meta", "RSuper", "Menu", "RCtrl", ~, "Left", "Down", "Right", ~, "KP_0", "KP_Decimal", ~]
+
+# When using the purple pointer...
+# None   - No special functionality when using purple pointer
+# Shift  - Use same functionality as the orange pointer
+# Ctrl   - Use Main layout with Ctrl modifier
+# Super  - Use Main layout with Super (WinKey) modifier
+# Meta   - Use Main layout with Meta (AltGr) modifier
+# Layout - Use the alt_layout defined below
+alt_layout_mode: Ctrl
+
+# The alt (fn) layout of the keyboard. Only takes effect when alt_layout_mode: Layout
+# Accepted values are the same as for MainLayout.
+# The default layout here is a dummy. Change it to fit your liking.
+alt_layout:
+    - ["Escape", ~, "F1", "F2", "F3", "F4", ~, "F5", "F6", "F7", "F8", ~, "F9", "F10", "F11", "F12", ~, "Print", "Scroll", "Pause", ~, "COPY", "PASTE"]
+    - ["Oem3", "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", "Minus", "Plus", "BackSpace", ~, "Insert", "Home", "Prior", ~, "NumLock", "KP_Divide", "KP_Multiply", "KP_Subtract"]
+    - ["Tab", "Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P", "Oem4", "Oem6", "Oem5", ~, "Delete", "End", "Next", ~, "KP_7", "KP_8", "KP_9", "KP_Add"]
+    - ["XF86Favorites", "A", "S", "D", "F", "G", "H", "J", "K", "L", "Oem1", "Oem7", "Return", ~, "KP_4", "KP_5", "KP_6", ~]
+    - ["LShift", "Oem102", "Z", "X", "C", "V", "B", "N", "M", "Comma", "Period", "Oem2", "RShift", ~, "Up", ~, "KP_1", "KP_2", "KP_3", "KP_Enter"]
+    - ["LCtrl", "LSuper", "LAlt", "Space", "Meta", "RSuper", "Menu", "RCtrl", ~, "Left", "Down", "Right", ~, "KP_0", "KP_Decimal", ~]
+
+# Shell commands to be used in a layout.
+# Value is an array of string arguments.
+exec_commands:
+    STT: [ "whisper_stt", "--lang", "en" ]
+
+# Series of keypresses to be used in a layout.
+# Format: keyName [DOWN|UP]
+# keyName must be a valid key from `xmodmap -pke`
+# DOWN|UP: can be omitted for an implicit "keyName DOWN, keyName UP"
+macros:
+    KILL: [ "LSuper DOWN", "LCtrl DOWN", "Escape", "LCtrl UP", "LSuper UP" ]
+    COPY: [ "LCtrl DOWN", "C", "LCtrl UP" ]
+    PASTE: [ "LCtrl DOWN", "V", "LCtrl UP" ]
+
+# Custom labels to use.
+# Key: element of MainLayout / AltLayout
+# Value: Array of strings. 0th element is the upper row, 1st element is lower row.
+# For empty labels, use [] (do not use ~)
+labels:
+    "N1":            ["1", "!"]
+    "N2":            ["2", "@"]
+    "N3":            ["3", "#"]
+    "N4":            ["4", "$"]
+    "N5":            ["5", "%"]
+    "N6":            ["6", "^"]
+    "N7":            ["7", "&"]
+    "N8":            ["8", "*"]
+    "N9":            ["9", "("]
+    "N0":            ["0", ")"]
+    "Escape":       ["Esc"]
+    "Prior":        ["PgUp"]
+    "Next":         ["PgDn"]
+    "NumLock":      ["Num"]
+    "Space":        []
+    "LAlt":         ["Alt"]
+    "LCtrl":        ["Ctrl"]
+    "RCtrl":        ["Ctrl"]
+    "LSuper":       ["Super"]
+    "RSuper":       ["Super"]
+    "LShift":       ["Shift"]
+    "RShift":       ["Shift"]
+    "Insert":       ["Ins"]
+    "Delete":       ["Del"]
+    "Minus":        ["-", "_"]
+    "Plus":         ["=", "+"]
+    "BackSpace":    ["<<"]
+    "Comma":        [" ,",  "<"]
+    "Period":       [" .",  ">"]
+    "Oem1":         [" ;",  ":"]
+    "Oem2":         [" /",  "?"]
+    "Oem3":         ["`", "~"]
+    "Oem4":         [" [",  "{"]
+    "Oem5":         [" \\", "|"]
+    "Oem6":         [" ]",  "}"]
+    "Oem7":         [" '",  "\""]
+    "Oem102":       [" \\", "|"]
+    "KP_Divide":    [" /"]
+    "KP_Add":       [" +"]
+    "KP_Multiply":  [" *"]
+    "KP_Decimal":   [" ."]
+    "KP_Subtract":  [" -"]
+    "KP_Enter":     ["Ent"]
+    "XF86Favorites": ["Rofi"]
+

src/shaders/mod.rs

@@ -0,0 +1,108 @@
+pub mod vert_common {
+    vulkano_shaders::shader! {
+        ty: "vertex",
+        src: r"#version 310 es
+            precision highp float;
+
+            layout (location = 0) in vec2 in_pos;
+            layout (location = 1) in vec2 in_uv;
+            layout (location = 0) out vec2 out_uv;
+
+            void main() {
+                out_uv = in_uv;
+                gl_Position = vec4(in_pos * 2. - 1., 0., 1.);
+            }
+        ",
+    }
+}
+
+pub mod frag_color {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        src: r"#version 310 es
+            precision highp float;
+
+            layout (location = 0) in vec2 in_uv;
+            layout (location = 0) out vec4 out_color;
+
+            layout (set = 0, binding = 0) uniform ColorBlock {
+                uniform vec4 in_color;
+            };
+
+            void main()
+            {
+                out_color = in_color;
+            }
+        ",
+    }
+}
+
+pub mod frag_glyph {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        src: r"#version 310 es
+            precision highp float;
+
+            layout (location = 0) in vec2 in_uv;
+            layout (location = 0) out vec4 out_color;
+
+            layout (set = 0, binding = 0) uniform sampler2D in_texture;
+
+            layout (set = 1, binding = 0) uniform ColorBlock {
+                uniform vec4 in_color;
+            };
+
+            void main()
+            {
+                float r = texture(in_texture, in_uv).r;
+                out_color = vec4(r,r,r,r) * in_color;
+            }
+        ",
+    }
+}
+
+pub mod frag_sprite {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        src: r"#version 310 es
+            precision highp float;
+
+            layout (location = 0) in vec2 in_uv;
+            layout (location = 0) out vec4 out_color;
+
+            layout (set = 0, binding = 0) uniform sampler2D in_texture;
+
+            void main()
+            {
+                vec4 c = texture(in_texture, in_uv);
+                out_color.rgb = c.rgb;
+                out_color.a = min((c.r + c.g + c.b)*100.0, 1.0);
+            }
+        ",
+    }
+}
+
+pub mod frag_srgb {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        src: r"#version 310 es
+            precision highp float;
+
+            layout (location = 0) in vec2 in_uv;
+            layout (location = 0) out vec4 out_color;
+
+            layout (set = 0, binding = 0) uniform sampler2D in_texture;
+
+            void main()
+            {
+                out_color = texture(in_texture, in_uv);
+                
+                bvec4 cutoff = lessThan(out_color, vec4(0.04045));
+                vec4 higher = pow((out_color + vec4(0.055))/vec4(1.055), vec4(2.4));
+                vec4 lower = out_color/vec4(12.92);
+
+                out_color = mix(higher, lower, cutoff);
+            }
+        ",
+    }
+}

src/shaders/src/color.frag

@@ -0,0 +1,15 @@
+#version 310 es
+precision highp float;
+
+layout (location = 0) in vec2 in_uv;
+layout (location = 0) out vec4 out_color;
+
+layout (set = 0, binding = 0) uniform ColorBlock {
+    uniform vec4 in_color;
+};
+
+void main()
+{
+    out_color = in_color;
+}
+

src/shaders/src/common.vert

@@ -0,0 +1,12 @@
+#version 310 es
+precision highp float;
+
+layout (location = 0) in vec2 in_pos;
+layout (location = 1) in vec2 in_uv;
+layout (location = 0) out vec2 out_uv;
+
+void main() {
+    out_uv = in_uv;
+    gl_Position = vec4(in_pos * 2. - 1., 0., 1.);
+}
+

src/shaders/src/glyph.frag

@@ -0,0 +1,18 @@
+#version 310 es
+precision highp float;
+
+layout (location = 0) in vec2 in_uv;
+layout (location = 0) out vec4 out_color;
+
+layout (set = 0, binding = 0) uniform sampler2D in_texture;
+
+layout (set = 0, binding = 1) uniform ColorBlock {
+    uniform vec4 in_color;
+};
+
+void main()
+{
+    float r = texture(in_texture, in_uv).r;
+    out_color = vec4(r,r,r,r) * in_color;
+}
+

src/shaders/src/sprite.frag

@@ -0,0 +1,14 @@
+#version 310 es
+precision highp float;
+
+layout (location = 0) in vec2 in_uv;
+layout (location = 0) out vec4 out_color;
+
+layout (set = 0, binding = 0) uniform sampler2D in_texture;
+
+void main()
+{
+    out_color = texture(in_texture, in_uv);
+    out_color.a = 1.;
+}
+

src/shaders/src/srgb.frag

@@ -0,0 +1,19 @@
+#version 310 es
+precision highp float;
+
+layout (location = 0) in vec2 in_uv;
+layout (location = 0) out vec4 out_color;
+
+layout (set = 0, binding = 0) uniform sampler2D in_texture;
+
+void main()
+{
+    out_color = texture(in_texture, in_uv);
+    
+    bvec4 cutoff = lessThan(out_color, vec4(0.04045));
+    vec4 higher = pow((out_color + vec4(0.055))/vec4(1.055), vec4(2.4));
+    vec4 lower = out_color/vec4(12.92);
+
+    out_color = mix(higher, lower, cutoff);
+}
+

src/state.rs

@@ -0,0 +1,103 @@
+use std::{collections::VecDeque, env::VarError, path::Path, sync::Arc};
+
+use glam::{Quat, Vec3};
+use log::warn;
+
+use crate::{
+    graphics::WlxGraphics, gui::font::FontCache, input::InputProvider, overlays::OverlayData,
+};
+
+pub const WATCH_DEFAULT_POS: Vec3 = Vec3::new(0., 0., 0.15);
+pub const WATCH_DEFAULT_ROT: Quat = Quat::from_xyzw(0.7071066, 0., 0.7071066, 0.0007963);
+
+pub type Task = Box<dyn FnOnce(&mut AppState, &mut [OverlayData]) + Send>;
+
+pub struct AppState {
+    pub fc: FontCache,
+    //pub input: InputState,
+    pub session: AppSession,
+    pub tasks: VecDeque<Task>,
+    pub graphics: Arc<WlxGraphics>,
+    pub format: vulkano::format::Format,
+    pub input: Box<dyn InputProvider>,
+}
+
+pub struct AppSession {
+    pub config_path: String,
+
+    pub show_screens: Vec<String>,
+    pub show_keyboard: bool,
+    pub keyboard_volume: f32,
+
+    pub screen_flip_h: bool,
+    pub screen_flip_v: bool,
+    pub screen_invert_color: bool,
+
+    pub watch_hand: usize,
+    pub watch_pos: Vec3,
+    pub watch_rot: Quat,
+
+    pub primary_hand: usize,
+
+    pub capture_method: String,
+
+    pub color_norm: Vec3,
+    pub color_shift: Vec3,
+    pub color_alt: Vec3,
+    pub color_grab: Vec3,
+
+    pub click_freeze_time_ms: u64,
+}
+
+impl AppSession {
+    pub fn load() -> AppSession {
+        let config_path = std::env::var("XDG_CONFIG_HOME")
+            .or_else(|_| std::env::var("HOME").map(|home| format!("{}/.config", home)))
+            .or_else(|_| {
+                warn!("Err: $XDG_CONFIG_HOME and $HOME are not set, using /tmp/wlxoverlay");
+                Ok::<String, VarError>("/tmp".to_string())
+            })
+            .map(|config| Path::new(&config).join("wlxoverlay"))
+            .ok()
+            .and_then(|path| path.to_str().map(|path| path.to_string()))
+            .unwrap();
+
+        let _ = std::fs::create_dir(&config_path);
+
+        AppSession {
+            config_path,
+            show_screens: vec!["DP-3".to_string()],
+            keyboard_volume: 0.5,
+            show_keyboard: false,
+            screen_flip_h: false,
+            screen_flip_v: false,
+            screen_invert_color: false,
+            capture_method: "auto".to_string(),
+            primary_hand: 1,
+            watch_hand: 1,
+            watch_pos: WATCH_DEFAULT_POS,
+            watch_rot: WATCH_DEFAULT_ROT,
+            color_norm: Vec3 {
+                x: 0.,
+                y: 1.,
+                z: 1.,
+            },
+            color_shift: Vec3 {
+                x: 1.,
+                y: 1.,
+                z: 0.,
+            },
+            color_alt: Vec3 {
+                x: 1.,
+                y: 0.,
+                z: 1.,
+            },
+            color_grab: Vec3 {
+                x: 1.,
+                y: 0.,
+                z: 0.,
+            },
+            click_freeze_time_ms: 300,
+        }
+    }
+}