progress commit: openxr + pipewire + refactor

Cargo.toml

@@ -10,10 +10,12 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+anyhow = "1.0.79"
 ash = "^0.37.2"
 ash-window = "0.12.0"
 chrono = "0.4.29"
 cstr = "0.2.11"
+ctrlc = { version = "3.4.2", features = ["termination"] }
 env_logger = "0.10.0"
 fontconfig-rs = { version = "0.1.1", features = ["dlopen"] }
 freetype-rs = "0.32.0"
@@ -25,7 +27,8 @@ input-linux = "0.6.0"
 libc = "0.2.147"
 log = "0.4.20"
 once_cell = "1.18.0"
-ovr_overlay = { features = ["ovr_input", "ovr_system"], path = "../ovr_overlay_oyasumi" }
+openxr = { version = "0.17.1", optional = true }
+ovr_overlay = { features = ["ovr_input", "ovr_system"], path = "../ovr_overlay_oyasumi", optional = true }
 png = "0.17.10"
 raw-window-handle = "0.5.2"
 regex = "1.9.5"
@@ -34,11 +37,17 @@ serde = { version = "1.0.188", features = ["derive"] }
 serde_yaml = "0.9.25"
 smallvec = "1.11.0"
 strum = { version = "0.25.0", features = ["derive"] }
+thiserror = "1.0.56"
 tinyvec = "1.6.0"
-vulkano = { version = "0.34.1", features = ["serde"] }
+vulkano = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
-vulkano-shaders = "0.34.0"
+vulkano-shaders = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
-vulkano-util = "0.34.1"
+vulkano-util = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
-vulkano-win = "0.34.0"
+vulkano-win = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
-winit = "0.28.6"
+winit = "0.29.10"
 wlx-capture = { path = "../wlx-capture" }
 
+[features]
+openvr = ["dep:ovr_overlay"]
+openxr = ["dep:openxr"]
+default = ["openvr", "openxr"]
+

src/backend/common.rs

@@ -18,6 +18,13 @@ use crate::{
 
 use super::overlay::{OverlayData, OverlayState};
 
+pub enum BackendError {
+    NotSupported,
+    Shutdown,
+    Restart,
+    Fatal,
+}
+
 pub struct OverlayContainer<T>
 where
     T: Default,
@@ -39,11 +46,13 @@ where
             get_screens_x11()
         };
 
-        //let watch = create_watch::<T>(&app, &screens);
+        let mut watch = create_watch::<T>(&app, &screens);
-        //overlays.insert(watch.state.id, watch);
+        watch.state.want_visible = false;
+        overlays.insert(watch.state.id, watch);
 
-        //let keyboard = create_keyboard(&app);
+        let mut keyboard = create_keyboard(&app);
-        //overlays.insert(keyboard.state.id, keyboard);
+        keyboard.state.want_visible = false;
+        overlays.insert(keyboard.state.id, keyboard);
 
         let mut first = true;
         for mut screen in screens {

src/backend/mod.rs

@@ -1,5 +1,7 @@
 pub mod common;
 pub mod input;
+#[cfg(feature = "openvr")]
 pub mod openvr;
+#[cfg(feature = "openxr")]
 pub mod openxr;
 pub mod overlay;

src/backend/openvr/lines.rs

@@ -29,7 +29,7 @@ impl LinePool {
 
         let buf = vec![255; 16];
 
-        let texture = command_buffer.texture2d(2, 2, Format::R8G8B8A8_UNORM, buf);
+        let texture = command_buffer.texture2d(2, 2, Format::R8G8B8A8_UNORM, &buf);
         command_buffer.build_and_execute_now();
 
         graphics

src/backend/openvr/mod.rs

@@ -1,6 +1,10 @@
 use glam::Vec4;
 use std::{
     collections::VecDeque,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
     time::{Duration, Instant},
 };
 
@@ -19,24 +23,27 @@ use crate::{
         input::interact,
         openvr::{input::OpenVrInputSource, lines::LinePool},
     },
+    graphics::WlxGraphics,
     state::AppState,
 };
 
 use self::{input::action_manifest_path, overlay::OpenVrOverlayData};
 
-use super::common::{OverlayContainer, TaskType};
+use super::common::{BackendError, OverlayContainer, TaskType};
 
 pub mod input;
 pub mod lines;
 pub mod overlay;
 
-pub fn openvr_run() {
+pub fn openvr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
     let app_type = EVRApplicationType::VRApplication_Overlay;
     let Ok(context) = ovr_overlay::Context::init(app_type) else {
-        log::error!("Failed to initialize OpenVR");
+        log::warn!("Will not use OpenVR: Context init failed");
-        return;
+        return Err(BackendError::NotSupported);
     };
 
+    log::info!("Using OpenVR runtime");
+
     let mut overlay_mngr = context.overlay_mngr();
     //let mut settings_mngr = context.settings_mngr();
     let mut input_mngr = context.input_mngr();
@@ -55,19 +62,23 @@ pub fn openvr_run() {
         InstanceExtensions::from_iter(names.iter().map(|s| s.as_str()))
     };
 
-    let mut state = AppState::new(instance_extensions, device_extensions_fn);
+    let mut state = {
+        let graphics = WlxGraphics::new(instance_extensions, device_extensions_fn);
+        AppState::from_graphics(graphics)
+    };
+
     let mut overlays = OverlayContainer::<OpenVrOverlayData>::new(&mut state);
 
     state.hid_provider.set_desktop_extent(overlays.extent);
 
     if let Err(e) = input_mngr.set_action_manifest(action_manifest_path()) {
         log::error!("Failed to set action manifest: {}", e.description());
-        return;
+        return Err(BackendError::Fatal);
     };
 
     let Ok(mut input_source) = OpenVrInputSource::new(&mut input_mngr) else {
         log::error!("Failed to initialize input");
-        return;
+        return Err(BackendError::Fatal);
     };
 
     let Ok(refresh_rate) = system_mngr.get_tracked_device_property::<f32>(
@@ -75,7 +86,7 @@ pub fn openvr_run() {
         ETrackedDeviceProperty::Prop_DisplayFrequency_Float,
     ) else {
         log::error!("Failed to get display refresh rate");
-        return;
+        return Err(BackendError::Fatal);
     };
 
     log::info!("HMD running @ {} Hz", refresh_rate);
@@ -90,12 +101,17 @@ pub fn openvr_run() {
         lines.allocate(&mut overlay_mngr, &mut state),
     ];
 
-    loop {
+    'main_loop: loop {
+        if !running.load(Ordering::Relaxed) {
+            log::warn!("Received shutdown signal.");
+            break 'main_loop;
+        }
+
         while let Some(event) = system_mngr.poll_next_event() {
             match event.event_type {
                 EVREventType::VREvent_Quit => {
-                    log::info!("Received quit event, shutting down.");
+                    log::warn!("Received quit event, shutting down.");
-                    return;
+                    break 'main_loop;
                 }
                 EVREventType::VREvent_TrackedDeviceActivated
                 | EVREventType::VREvent_TrackedDeviceDeactivated
@@ -173,4 +189,9 @@ pub fn openvr_run() {
             },
         ));
     }
+
+    log::warn!("OpenVR shutdown");
+    // context.shutdown() called by Drop
+
+    Ok(())
 }

src/backend/openxr/input.rs

@@ -0,0 +1,232 @@
+use glam::{bool, Affine3A, Quat, Vec3};
+use openxr as xr;
+
+use crate::{backend::input::Pointer, state::AppState};
+
+type XrSession = xr::Session<xr::Vulkan>;
+
+pub(super) struct OpenXrInputSource {
+    action_set: xr::ActionSet,
+    hands: [OpenXrHand; 2],
+    pub(super) stage: xr::Space,
+}
+
+pub(super) struct OpenXrHand {
+    source: OpenXrHandSource,
+    space: xr::Space,
+}
+
+pub(super) struct OpenXrHandSource {
+    action_pose: xr::Action<xr::Posef>,
+    action_click: xr::Action<bool>,
+    action_grab: xr::Action<bool>,
+    action_scroll: xr::Action<xr::Vector2f>,
+    action_alt_click: xr::Action<bool>,
+    action_show_hide: xr::Action<bool>,
+    action_click_modifier_right: xr::Action<bool>,
+    action_click_modifier_middle: xr::Action<bool>,
+    action_haptics: xr::Action<xr::Haptic>,
+}
+
+impl OpenXrInputSource {
+    pub fn new(session: XrSession) -> Self {
+        let mut action_set = session
+            .instance()
+            .create_action_set("wlx-overlay-s", "WlxOverlay-S Actions", 0)
+            .expect("Failed to create action set");
+
+        let left_source = OpenXrHandSource::new(&mut action_set, "left");
+        let right_source = OpenXrHandSource::new(&mut action_set, "right");
+
+        session.attach_action_sets(&[&action_set]).unwrap();
+
+        let stage = session
+            .create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)
+            .unwrap();
+
+        Self {
+            action_set,
+            hands: [
+                OpenXrHand::new(session.clone(), left_source),
+                OpenXrHand::new(session, right_source),
+            ],
+            stage,
+        }
+    }
+
+    pub fn update(&self, session: &XrSession, time: xr::Time, state: &mut AppState) {
+        session.sync_actions(&[(&self.action_set).into()]).unwrap();
+
+        for i in 0..2 {
+            self.hands[i].update(
+                &mut state.input_state.pointers[i],
+                &self.stage,
+                session,
+                time,
+            );
+        }
+    }
+}
+
+impl OpenXrHand {
+    pub(super) fn new(session: XrSession, source: OpenXrHandSource) -> Self {
+        let space = source
+            .action_pose
+            .create_space(session.clone(), xr::Path::NULL, xr::Posef::IDENTITY)
+            .unwrap();
+
+        Self { source, space }
+    }
+
+    pub(super) fn update(
+        &self,
+        pointer: &mut Pointer,
+        stage: &xr::Space,
+        session: &XrSession,
+        time: xr::Time,
+    ) {
+        let location = self.space.locate(stage, time).unwrap();
+        if location
+            .location_flags
+            .contains(xr::SpaceLocationFlags::ORIENTATION_VALID)
+        {
+            let quat = unsafe { std::mem::transmute::<_, Quat>(location.pose.orientation) };
+            let pos = unsafe { std::mem::transmute::<_, Vec3>(location.pose.position) };
+            pointer.pose = Affine3A::from_rotation_translation(quat, pos);
+        }
+
+        pointer.now.click = self
+            .source
+            .action_click
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+
+        pointer.now.grab = self
+            .source
+            .action_grab
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+
+        pointer.now.scroll = self
+            .source
+            .action_scroll
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state
+            .y;
+
+        pointer.now.alt_click = self
+            .source
+            .action_alt_click
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+
+        pointer.now.show_hide = self
+            .source
+            .action_show_hide
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+
+        pointer.now.click_modifier_right = self
+            .source
+            .action_click_modifier_right
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+
+        pointer.now.click_modifier_middle = self
+            .source
+            .action_click_modifier_middle
+            .state(session, xr::Path::NULL)
+            .unwrap()
+            .current_state;
+    }
+}
+
+// supported action types: Haptic, Posef, Vector2f, f32, bool
+impl OpenXrHandSource {
+    pub(super) fn new(action_set: &mut xr::ActionSet, side: &str) -> Self {
+        let action_pose = action_set
+            .create_action::<xr::Posef>(
+                &format!("{}_hand", side),
+                &format!("{} hand pose", side),
+                &[],
+            )
+            .unwrap();
+
+        let action_click = action_set
+            .create_action::<bool>(
+                &format!("{}_click", side),
+                &format!("{} hand click", side),
+                &[],
+            )
+            .unwrap();
+        let action_grab = action_set
+            .create_action::<bool>(
+                &format!("{}_grab", side),
+                &format!("{} hand grab", side),
+                &[],
+            )
+            .unwrap();
+        let action_scroll = action_set
+            .create_action::<xr::Vector2f>(
+                &format!("{}_scroll", side),
+                &format!("{} hand scroll", side),
+                &[],
+            )
+            .unwrap();
+        let action_alt_click = action_set
+            .create_action::<bool>(
+                &format!("{}_alt_click", side),
+                &format!("{} hand alt click", side),
+                &[],
+            )
+            .unwrap();
+        let action_show_hide = action_set
+            .create_action::<bool>(
+                &format!("{}_show_hide", side),
+                &format!("{} hand show/hide", side),
+                &[],
+            )
+            .unwrap();
+        let action_click_modifier_right = action_set
+            .create_action::<bool>(
+                &format!("{}_click_modifier_right", side),
+                &format!("{} hand right click modifier", side),
+                &[],
+            )
+            .unwrap();
+        let action_click_modifier_middle = action_set
+            .create_action::<bool>(
+                &format!("{}_click_modifier_middle", side),
+                &format!("{} hand middle click modifier", side),
+                &[],
+            )
+            .unwrap();
+        let action_haptics = action_set
+            .create_action::<xr::Haptic>(
+                &format!("{}_haptics", side),
+                &format!("{} hand haptics", side),
+                &[],
+            )
+            .unwrap();
+
+        // TODO suggest bindings
+
+        Self {
+            action_pose,
+            action_click,
+            action_grab,
+            action_scroll,
+            action_alt_click,
+            action_show_hide,
+            action_click_modifier_right,
+            action_click_modifier_middle,
+            action_haptics,
+        }
+    }
+}

src/backend/openxr/mod.rs

@@ -0,0 +1,363 @@
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+
+use anyhow::{bail, ensure};
+use ash::vk::{self};
+use glam::{Affine3A, Quat, Vec3};
+use openxr as xr;
+use vulkano::{
+    image::{view::ImageView, ImageCreateInfo, ImageUsage},
+    render_pass::{Framebuffer, FramebufferCreateInfo},
+    Handle, VulkanObject,
+};
+
+use crate::{
+    backend::{common::OverlayContainer, input::interact, openxr::overlay::OpenXrOverlayData},
+    graphics::WlxGraphics,
+    state::AppState,
+};
+
+use super::common::BackendError;
+
+mod input;
+mod lines;
+mod overlay;
+
+const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
+const VIEW_COUNT: u32 = 2;
+
+pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
+    let (xr_instance, system) = match init_xr() {
+        Ok((xr_instance, system)) => (xr_instance, system),
+        Err(e) => {
+            log::warn!("Will not use OpenXR: {}", e);
+            return Err(BackendError::NotSupported);
+        }
+    };
+
+    let environment_blend_mode = xr_instance
+        .enumerate_environment_blend_modes(system, VIEW_TYPE)
+        .unwrap()[0];
+    log::info!("Using environment blend mode: {:?}", environment_blend_mode);
+
+    let mut state = {
+        let graphics = WlxGraphics::new_xr(xr_instance.clone(), system);
+        AppState::from_graphics(graphics)
+    };
+
+    let mut overlays = OverlayContainer::<OpenXrOverlayData>::new(&mut state);
+
+    state.hid_provider.set_desktop_extent(overlays.extent);
+
+    let (session, mut frame_wait, mut frame_stream) = unsafe {
+        xr_instance
+            .create_session::<xr::Vulkan>(
+                system,
+                &xr::vulkan::SessionCreateInfo {
+                    instance: state.graphics.instance.handle().as_raw() as _,
+                    physical_device: state.graphics.device.physical_device().handle().as_raw() as _,
+                    device: state.graphics.device.handle().as_raw() as _,
+                    queue_family_index: state.graphics.queue.queue_family_index(),
+                    queue_index: 0,
+                },
+            )
+            .unwrap()
+    };
+
+    let input_source = input::OpenXrInputSource::new(session.clone());
+
+    let mut swapchain = None;
+    let mut session_running = false;
+    let mut event_storage = xr::EventDataBuffer::new();
+
+    'main_loop: loop {
+        if !running.load(Ordering::Relaxed) {
+            log::warn!("Received shutdown signal.");
+            match session.request_exit() {
+                Ok(_) => log::info!("OpenXR session exit requested."),
+                Err(xr::sys::Result::ERROR_SESSION_NOT_RUNNING) => break 'main_loop,
+                Err(e) => {
+                    log::error!("Failed to request OpenXR session exit: {}", e);
+                    break 'main_loop;
+                }
+            }
+        }
+
+        while let Some(event) = xr_instance.poll_event(&mut event_storage).unwrap() {
+            use xr::Event::*;
+            match event {
+                SessionStateChanged(e) => {
+                    // Session state change is where we can begin and end sessions, as well as
+                    // find quit messages!
+                    println!("entered state {:?}", e.state());
+                    match e.state() {
+                        xr::SessionState::READY => {
+                            session.begin(VIEW_TYPE).unwrap();
+                            session_running = true;
+                        }
+                        xr::SessionState::STOPPING => {
+                            session.end().unwrap();
+                            session_running = false;
+                        }
+                        xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => {
+                            break 'main_loop;
+                        }
+                        _ => {}
+                    }
+                }
+                InstanceLossPending(_) => {
+                    break 'main_loop;
+                }
+                EventsLost(e) => {
+                    println!("lost {} events", e.lost_event_count());
+                }
+                _ => {}
+            }
+        }
+
+        if !session_running {
+            std::thread::sleep(Duration::from_millis(100));
+            continue 'main_loop;
+        }
+
+        let xr_frame_state = frame_wait.wait().unwrap();
+        frame_stream.begin().unwrap();
+
+        if !xr_frame_state.should_render {
+            frame_stream
+                .end(
+                    xr_frame_state.predicted_display_time,
+                    environment_blend_mode,
+                    &[],
+                )
+                .unwrap();
+            continue 'main_loop;
+        }
+
+        state.input_state.pre_update();
+        input_source.update(&session, xr_frame_state.predicted_display_time, &mut state);
+        state.input_state.post_update();
+
+        let (_, views) = session
+            .locate_views(
+                VIEW_TYPE,
+                xr_frame_state.predicted_display_time,
+                &input_source.stage,
+            )
+            .unwrap();
+
+        state.input_state.hmd = hmd_pose_from_views(&views);
+
+        let _pointer_lengths = interact(&mut overlays, &mut state);
+
+        //TODO lines
+
+        overlays
+            .iter_mut()
+            .filter(|o| o.state.want_visible)
+            .for_each(|o| o.render(&mut state));
+
+        state.hid_provider.on_new_frame();
+
+        let swapchain = swapchain.get_or_insert_with(|| {
+            let views = xr_instance
+                .enumerate_view_configuration_views(system, VIEW_TYPE)
+                .unwrap();
+            debug_assert_eq!(views.len(), VIEW_COUNT as usize);
+            debug_assert_eq!(views[0], views[1]);
+
+            let resolution = vk::Extent2D {
+                width: views[0].recommended_image_rect_width,
+                height: views[0].recommended_image_rect_height,
+            };
+            log::info!(
+                "Swapchain resolution: {}x{}",
+                resolution.width,
+                resolution.height
+            );
+            let swapchain = session
+                .create_swapchain(&xr::SwapchainCreateInfo {
+                    create_flags: xr::SwapchainCreateFlags::EMPTY,
+                    usage_flags: xr::SwapchainUsageFlags::COLOR_ATTACHMENT
+                        | xr::SwapchainUsageFlags::SAMPLED,
+                    format: state.graphics.native_format as _,
+                    sample_count: 1,
+                    width: resolution.width,
+                    height: resolution.height,
+                    face_count: 1,
+                    array_size: VIEW_COUNT,
+                    mip_count: 1,
+                })
+                .unwrap();
+
+            // thanks @yshui
+            let swapchain_images = swapchain
+                .enumerate_images()
+                .unwrap()
+                .into_iter()
+                .map(|handle| {
+                    let vk_image = vk::Image::from_raw(handle);
+                    let raw_image = unsafe {
+                        vulkano::image::sys::RawImage::from_handle(
+                            state.graphics.device.clone(),
+                            vk_image,
+                            ImageCreateInfo {
+                                format: state.graphics.native_format,
+                                extent: [resolution.width * 2, resolution.height, 1],
+                                usage: ImageUsage::COLOR_ATTACHMENT | ImageUsage::TRANSFER_DST,
+                                ..Default::default()
+                            },
+                        )
+                        .unwrap()
+                    };
+                    // SAFETY: OpenXR guarantees that the image is a swapchain image, thus has memory backing it.
+                    let image = Arc::new(unsafe { raw_image.assume_bound() });
+                    let view = ImageView::new_default(image).unwrap();
+                    let fb = Framebuffer::new(
+                        todo!(),
+                        FramebufferCreateInfo {
+                            attachments: vec![view.clone()],
+                            ..Default::default()
+                        },
+                    )
+                    .unwrap();
+
+                    XrFramebuffer {
+                        framebuffer: fb,
+                        color: view,
+                    }
+                })
+                .collect();
+
+            XrSwapchain {
+                handle: swapchain,
+                buffers: swapchain_images,
+                resolution: [resolution.width, resolution.height, 1],
+            }
+        });
+
+        let image_index = swapchain.handle.acquire_image().unwrap();
+    }
+
+    Ok(())
+}
+
+fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
+    let Ok(entry) = (unsafe { xr::Entry::load() }) else {
+        bail!("OpenXR Loader not found.");
+    };
+
+    let Ok(available_extensions) = entry.enumerate_extensions() else {
+        bail!("Failed to enumerate OpenXR extensions.");
+    };
+    ensure!(
+        available_extensions.khr_vulkan_enable2,
+        "Missing KHR_vulkan_enable2 extension."
+    );
+    ensure!(
+        available_extensions.extx_overlay,
+        "Missing EXTX_overlay extension."
+    );
+
+    let mut enabled_extensions = xr::ExtensionSet::default();
+    enabled_extensions.khr_vulkan_enable2 = true;
+    enabled_extensions.extx_overlay = true;
+
+    #[cfg(not(debug_assertions))]
+    let layers = [];
+    #[cfg(debug_assertions)]
+    let layers = [
+        "XR_APILAYER_LUNARG_api_dump",
+        "XR_APILAYER_LUNARG_standard_validation",
+    ];
+
+    let Ok(xr_instance) = entry.create_instance(
+        &xr::ApplicationInfo {
+            application_name: "wlx-overlay-s",
+            application_version: 0,
+            engine_name: "wlx-overlay-s",
+            engine_version: 0,
+        },
+        &enabled_extensions,
+        &layers,
+    ) else {
+        bail!("Failed to create OpenXR instance.");
+    };
+
+    let Ok(instance_props) = xr_instance.properties() else {
+        bail!("Failed to query OpenXR instance properties.");
+    };
+    log::info!(
+        "Using OpenXR runtime: {} {}",
+        instance_props.runtime_name,
+        instance_props.runtime_version
+    );
+
+    let Ok(system) = xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY) else {
+        bail!("Failed to access OpenXR HMD system.");
+    };
+
+    let vk_target_version_xr = xr::Version::new(1, 1, 0);
+
+    let Ok(reqs) = xr_instance.graphics_requirements::<xr::Vulkan>(system) else {
+        bail!("Failed to query OpenXR Vulkan requirements.");
+    };
+
+    if vk_target_version_xr < reqs.min_api_version_supported
+        || vk_target_version_xr.major() > reqs.max_api_version_supported.major()
+    {
+        bail!(
+            "OpenXR runtime requires Vulkan version > {}, < {}.0.0",
+            reqs.min_api_version_supported,
+            reqs.max_api_version_supported.major() + 1
+        );
+    }
+
+    Ok((xr_instance, system))
+}
+
+struct XrSwapchain {
+    handle: xr::Swapchain<xr::Vulkan>,
+    buffers: Vec<XrFramebuffer>,
+    resolution: [u32; 3],
+}
+
+struct XrFramebuffer {
+    framebuffer: Arc<Framebuffer>,
+    color: Arc<ImageView>,
+}
+
+fn hmd_pose_from_views(views: &Vec<xr::View>) -> Affine3A {
+    let pos = {
+        let pos0: Vec3 = unsafe { std::mem::transmute(views[0].pose.position) };
+        let pos1: Vec3 = unsafe { std::mem::transmute(views[1].pose.position) };
+        (pos0 + pos1) * 0.5
+    };
+    let rot = {
+        let rot0 = unsafe { std::mem::transmute(views[0].pose.orientation) };
+        let rot1 = unsafe { std::mem::transmute(views[1].pose.orientation) };
+        quat_lerp(rot0, rot1, 0.5)
+    };
+
+    Affine3A::from_rotation_translation(rot, pos)
+}
+
+fn quat_lerp(a: Quat, mut b: Quat, t: f32) -> Quat {
+    let l2 = a.dot(b);
+    if l2 < 0.0 {
+        b = -b;
+    }
+
+    Quat::from_xyzw(
+        a.x - t * (a.x - b.x),
+        a.y - t * (a.y - b.y),
+        a.z - t * (a.z - b.z),
+        a.w - t * (a.w - b.w),
+    )
+    .normalize()
+}

src/backend/openxr/overlay.rs

@@ -0,0 +1,7 @@
+pub struct OpenXrOverlayData {}
+
+impl Default for OpenXrOverlayData {
+    fn default() -> Self {
+        Self {}
+    }
+}

src/graphics.rs

@@ -6,7 +6,7 @@ use std::{
     sync::Arc,
 };
 
-use ash::vk::SubmitInfo;
+use ash::vk::{self, SubmitInfo};
 use smallvec::{smallvec, SmallVec};
 use vulkano::{
     buffer::{
@@ -15,15 +15,14 @@ use vulkano::{
     },
     command_buffer::{
         allocator::{StandardCommandBufferAllocator, StandardCommandBufferAllocatorCreateInfo},
-        sys::{CommandBufferBeginInfo, UnsafeCommandBufferBuilder},
+        sys::{CommandBufferBeginInfo, RawRecordingCommandBuffer},
-        AutoCommandBufferBuilder, CommandBufferExecFuture, CommandBufferInheritanceInfo,
+        CommandBuffer, CommandBufferExecFuture, CommandBufferInheritanceInfo,
         CommandBufferInheritanceRenderPassInfo, CommandBufferInheritanceRenderPassType,
-        CommandBufferLevel, CommandBufferUsage, CopyBufferToImageInfo, PrimaryAutoCommandBuffer,
+        CommandBufferLevel, CommandBufferUsage, CopyBufferToImageInfo, RecordingCommandBuffer,
-        PrimaryCommandBufferAbstract, RenderPassBeginInfo, SecondaryAutoCommandBuffer,
+        RenderPassBeginInfo, SubpassBeginInfo, SubpassContents, SubpassEndInfo,
-        SubpassBeginInfo, SubpassContents, SubpassEndInfo,
     },
     descriptor_set::{
-        allocator::StandardDescriptorSetAllocator, PersistentDescriptorSet, WriteDescriptorSet,
+        allocator::StandardDescriptorSetAllocator, DescriptorSet, WriteDescriptorSet,
     },
     device::{
         physical::{PhysicalDevice, PhysicalDeviceType},
@@ -34,8 +33,8 @@ use vulkano::{
         sampler::{Filter, Sampler, SamplerAddressMode, SamplerCreateInfo},
         sys::RawImage,
         view::ImageView,
-        Image, ImageCreateInfo, ImageLayout, ImageTiling, ImageType, ImageUsage, SampleCount,
+        Image, ImageCreateFlags, ImageCreateInfo, ImageLayout, ImageTiling, ImageType, ImageUsage,
-        SubresourceLayout,
+        SampleCount, SubresourceLayout,
     },
     instance::{Instance, InstanceCreateFlags, InstanceCreateInfo, InstanceExtensions},
     memory::{
@@ -65,19 +64,15 @@ use vulkano::{
         SubpassDescription,
     },
     shader::ShaderModule,
-    swapchain::{CompositeAlpha, Surface, Swapchain, SwapchainCreateInfo},
     sync::{
         fence::Fence, future::NowFuture, AccessFlags, DependencyInfo, GpuFuture,
         ImageMemoryBarrier, PipelineStages,
     },
     DeviceSize, VulkanLibrary, VulkanObject,
 };
-use winit::{
-    event_loop::EventLoop,
-    window::{Window, WindowBuilder},
-};
 use wlx_capture::frame::{
-    DmabufFrame, DRM_FORMAT_ABGR8888, DRM_FORMAT_ARGB8888, DRM_FORMAT_XBGR8888, DRM_FORMAT_XRGB8888,
+    DmabufFrame, FourCC, DRM_FORMAT_ABGR8888, DRM_FORMAT_ARGB8888, DRM_FORMAT_XBGR8888,
+    DRM_FORMAT_XRGB8888,
 };
 
 #[repr(C)]
@@ -96,7 +91,7 @@ pub struct WlxGraphics {
     pub device: Arc<Device>,
     pub queue: Arc<Queue>,
 
-    pub surface: Arc<Surface>,
+    pub native_format: Format,
 
     pub memory_allocator: Arc<StandardMemoryAllocator>,
     pub command_buffer_allocator: Arc<StandardCommandBufferAllocator>,
@@ -107,38 +102,155 @@ pub struct WlxGraphics {
 }
 
 impl WlxGraphics {
+    #[cfg(feature = "openxr")]
+    pub fn new_xr(xr_instance: openxr::Instance, system: openxr::SystemId) -> Arc<Self> {
+        use vulkano::Handle;
+
+        let vk_target_version = vk::make_api_version(0, 1, 1, 0); // Vulkan 1.1 guarantees multiview support
+        let target_version = vulkano::Version::V1_1;
+        let library = VulkanLibrary::new().unwrap();
+        let vk_entry = unsafe { ash::Entry::load().unwrap() };
+
+        let vk_app_info = vk::ApplicationInfo::builder()
+            .application_version(0)
+            .engine_version(0)
+            .api_version(vk_target_version);
+
+        let instance = unsafe {
+            let vk_instance = xr_instance
+                .create_vulkan_instance(
+                    system,
+                    std::mem::transmute(vk_entry.static_fn().get_instance_proc_addr),
+                    &vk::InstanceCreateInfo::builder().application_info(&vk_app_info) as *const _
+                        as *const _,
+                )
+                .expect("XR error creating Vulkan instance")
+                .map_err(vk::Result::from_raw)
+                .expect("Vulkan error creating Vulkan instance");
+
+            Instance::from_handle(
+                library,
+                ash::vk::Instance::from_raw(vk_instance as _),
+                InstanceCreateInfo::default(), // FIXME
+            )
+        };
+
+        let physical_device = unsafe {
+            PhysicalDevice::from_handle(
+                instance.clone(),
+                vk::PhysicalDevice::from_raw(
+                    xr_instance
+                        .vulkan_graphics_device(system, instance.handle().as_raw() as _)
+                        .unwrap() as _,
+                ),
+            )
+        }
+        .unwrap();
+
+        let vk_device_properties = physical_device.properties();
+        if vk_device_properties.api_version < target_version {
+            panic!(
+                "Vulkan physical device doesn't support Vulkan {}",
+                target_version
+            );
+        }
+
+        log::info!(
+            "Using vkPhysicalDevice: {}",
+            physical_device.properties().device_name,
+        );
+
+        let queue_family_index = physical_device
+            .queue_family_properties()
+            .iter()
+            .enumerate()
+            .position(|(_, q)| q.queue_flags.intersects(QueueFlags::GRAPHICS))
+            .expect("Vulkan device has no graphics queue") as u32;
+
+        let (device, mut queues) = unsafe {
+            let vk_device = xr_instance
+                .create_vulkan_device(
+                    system,
+                    std::mem::transmute(vk_entry.static_fn().get_instance_proc_addr),
+                    physical_device.handle().as_raw() as _,
+                    &vk::DeviceCreateInfo::builder()
+                        .queue_create_infos(&[vk::DeviceQueueCreateInfo::builder()
+                            .queue_family_index(queue_family_index)
+                            .queue_priorities(&[1.0])
+                            .build()])
+                        .push_next(&mut vk::PhysicalDeviceMultiviewFeatures {
+                            multiview: vk::TRUE,
+                            ..Default::default()
+                        }) as *const _ as *const _,
+                )
+                .expect("XR error creating Vulkan device")
+                .map_err(vk::Result::from_raw)
+                .expect("Vulkan error creating Vulkan device");
+
+            vulkano::device::Device::from_handle(
+                physical_device,
+                vk::Device::from_raw(vk_device as _),
+                DeviceCreateInfo::default(), // FIXME
+            )
+        };
+
+        let queue = queues.next().unwrap();
+
+        let memory_allocator = Arc::new(StandardMemoryAllocator::new_default(device.clone()));
+        let command_buffer_allocator = Arc::new(StandardCommandBufferAllocator::new(
+            device.clone(),
+            StandardCommandBufferAllocatorCreateInfo {
+                secondary_buffer_count: 32,
+                ..Default::default()
+            },
+        ));
+        let descriptor_set_allocator = Arc::new(StandardDescriptorSetAllocator::new(
+            device.clone(),
+            Default::default(),
+        ));
+
+        let (quad_verts, quad_indices) = Self::default_quad(memory_allocator.clone());
+
+        let me = Self {
+            instance,
+            device,
+            queue,
+            native_format: Format::R8G8B8A8_UNORM,
+            memory_allocator,
+            command_buffer_allocator,
+            descriptor_set_allocator,
+            quad_indices,
+            quad_verts,
+        };
+
+        Arc::new(me)
+    }
+
     pub fn new(
         vk_instance_extensions: InstanceExtensions,
         mut vk_device_extensions_fn: impl FnMut(&PhysicalDevice) -> DeviceExtensions,
-    ) -> (Arc<Self>, EventLoop<()>) {
+    ) -> Arc<Self> {
         #[cfg(debug_assertions)]
         let layers = vec!["VK_LAYER_KHRONOS_validation".to_owned()];
         #[cfg(not(debug_assertions))]
         let layers = vec![];
 
-        // TODO headless
-        let event_loop = EventLoop::new();
-        let library_extensions = Surface::required_extensions(&event_loop);
-
         let library = VulkanLibrary::new().unwrap();
-        let required_extensions = library_extensions.union(&vk_instance_extensions);
 
-        log::debug!("Instance exts for app: {:?}", &required_extensions);
         log::debug!("Instance exts for runtime: {:?}", &vk_instance_extensions);
 
         let instance = Instance::new(
             library,
             InstanceCreateInfo {
                 flags: InstanceCreateFlags::ENUMERATE_PORTABILITY,
-                enabled_extensions: required_extensions,
+                enabled_extensions: vk_instance_extensions,
                 enabled_layers: layers,
                 ..Default::default()
             },
         )
         .unwrap();
 
-        let mut device_extensions = DeviceExtensions {
+        let device_extensions = DeviceExtensions {
-            khr_swapchain: true,
             khr_external_memory: true,
             khr_external_memory_fd: true,
             ext_external_memory_dma_buf: true,
@@ -148,10 +260,6 @@ impl WlxGraphics {
 
         log::debug!("Device exts for app: {:?}", &device_extensions);
 
-        // TODO headless
-        let window = Arc::new(WindowBuilder::new().build(&event_loop).unwrap());
-        let surface = Surface::from_window(instance.clone(), window.clone()).unwrap();
-
         let (physical_device, my_extensions, queue_family_index) = instance
             .enumerate_physical_devices()
             .unwrap()
@@ -176,10 +284,7 @@ impl WlxGraphics {
                 p.queue_family_properties()
                     .iter()
                     .enumerate()
-                    .position(|(i, q)| {
+                    .position(|(_, q)| q.queue_flags.intersects(QueueFlags::GRAPHICS))
-                        q.queue_flags.intersects(QueueFlags::GRAPHICS)
-                            && p.surface_support(i as u32, &surface).unwrap_or(false)
-                    })
                     .map(|i| (p, my_extensions, i as u32))
             })
             .min_by_key(|(p, _, _)| match p.properties().device_type {
@@ -233,6 +338,26 @@ impl WlxGraphics {
             Default::default(),
         ));
 
+        let (quad_verts, quad_indices) = Self::default_quad(memory_allocator.clone());
+
+        let me = Self {
+            instance,
+            device,
+            queue,
+            memory_allocator,
+            native_format: Format::R8G8B8A8_UNORM,
+            command_buffer_allocator,
+            descriptor_set_allocator,
+            quad_indices,
+            quad_verts,
+        };
+
+        Arc::new(me)
+    }
+
+    fn default_quad(
+        memory_allocator: Arc<StandardMemoryAllocator>,
+    ) -> (Subbuffer<[Vert2Uv]>, Subbuffer<[u16]>) {
         let vertices = [
             Vert2Uv {
                 in_pos: [0., 0.],
@@ -267,7 +392,7 @@ impl WlxGraphics {
         .unwrap();
 
         let quad_indices = Buffer::from_iter(
-            memory_allocator.clone(),
+            memory_allocator,
             BufferCreateInfo {
                 usage: BufferUsage::INDEX_BUFFER,
                 ..Default::default()
@@ -281,72 +406,7 @@ impl WlxGraphics {
         )
         .unwrap();
 
-        let me = Self {
+        (quad_verts, quad_indices)
-            instance,
-            device,
-            queue,
-            surface,
-            memory_allocator,
-            command_buffer_allocator,
-            descriptor_set_allocator,
-            quad_indices,
-            quad_verts,
-        };
-
-        (Arc::new(me), event_loop)
-    }
-
-    #[allow(dead_code)]
-    pub fn create_swapchain(&self, format: Option<Format>) -> (Arc<Swapchain>, Vec<Arc<Image>>) {
-        let (min_image_count, composite_alpha, image_format) = if let Some(format) = format {
-            (1, CompositeAlpha::Opaque, format)
-        } else {
-            let surface_capabilities = self
-                .device
-                .physical_device()
-                .surface_capabilities(&self.surface, Default::default())
-                .unwrap();
-
-            let composite_alpha = surface_capabilities
-                .supported_composite_alpha
-                .into_iter()
-                .next()
-                .unwrap();
-
-            let image_format = Some(
-                self.device
-                    .physical_device()
-                    .surface_formats(&self.surface, Default::default())
-                    .unwrap()[0]
-                    .0,
-            );
-            (
-                surface_capabilities.min_image_count,
-                composite_alpha,
-                image_format.unwrap(),
-            )
-        };
-        let window = self
-            .surface
-            .object()
-            .unwrap()
-            .downcast_ref::<Window>()
-            .unwrap();
-        let swapchain = Swapchain::new(
-            self.device.clone(),
-            self.surface.clone(),
-            SwapchainCreateInfo {
-                min_image_count,
-                image_format,
-                image_extent: window.inner_size().into(),
-                image_usage: ImageUsage::COLOR_ATTACHMENT,
-                composite_alpha,
-                ..Default::default()
-            },
-        )
-        .unwrap();
-
-        swapchain
     }
 
     pub fn upload_verts(
@@ -411,13 +471,7 @@ impl WlxGraphics {
     pub fn dmabuf_texture(&self, frame: DmabufFrame) -> Option<Arc<Image>> {
         let extent = [frame.format.width, frame.format.height, 1];
 
-        let format = match frame.format.fourcc {
+        let format = fourcc_to_vk(frame.format.fourcc);
-            DRM_FORMAT_ABGR8888 => Format::R8G8B8A8_UNORM,
-            DRM_FORMAT_XBGR8888 => Format::R8G8B8A8_UNORM,
-            DRM_FORMAT_ARGB8888 => Format::B8G8R8A8_UNORM,
-            DRM_FORMAT_XRGB8888 => Format::B8G8R8A8_UNORM,
-            _ => panic!("Unsupported dmabuf format {:x}", frame.format.fourcc),
-        };
 
         let layouts: Vec<SubresourceLayout> = (0..frame.num_planes)
             .into_iter()
@@ -435,6 +489,12 @@ impl WlxGraphics {
 
         let external_memory_handle_types = ExternalMemoryHandleTypes::DMA_BUF;
 
+        let flags = if frame.num_planes > 9 {
+            ImageCreateFlags::DISJOINT
+        } else {
+            ImageCreateFlags::empty()
+        };
+
         let image = RawImage::new(
             self.device.clone(),
             ImageCreateInfo {
@@ -444,6 +504,7 @@ impl WlxGraphics {
                 usage: ImageUsage::SAMPLED | ImageUsage::TRANSFER_SRC,
                 external_memory_handle_types,
                 tiling: ImageTiling::DrmFormatModifier,
+                flags,
                 drm_format_modifiers: vec![frame.format.modifier],
                 drm_format_modifier_plane_layouts: layouts,
                 ..Default::default()
@@ -464,11 +525,9 @@ impl WlxGraphics {
         debug_assert!(self.device.enabled_extensions().khr_external_memory);
         debug_assert!(self.device.enabled_extensions().ext_external_memory_dma_buf);
 
-        let memory = unsafe {
+        let mut allocations: SmallVec<[ResourceMemory; 4]> = smallvec![];
-            if frame.num_planes != 1 {
+
-                log::error!("Unsupported number of DMA-buf planes: {}", frame.num_planes);
+        unsafe {
-                return None;
-            }
             let Some(fd) = frame.planes[0].fd else {
                 log::error!("DMA-buf plane has no FD");
                 return None;
@@ -478,7 +537,7 @@ impl WlxGraphics {
             let new_file = file.try_clone().unwrap();
             file.into_raw_fd();
 
-            DeviceMemory::import(
+            let memory = DeviceMemory::import(
                 self.device.clone(),
                 MemoryAllocateInfo {
                     allocation_size: requirements.layout.size(),
@@ -491,16 +550,17 @@ impl WlxGraphics {
                     handle_type: ExternalMemoryHandleType::DmaBuf,
                 },
             )
-            .unwrap()
+            .unwrap();
-        };
 
-        let allocations: SmallVec<[ResourceMemory; 1]> =
+            allocations.push(ResourceMemory::new_dedicated(memory));
-            smallvec![ResourceMemory::new_dedicated(memory)];
+        }
 
-        if let Some(image) = image.bind_memory(allocations).ok() {
+        match unsafe { image.bind_memory(allocations) } {
-            Some(Arc::new(image))
+            Ok(image) => Some(Arc::new(image)),
-        } else {
+            Err(e) => {
-            None
+                log::warn!("Failed to bind memory to image: {}", e.0.to_string());
+                return None;
+            }
         }
     }
 
@@ -558,10 +618,15 @@ impl WlxGraphics {
     }
 
     pub fn create_command_buffer(self: &Arc<Self>, usage: CommandBufferUsage) -> WlxCommandBuffer {
-        let command_buffer = AutoCommandBufferBuilder::primary(
+        let command_buffer = RecordingCommandBuffer::new(
-            &self.command_buffer_allocator,
+            self.command_buffer_allocator.clone(),
             self.queue.queue_family_index(),
-            usage,
+            CommandBufferLevel::Primary,
+            CommandBufferBeginInfo {
+                usage,
+                inheritance_info: None,
+                ..Default::default()
+            },
         )
         .unwrap();
         WlxCommandBuffer {
@@ -588,8 +653,8 @@ impl WlxGraphics {
         };
 
         let command_buffer = unsafe {
-            let mut builder = UnsafeCommandBufferBuilder::new(
+            let mut builder = RawRecordingCommandBuffer::new(
-                &self.command_buffer_allocator,
+                self.command_buffer_allocator.clone(),
                 self.queue.queue_family_index(),
                 CommandBufferLevel::Primary,
                 CommandBufferBeginInfo {
@@ -606,7 +671,7 @@ impl WlxGraphics {
                     ..Default::default()
                 })
                 .unwrap();
-            builder.build().unwrap()
+            builder.end().unwrap()
         };
 
         let fence = vulkano::sync::fence::Fence::new(
@@ -636,10 +701,7 @@ impl WlxGraphics {
 
 pub struct WlxCommandBuffer {
     graphics: Arc<WlxGraphics>,
-    command_buffer: AutoCommandBufferBuilder<
+    command_buffer: RecordingCommandBuffer,
-        PrimaryAutoCommandBuffer<Arc<StandardCommandBufferAllocator>>,
-        Arc<StandardCommandBufferAllocator>,
-    >,
 }
 
 impl WlxCommandBuffer {
@@ -672,7 +734,7 @@ impl WlxCommandBuffer {
         width: u32,
         height: u32,
         format: Format,
-        data: Vec<u8>,
+        data: &[u8],
     ) -> Arc<Image> {
         let image = Image::new(
             self.graphics.memory_allocator.clone(),
@@ -702,7 +764,7 @@ impl WlxCommandBuffer {
         )
         .unwrap();
 
-        buffer.write().unwrap().copy_from_slice(data.as_slice());
+        buffer.write().unwrap().copy_from_slice(data);
 
         self.command_buffer
             .copy_buffer_to_image(CopyBufferToImageInfo::buffer_image(buffer, image.clone()))
@@ -722,7 +784,7 @@ impl WlxCommandBuffer {
         let mut image_data = Vec::new();
         image_data.resize((info.width * info.height * 4) as usize, 0);
         reader.next_frame(&mut image_data).unwrap();
-        self.texture2d(width, height, Format::R8G8B8A8_UNORM, image_data)
+        self.texture2d(width, height, Format::R8G8B8A8_UNORM, &image_data)
     }
 }
 
@@ -734,8 +796,8 @@ impl WlxCommandBuffer {
         self
     }
 
-    pub fn build(self) -> Arc<PrimaryAutoCommandBuffer<Arc<StandardCommandBufferAllocator>>> {
+    pub fn build(self) -> Arc<CommandBuffer> {
-        self.command_buffer.build().unwrap()
+        self.command_buffer.end().unwrap()
     }
 
     pub fn build_and_execute(self) -> CommandBufferExecFuture<NowFuture> {
@@ -894,6 +956,12 @@ impl WlxPipeline {
         }
     }
 
+    pub fn swap_framebuffer(&mut self, new_framebuffer: Arc<Framebuffer>) -> Arc<Framebuffer> {
+        let old = self.framebuffer.clone();
+        self.framebuffer = new_framebuffer;
+        old
+    }
+
     pub fn inner(&self) -> Arc<GraphicsPipeline> {
         self.pipeline.clone()
     }
@@ -903,7 +971,7 @@ impl WlxPipeline {
         set: usize,
         texture: Arc<ImageView>,
         filter: Filter,
-    ) -> Arc<PersistentDescriptorSet> {
+    ) -> Arc<DescriptorSet> {
         let sampler = Sampler::new(
             self.graphics.device.clone(),
             SamplerCreateInfo {
@@ -917,8 +985,8 @@ impl WlxPipeline {
 
         let layout = self.pipeline.layout().set_layouts().get(set).unwrap();
 
-        PersistentDescriptorSet::new(
+        DescriptorSet::new(
-            &self.graphics.descriptor_set_allocator,
+            self.graphics.descriptor_set_allocator.clone(),
             layout.clone(),
             [WriteDescriptorSet::image_view_sampler(0, texture, sampler)],
             [],
@@ -926,7 +994,7 @@ impl WlxPipeline {
         .unwrap()
     }
 
-    pub fn uniform_buffer<T>(&self, set: usize, data: Vec<T>) -> Arc<PersistentDescriptorSet>
+    pub fn uniform_buffer<T>(&self, set: usize, data: Vec<T>) -> Arc<DescriptorSet>
     where
         T: BufferContents + Copy,
     {
@@ -947,8 +1015,8 @@ impl WlxPipeline {
         };
 
         let layout = self.pipeline.layout().set_layouts().get(set).unwrap();
-        PersistentDescriptorSet::new(
+        DescriptorSet::new(
-            &self.graphics.descriptor_set_allocator,
+            self.graphics.descriptor_set_allocator.clone(),
             layout.clone(),
             [WriteDescriptorSet::buffer(0, uniform_buffer_subbuffer)],
             [],
@@ -961,7 +1029,7 @@ impl WlxPipeline {
         dimensions: [f32; 2],
         vertex_buffer: Subbuffer<[Vert2Uv]>,
         index_buffer: Subbuffer<[u16]>,
-        descriptor_sets: Vec<Arc<PersistentDescriptorSet>>,
+        descriptor_sets: Vec<Arc<DescriptorSet>>,
     ) -> WlxPass {
         WlxPass::new(
             self.clone(),
@@ -978,8 +1046,8 @@ pub struct WlxPass {
     pipeline: Arc<WlxPipeline>,
     vertex_buffer: Subbuffer<[Vert2Uv]>,
     index_buffer: Subbuffer<[u16]>,
-    descriptor_sets: Vec<Arc<PersistentDescriptorSet>>,
+    descriptor_sets: Vec<Arc<DescriptorSet>>,
-    pub command_buffer: Arc<SecondaryAutoCommandBuffer<Arc<StandardCommandBufferAllocator>>>,
+    pub command_buffer: Arc<CommandBuffer>,
 }
 
 impl WlxPass {
@@ -988,7 +1056,7 @@ impl WlxPass {
         dimensions: [f32; 2],
         vertex_buffer: Subbuffer<[Vert2Uv]>,
         index_buffer: Subbuffer<[u16]>,
-        descriptor_sets: Vec<Arc<PersistentDescriptorSet>>,
+        descriptor_sets: Vec<Arc<DescriptorSet>>,
     ) -> Self {
         let viewport = Viewport {
             offset: [0.0, 0.0],
@@ -997,53 +1065,69 @@ impl WlxPass {
         };
 
         let pipeline_inner = pipeline.inner().clone();
-        let mut command_buffer = AutoCommandBufferBuilder::secondary(
+        let mut command_buffer = RecordingCommandBuffer::new(
-            &pipeline.graphics.command_buffer_allocator,
+            pipeline.graphics.command_buffer_allocator.clone(),
             pipeline.graphics.queue.queue_family_index(),
-            CommandBufferUsage::MultipleSubmit,
+            CommandBufferLevel::Secondary,
-            CommandBufferInheritanceInfo {
+            CommandBufferBeginInfo {
-                render_pass: Some(CommandBufferInheritanceRenderPassType::BeginRenderPass(
+                usage: CommandBufferUsage::MultipleSubmit,
-                    CommandBufferInheritanceRenderPassInfo {
+                inheritance_info: Some(CommandBufferInheritanceInfo {
-                        subpass: Subpass::from(pipeline.render_pass.clone(), 0).unwrap(),
+                    render_pass: Some(CommandBufferInheritanceRenderPassType::BeginRenderPass(
-                        framebuffer: None,
+                        CommandBufferInheritanceRenderPassInfo {
-                    },
+                            subpass: Subpass::from(pipeline.render_pass.clone(), 0).unwrap(),
-                )),
+                            framebuffer: None,
+                        },
+                    )),
+                    ..Default::default()
+                }),
                 ..Default::default()
             },
         )
         .unwrap();
 
-        command_buffer
+        unsafe {
-            .set_viewport(0, smallvec![viewport])
+            command_buffer
-            .unwrap()
+                .set_viewport(0, smallvec![viewport])
-            .bind_pipeline_graphics(pipeline_inner)
+                .unwrap()
-            .unwrap()
+                .bind_pipeline_graphics(pipeline_inner)
-            .bind_descriptor_sets(
+                .unwrap()
-                PipelineBindPoint::Graphics,
+                .bind_descriptor_sets(
-                pipeline.inner().layout().clone(),
+                    PipelineBindPoint::Graphics,
-                0,
+                    pipeline.inner().layout().clone(),
-                descriptor_sets.clone(),
+                    0,
-            )
+                    descriptor_sets.clone(),
-            .unwrap()
+                )
-            .bind_vertex_buffers(0, vertex_buffer.clone())
+                .unwrap()
-            .unwrap()
+                .bind_vertex_buffers(0, vertex_buffer.clone())
-            .bind_index_buffer(index_buffer.clone())
+                .unwrap()
-            .unwrap()
+                .bind_index_buffer(index_buffer.clone())
-            .draw_indexed(index_buffer.len() as u32, 1, 0, 0, 0)
+                .unwrap()
-            .or_else(|err| {
+                .draw_indexed(index_buffer.len() as u32, 1, 0, 0, 0)
-                if let Some(source) = err.source() {
+                .or_else(|err| {
-                    log::error!("Failed to draw: {}", source);
+                    if let Some(source) = err.source() {
-                }
+                        log::error!("Failed to draw: {}", source);
-                Err(err)
+                    }
-            })
+                    Err(err)
-            .unwrap();
+                })
+                .unwrap()
+        };
 
         Self {
             pipeline,
             vertex_buffer,
             index_buffer,
             descriptor_sets,
-            command_buffer: command_buffer.build().unwrap(),
+            command_buffer: command_buffer.end().unwrap(),
         }
     }
 }
+
+pub fn fourcc_to_vk(fourcc: FourCC) -> Format {
+    match fourcc.value {
+        DRM_FORMAT_ABGR8888 => Format::R8G8B8A8_UNORM,
+        DRM_FORMAT_XBGR8888 => Format::R8G8B8A8_UNORM,
+        DRM_FORMAT_ARGB8888 => Format::B8G8R8A8_UNORM,
+        DRM_FORMAT_XRGB8888 => Format::B8G8R8A8_UNORM,
+        _ => panic!("Unsupported memfd format {}", fourcc),
+    }
+}

src/gui/font.rs

@@ -201,7 +201,7 @@ impl FontCache {
         };
 
         let mut cmd_buffer = graphics.create_command_buffer(CommandBufferUsage::OneTimeSubmit);
-        let texture = cmd_buffer.texture2d(bmp.width() as _, bmp.rows() as _, format, buf);
+        let texture = cmd_buffer.texture2d(bmp.width() as _, bmp.rows() as _, format, &buf);
         cmd_buffer.build_and_execute_now();
 
         let g = Glyph {

src/main.rs

@@ -7,7 +7,11 @@ mod overlays;
 mod shaders;
 mod state;
 
-use crate::backend::openvr::openvr_run;
+use std::sync::{
+    atomic::{AtomicBool, Ordering},
+    Arc,
+};
+
 use env_logger::Env;
 
 fn main() {
@@ -18,5 +22,40 @@ fn main() {
         env!("CARGO_PKG_VERSION")
     );
 
-    openvr_run();
+    let running = Arc::new(AtomicBool::new(true));
+    let _ = ctrlc::set_handler({
+        let running = running.clone();
+        move || {
+            running.store(false, Ordering::Relaxed);
+        }
+    });
+
+    #[cfg(all(feature = "openxr", feature = "openvr"))]
+    auto_run(running);
+
+    #[cfg(all(feature = "openvr", not(feature = "openxr")))]
+    crate::backend::openvr::openvr_run(running);
+
+    #[cfg(all(feature = "openxr", not(feature = "openvr")))]
+    crate::backend::openxr::openxr_run(running);
+
+    #[cfg(not(any(feature = "openxr", feature = "openvr")))]
+    compile_error!("You must enable at least one backend feature (openxr or openvr)");
+}
+
+#[cfg(all(feature = "openxr", feature = "openvr"))]
+fn auto_run(running: Arc<AtomicBool>) {
+    use crate::backend::openvr::openvr_run;
+    use crate::backend::openxr::openxr_run;
+    use backend::common::BackendError;
+
+    let Err(BackendError::NotSupported) = openxr_run(running.clone()) else {
+        return;
+    };
+
+    let Err(BackendError::NotSupported) = openvr_run(running) else {
+        return;
+    };
+
+    log::error!("No supported backends found");
 }

src/overlays/screen.rs

@@ -1,8 +1,11 @@
+use core::slice;
 use std::{
     f32::consts::PI,
     path::Path,
+    ptr,
     sync::{mpsc::Receiver, Arc},
     time::{Duration, Instant},
+    usize,
 };
 use vulkano::{
     buffer::Subbuffer,
@@ -27,7 +30,7 @@ use crate::{
         input::{InteractionHandler, PointerHit, PointerMode},
         overlay::{OverlayData, OverlayRenderer, OverlayState, SplitOverlayBackend},
     },
-    graphics::{Vert2Uv, WlxGraphics, WlxPipeline},
+    graphics::{fourcc_to_vk, Vert2Uv, WlxGraphics, WlxPipeline},
     hid::{MOUSE_LEFT, MOUSE_MIDDLE, MOUSE_RIGHT},
     shaders::{frag_screen, vert_common},
     state::{AppSession, AppState},
@@ -116,17 +119,21 @@ impl ScreenPipeline {
         let vertex_buffer =
             graphics.upload_verts(dim[0] as _, dim[1] as _, 0.0, 0.0, dim[0] as _, dim[1] as _);
 
-        let render_texture = graphics.render_texture(dim[0], dim[1], Format::R8G8B8A8_UNORM);
+        // TODO make this a setting
+        let render_w = dim[0].min(1920);
+        let render_h = (dim[1] as f32 / dim[0] as f32 * render_w as f32) as u32;
+
+        let render_texture = graphics.render_texture(render_w, render_h, Format::R8G8B8A8_UNORM);
 
         let view = ImageView::new_default(render_texture).unwrap();
 
-        let pipeline = graphics.create_pipeline_with_layouts(
+        let pipeline = graphics.create_pipeline(
             view,
             vert_common::load(graphics.device.clone()).unwrap(),
             frag_screen::load(graphics.device.clone()).unwrap(),
             Format::R8G8B8A8_UNORM,
-            ImageLayout::ColorAttachmentOptimal,
+            //            ImageLayout::TransferSrcOptimal,
-            ImageLayout::ColorAttachmentOptimal,
+            //            ImageLayout::TransferSrcOptimal,
         );
 
         Self {
@@ -141,6 +148,15 @@ impl ScreenPipeline {
             return;
         }
 
+        self.graphics
+            .transition_layout(
+                self.pipeline.view.image().clone(),
+                ImageLayout::TransferSrcOptimal,
+                ImageLayout::General,
+            )
+            .wait(None)
+            .unwrap();
+
         let mut command_buffer = self
             .graphics
             .create_command_buffer(CommandBufferUsage::OneTimeSubmit)
@@ -168,6 +184,15 @@ impl ScreenPipeline {
             exec.flush().unwrap();
             exec.cleanup_finished();
         }
+
+        self.graphics
+            .transition_layout(
+                self.pipeline.view.image().clone(),
+                ImageLayout::General,
+                ImageLayout::TransferSrcOptimal,
+            )
+            .wait(None)
+            .unwrap();
     }
 
     pub(super) fn view(&self) -> Arc<ImageView> {
@@ -228,6 +253,10 @@ impl OverlayRenderer for ScreenRenderer {
         for frame in receiver.try_iter() {
             match frame {
                 WlxFrame::Dmabuf(frame) => {
+                    if !frame.is_valid() {
+                        log::error!("Invalid frame");
+                        continue;
+                    }
                     if let Some(new) = app.graphics.dmabuf_texture(frame) {
                         let pipeline = self
                             .pipeline
@@ -237,11 +266,55 @@ impl OverlayRenderer for ScreenRenderer {
                         self.last_image = Some(pipeline.view());
                     }
                 }
-                WlxFrame::MemFd(_frame) => {
+                WlxFrame::MemFd(frame) => {
-                    todo!()
+                    let mut upload = app
+                        .graphics
+                        .create_command_buffer(CommandBufferUsage::OneTimeSubmit);
+
+                    let Some(fd) = frame.plane.fd else {
+                        log::error!("No fd");
+                        continue;
+                    };
+                    let format = fourcc_to_vk(frame.format.fourcc);
+
+                    let len = frame.plane.stride as usize * frame.format.height as usize;
+                    let offset = frame.plane.offset as i64;
+
+                    let map = unsafe {
+                        libc::mmap(
+                            ptr::null_mut(),
+                            len,
+                            libc::PROT_READ,
+                            libc::MAP_SHARED,
+                            fd,
+                            offset,
+                        )
+                    } as *const u8;
+
+                    let data = unsafe { slice::from_raw_parts(map, len) };
+
+                    let image =
+                        upload.texture2d(frame.format.width, frame.format.height, format, &data);
+                    upload.build_and_execute_now();
+
+                    unsafe { libc::munmap(map as *mut _, len) };
+
+                    self.last_image = Some(ImageView::new_default(image).unwrap());
                 }
-                WlxFrame::MemPtr(_frame) => {
+                WlxFrame::MemPtr(frame) => {
-                    todo!()
+                    let mut upload = app
+                        .graphics
+                        .create_command_buffer(CommandBufferUsage::OneTimeSubmit);
+                    let format = fourcc_to_vk(frame.format.fourcc);
+
+                    let len = frame.format.width as usize * frame.format.height as usize;
+                    let data = unsafe { slice::from_raw_parts(frame.ptr as *const u8, len) };
+
+                    let image =
+                        upload.texture2d(frame.format.width, frame.format.height, format, &data);
+                    upload.build_and_execute_now();
+
+                    self.last_image = Some(ImageView::new_default(image).unwrap());
                 }
                 _ => {}
             };
@@ -278,7 +351,7 @@ where
 
     if session.capture_method == "auto" && wl.maybe_wlr_dmabuf_mgr.is_some() {
         log::info!("{}: Using Wlr DMA-Buf", &output.name);
-        capture = ScreenRenderer::new_wlr(output);
+        //capture = ScreenRenderer::new_wlr(output);
     }
 
     if capture.is_none() {

src/state.rs

@@ -1,11 +1,7 @@
 use std::{env::VarError, path::Path, sync::Arc};
 
 use glam::{Quat, Vec3};
-use vulkano::{
+use vulkano::format::Format;
-    device::{physical::PhysicalDevice, DeviceExtensions},
-    format::Format,
-    instance::InstanceExtensions,
-};
 
 use crate::{
     backend::{common::TaskContainer, input::InputState},
@@ -28,18 +24,12 @@ pub struct AppState {
 }
 
 impl AppState {
-    pub fn new(
+    pub fn from_graphics(graphics: Arc<WlxGraphics>) -> Self {
-        vk_instance_extensions: InstanceExtensions,
-        vk_device_extensions_fn: impl FnMut(&PhysicalDevice) -> DeviceExtensions,
-    ) -> Self {
-        let (graphics, _event_loop) =
-            WlxGraphics::new(vk_instance_extensions, vk_device_extensions_fn);
-
         AppState {
             fc: FontCache::new(),
             session: AppSession::load(),
             tasks: TaskContainer::new(),
-            graphics: graphics.clone(),
+            graphics,
             format: Format::R8G8B8A8_UNORM,
             input_state: InputState::new(),
             hid_provider: crate::hid::initialize(),