use-bigger-hammer style fix for wayvr dmabufs

src/graphics/dmabuf.rs

@@ -0,0 +1,172 @@
+use std::{mem::MaybeUninit, sync::Arc};
+
+use smallvec::SmallVec;
+use vulkano::{
+    device::Device,
+    image::{sys::RawImage, ImageCreateInfo, SubresourceLayout},
+    sync::Sharing,
+    VulkanError, VulkanObject,
+};
+
+#[allow(clippy::all, clippy::pedantic)]
+pub(super) unsafe fn create_dmabuf_image(
+    device: Arc<Device>,
+    create_info: ImageCreateInfo,
+) -> Result<RawImage, VulkanError> {
+    let &ImageCreateInfo {
+        flags,
+        image_type,
+        format,
+        ref view_formats,
+        extent,
+        array_layers,
+        mip_levels,
+        samples,
+        tiling,
+        usage,
+        stencil_usage,
+        ref sharing,
+        initial_layout,
+        ref drm_format_modifiers,
+        ref drm_format_modifier_plane_layouts,
+        external_memory_handle_types,
+        _ne: _,
+    } = &create_info;
+
+    let (sharing_mode, queue_family_index_count, p_queue_family_indices) = match sharing {
+        Sharing::Exclusive => (ash::vk::SharingMode::EXCLUSIVE, 0, &[] as _),
+        Sharing::Concurrent(queue_family_indices) => (
+            ash::vk::SharingMode::CONCURRENT,
+            queue_family_indices.len() as u32,
+            queue_family_indices.as_ptr(),
+        ),
+    };
+
+    let mut create_info_vk = ash::vk::ImageCreateInfo {
+        flags: flags.into(),
+        image_type: image_type.into(),
+        format: format.into(),
+        extent: ash::vk::Extent3D {
+            width: extent[0],
+            height: extent[1],
+            depth: extent[2],
+        },
+        mip_levels,
+        array_layers,
+        samples: samples.into(),
+        tiling: tiling.into(),
+        usage: usage.into(),
+        sharing_mode,
+        queue_family_index_count,
+        p_queue_family_indices,
+        initial_layout: initial_layout.into(),
+        ..Default::default()
+    };
+
+    let mut drm_format_modifier_explicit_info_vk = None;
+    let drm_format_modifier_plane_layouts_vk: SmallVec<[_; 4]>;
+    let mut drm_format_modifier_list_info_vk = None;
+    let mut external_memory_info_vk = None;
+    let mut format_list_info_vk = None;
+    let format_list_view_formats_vk: Vec<_>;
+    let mut stencil_usage_info_vk = None;
+
+    if drm_format_modifiers.len() == 1 {
+        drm_format_modifier_plane_layouts_vk = drm_format_modifier_plane_layouts
+            .iter()
+            .map(|subresource_layout| {
+                let &SubresourceLayout {
+                    offset,
+                    size,
+                    row_pitch,
+                    array_pitch,
+                    depth_pitch,
+                } = subresource_layout;
+
+                ash::vk::SubresourceLayout {
+                    offset,
+                    size,
+                    row_pitch,
+                    array_pitch: array_pitch.unwrap_or(0),
+                    depth_pitch: depth_pitch.unwrap_or(0),
+                }
+            })
+            .collect();
+
+        let next = drm_format_modifier_explicit_info_vk.insert(
+            ash::vk::ImageDrmFormatModifierExplicitCreateInfoEXT {
+                drm_format_modifier: drm_format_modifiers[0],
+                drm_format_modifier_plane_count: drm_format_modifier_plane_layouts_vk.len() as u32,
+                p_plane_layouts: drm_format_modifier_plane_layouts_vk.as_ptr(),
+                ..Default::default()
+            },
+        );
+
+        next.p_next = create_info_vk.p_next;
+        create_info_vk.p_next = next as *const _ as *const _;
+    } else if drm_format_modifiers.len() > 1 {
+        let next = drm_format_modifier_list_info_vk.insert(
+            ash::vk::ImageDrmFormatModifierListCreateInfoEXT {
+                drm_format_modifier_count: drm_format_modifiers.len() as u32,
+                p_drm_format_modifiers: drm_format_modifiers.as_ptr(),
+                ..Default::default()
+            },
+        );
+
+        next.p_next = create_info_vk.p_next;
+        create_info_vk.p_next = next as *const _ as *const _;
+    }
+
+    if !external_memory_handle_types.is_empty() {
+        let next = external_memory_info_vk.insert(ash::vk::ExternalMemoryImageCreateInfo {
+            handle_types: external_memory_handle_types.into(),
+            ..Default::default()
+        });
+
+        next.p_next = create_info_vk.p_next;
+        create_info_vk.p_next = next as *const _ as *const _;
+    }
+
+    if !view_formats.is_empty() {
+        format_list_view_formats_vk = view_formats
+            .iter()
+            .copied()
+            .map(ash::vk::Format::from)
+            .collect();
+
+        let next = format_list_info_vk.insert(ash::vk::ImageFormatListCreateInfo {
+            view_format_count: format_list_view_formats_vk.len() as u32,
+            p_view_formats: format_list_view_formats_vk.as_ptr(),
+            ..Default::default()
+        });
+
+        next.p_next = create_info_vk.p_next;
+        create_info_vk.p_next = next as *const _ as *const _;
+    }
+
+    if let Some(stencil_usage) = stencil_usage {
+        let next = stencil_usage_info_vk.insert(ash::vk::ImageStencilUsageCreateInfo {
+            stencil_usage: stencil_usage.into(),
+            ..Default::default()
+        });
+
+        next.p_next = create_info_vk.p_next;
+        create_info_vk.p_next = next as *const _ as *const _;
+    }
+
+    let handle = {
+        let fns = device.fns();
+        let mut output = MaybeUninit::uninit();
+        (fns.v1_0.create_image)(
+            device.handle(),
+            &create_info_vk,
+            std::ptr::null(),
+            output.as_mut_ptr(),
+        )
+        .result()
+        .map_err(VulkanError::from)?;
+        output.assume_init()
+    };
+
+    RawImage::from_handle(device, handle, create_info)
+}

src/graphics/mod.rs

@@ -1,4 +1,5 @@
 pub mod dds;
+pub mod dmabuf;
 
 use std::{
     collections::HashMap,
@@ -9,6 +10,7 @@ use std::{
 
 use anyhow::{anyhow, bail};
 use ash::vk::SubmitInfo;
+use dmabuf::create_dmabuf_image;
 use smallvec::smallvec;
 
 #[cfg(feature = "openvr")]
@@ -41,7 +43,6 @@ use vulkano::{
     format::Format,
     image::{
         sampler::{Filter, Sampler, SamplerAddressMode, SamplerCreateInfo},
-        sys::RawImage,
         view::ImageView,
         Image, ImageCreateInfo, ImageLayout, ImageTiling, ImageType, ImageUsage, SubresourceLayout,
     },
@@ -912,7 +913,7 @@ impl WlxGraphics {
         final_formats
     }
 
-    fn dmabuf_texture_ex(
+    pub fn dmabuf_texture_ex(
         &self,
         frame: DmabufFrame,
         tiling: ImageTiling,
@@ -922,7 +923,8 @@ impl WlxGraphics {
         let extent = [frame.format.width, frame.format.height, 1];
         let format = fourcc_to_vk(frame.format.fourcc)?;
 
-        let image = RawImage::new(
+        let image = unsafe {
+            create_dmabuf_image(
                 self.device.clone(),
                 ImageCreateInfo {
                     format,
@@ -934,7 +936,8 @@ impl WlxGraphics {
                     drm_format_modifier_plane_layouts: layouts,
                     ..Default::default()
                 },
-        )?;
+            )?
+        };
 
         let requirements = image.memory_requirements()[0];
         let memory_type_index = self

src/overlays/wayvr.rs

@@ -623,7 +623,20 @@ impl WayVRRenderer {
 
         drop(wayvr);
 
-        let tex = self.graphics.dmabuf_texture(frame)?;
+        let layouts: Vec<SubresourceLayout> = vec![SubresourceLayout {
+            offset: data.offset as _,
+            size: 0,
+            row_pitch: data.stride as _,
+            array_pitch: None,
+            depth_pitch: None,
+        }];
+
+        let tex = self.graphics.dmabuf_texture_ex(
+            frame,
+            vulkano::image::ImageTiling::DrmFormatModifier,
+            layouts,
+            &data.mod_info.modifiers,
+        )?;
 
         self.vk_image = Some(tex.clone());
         self.vk_image_view = Some(vulkano::image::view::ImageView::new_default(tex).unwrap());