WayVR: Vulkano dmabuf attempt

src/backend/wayvr/egl_data.rs

@@ -2,6 +2,7 @@ use std::sync::Arc;
 
 use super::egl_ex;
 use anyhow::anyhow;
+use smithay::backend::egl::ffi::egl::types::EGLBoolean;
 
 #[derive(Debug)]
 pub struct EGLData {
@@ -134,7 +135,7 @@ impl EGLData {
                 &mut num_formats,
             );
 
-            // Retrieve formt list
+            // Retrieve format list
             let mut formats: Vec<i32> = vec![0; num_formats as usize];
             egl_query_dmabuf_formats_ext(
                 self.display.as_ptr(),
@@ -178,23 +179,24 @@ impl EGLData {
             }
 
             let mut mods: Vec<u64> = vec![0; num_mods as usize];
+            let mut external: Vec<EGLBoolean> = vec![2; num_mods as usize];
             egl_query_dmabuf_modifiers_ext(
                 self.display.as_ptr(),
                 target_fourcc,
                 num_mods,
                 mods.as_mut_ptr(),
-                std::ptr::null_mut(),
+                external.as_mut_ptr(),
                 &mut num_mods,
             );
 
-            if mods[0] == 0xFFFF_FFFF_FFFF_FFFF {
+            /*if mods[0] == 0xFFFF_FFFF_FFFF_FFFF {
                 anyhow::bail!("modifier is -1")
             }
 
-            log::trace!("Modifier list:");
+            /*log::error!("Modifier list:");
             for modifier in &mods {
-                log::trace!("{modifier:#x}");
+                log::error!("{modifier:#x}");
-            }
+            }*/
 
             // We should not change these modifier values. Passing all of them to the Vulkan dmabuf
             // texture system causes significant graphical corruption due to invalid memory layout and
@@ -213,10 +215,19 @@ impl EGLData {
                     mods = vec![*modifier, 0x0 /* also important (???) */];
                     break;
                 }
-            }
+            }*/
+
+            let mods_filtered: Vec<u64> = mods
+                .into_iter()
+                .zip(external)
+                .map(|(modifier, external)| {
+                    log::info!("modifier {modifier}, external {external}");
+                    modifier
+                })
+                .collect();
 
             Ok(DMAbufModifierInfo {
-                modifiers: mods,
+                modifiers: mods_filtered,
                 fourcc: target_fourcc as u32,
             })
         }

src/overlays/screen.rs

@@ -537,7 +537,7 @@ impl OverlayRenderer for ScreenRenderer {
             };
 
             self.capture
-                .init(dmabuf_formats, user_data, receive_callback);
+                .init(&dmabuf_formats, user_data, receive_callback);
             self.capture.request_new_frame();
             return Ok(ShouldRender::Unable);
         }

src/overlays/wayvr.rs

@@ -1,11 +1,8 @@
 use glam::{vec3a, Affine2, Vec3, Vec3A};
 use std::{cell::RefCell, collections::HashMap, rc::Rc, sync::Arc};
-use vulkano::{
+use vulkano::{command_buffer::CommandBufferUsage, image::view::ImageView};
-    command_buffer::CommandBufferUsage,
-    image::{view::ImageView, SubresourceLayout},
-};
 use wayvr_ipc::packet_server::{self, PacketServer, WvrStateChanged};
-use wlx_capture::frame::{DmabufFrame, FourCC, FrameFormat, FramePlane};
+use wlx_capture::frame::{DmabufFrame, FrameFormat, FramePlane};
 
 use crate::{
     backend::{
@@ -607,26 +604,66 @@ impl WayVRRenderer {
             anyhow::bail!("Failed to fetch WayVR display")
         };
 
+        let drm_formats = &self.graphics.drm_formats;
+
+        let mut drm_format = None;
+        for fmt in drm_formats {
+            if fmt.fourcc.value == 0x3432_4258
+            /* XB24 */
+            {
+                drm_format = Some(fmt);
+                log::info!("format fourcc {} mod {:?}", fmt.fourcc.value, fmt.modifiers);
+                break;
+            }
+        }
+
+        let Some(drm_format) = drm_format else {
+            anyhow::bail!("Couldn't find XB24 dmabuf format");
+        };
+
+        let tex = self.graphics.dmabuf_texture(DmabufFrame {
+            format: FrameFormat {
+                width: u32::from(disp.width),
+                height: u32::from(disp.height),
+                fourcc: drm_format.fourcc,
+                modifier: drm_format.modifiers[1], /* FIXME !!!!!!!!!!!!!!!!!!!!!!!!!!! */
+                ..Default::default()
+            },
+            num_planes: 1,
+            planes,
+        })?;
+
+        /*let layouts: Vec<SubresourceLayout> = vec![SubresourceLayout {
+            offset: data.offset as _,
+            size: 0,
+            row_pitch: data.stride as _,
+            array_pitch: None,
+            depth_pitch: None,
+        }];
+
         let frame = DmabufFrame {
             format: FrameFormat {
                 width: u32::from(disp.width),
                 height: u32::from(disp.height),
-                fourcc: FourCC {
+                fourcc: drm_format.fourcc,
-                    value: data.mod_info.fourcc,
+                modifier: DRM_FORMAT_MOD_INVALID, /* possibly not proper? */
-                },
-                modifier: data.mod_info.modifiers[0], /* possibly not proper? */
                 ..Default::default()
             },
             num_planes: 1,
             planes,
         };
 
+        let tex = self.graphics.dmabuf_texture_ex(
+            frame,
+            vulkano::image::ImageTiling::DrmFormatModifier,
+            layouts,
+            &data.mod_info.modifiers,
+        )?;*/
+
         drop(wayvr);
 
-        let tex = self.graphics.dmabuf_texture(frame)?;
-
         self.vk_image = Some(tex.clone());
-        self.vk_image_view = Some(vulkano::image::view::ImageView::new_default(tex).unwrap());
+        self.vk_image_view = Some(vulkano::image::view::ImageView::new_default(tex)?);
         Ok(())
     }
 }