WayVR: Implement software texture blitting as an alternative to dmabuf (Closes #174) (#178)

src/backend/wayvr/display.rs

@@ -1,4 +1,4 @@
-use std::{cell::RefCell, rc::Rc};
+use std::{cell::RefCell, rc::Rc, sync::Arc};
 
 use smithay::{
     backend::renderer::{
@@ -23,7 +23,7 @@ use crate::{
 
 use super::{
     client::WayVRCompositor, comp::send_frames_surface_tree, egl_data, event_queue::SyncEventQueue,
-    process, smithay_wrapper, time, window, WayVRSignal,
+    process, smithay_wrapper, time, window, BlitMethod, WayVRSignal,
 };
 
 fn generate_auth_key() -> String {
@@ -71,7 +71,8 @@ pub struct Display {
     gles_texture: GlesTexture, // TODO: drop texture
     egl_image: khronos_egl::Image,
     egl_data: Rc<egl_data::EGLData>,
-    pub dmabuf_data: egl_data::DMAbufData,
+
+    pub render_data: egl_data::RenderData,
 
     pub tasks: SyncEventQueue<DisplayTask>,
 }
@@ -87,6 +88,7 @@ impl Drop for Display {
 
 pub struct DisplayInitParams<'a> {
     pub wm: Rc<RefCell<window::WindowManager>>,
+    pub config: &'a super::Config,
     pub renderer: &'a mut GlesRenderer,
     pub egl_data: Rc<egl_data::EGLData>,
     pub wayland_env: super::WaylandEnv,
@@ -128,7 +130,13 @@ impl Display {
         })?;
 
         let egl_image = params.egl_data.create_egl_image(tex_id)?;
-        let dmabuf_data = params.egl_data.create_dmabuf_data(&egl_image)?;
+
+        let render_data = match params.config.blit_method {
+            BlitMethod::Dmabuf => {
+                egl_data::RenderData::Dmabuf(params.egl_data.create_dmabuf_data(&egl_image)?)
+            }
+            BlitMethod::Software => egl_data::RenderData::Software(None),
+        };
 
         let opaque = false;
         let size = (params.width as i32, params.height as i32).into();
@@ -145,7 +153,7 @@ impl Display {
             wayland_env: params.wayland_env,
             wm: params.wm,
             displayed_windows: Vec::new(),
-            dmabuf_data,
+            render_data,
             egl_image,
             gles_texture,
             last_pressed_time_ms: 0,
@@ -292,7 +300,7 @@ impl Display {
         }
     }
 
-    pub fn tick_render(&self, renderer: &mut GlesRenderer, time_ms: u64) -> anyhow::Result<()> {
+    pub fn tick_render(&mut self, renderer: &mut GlesRenderer, time_ms: u64) -> anyhow::Result<()> {
         renderer.bind(self.gles_texture.clone())?;
 
         let size = Size::from((self.width as i32, self.height as i32));
@@ -340,6 +348,35 @@ impl Display {
             send_frames_surface_tree(window.toplevel.wl_surface(), time_ms as u32);
         }
 
+        if let egl_data::RenderData::Software(_) = &self.render_data {
+            // Read OpenGL texture into memory. Slow!
+            let pixel_data = renderer.with_context(|gl| unsafe {
+                gl.BindTexture(ffi::TEXTURE_2D, self.gles_texture.tex_id());
+
+                let len = self.width as usize * self.height as usize * 4;
+                let mut data: Box<[u8]> = Box::new_uninit_slice(len).assume_init();
+                gl.ReadPixels(
+                    0,
+                    0,
+                    self.width as i32,
+                    self.height as i32,
+                    ffi::RGBA,
+                    ffi::UNSIGNED_BYTE,
+                    data.as_mut_ptr().cast(),
+                );
+
+                let data: Arc<[u8]> = Arc::from(data);
+                data
+            })?;
+
+            self.render_data =
+                egl_data::RenderData::Software(Some(egl_data::RenderSoftwarePixelsData {
+                    data: pixel_data,
+                    width: self.width,
+                    height: self.height,
+                }));
+        }
+
         Ok(())
     }
 

src/backend/wayvr/egl_data.rs

@@ -1,3 +1,5 @@
+use std::sync::Arc;
+
 use super::egl_ex;
 use anyhow::anyhow;
 
@@ -23,13 +25,26 @@ pub struct DMAbufModifierInfo {
 }
 
 #[derive(Debug, Clone)]
-pub struct DMAbufData {
+pub struct RenderDMAbufData {
     pub fd: i32,
     pub stride: i32,
     pub offset: i32,
     pub mod_info: DMAbufModifierInfo,
 }
 
+#[derive(Debug, Clone)]
+pub struct RenderSoftwarePixelsData {
+    pub data: Arc<[u8]>,
+    pub width: u16,
+    pub height: u16,
+}
+
+#[derive(Debug, Clone)]
+pub enum RenderData {
+    Dmabuf(RenderDMAbufData),
+    Software(Option<RenderSoftwarePixelsData>), // will be set if the next image data is available
+}
+
 impl EGLData {
     pub fn load_func(&self, func_name: &str) -> anyhow::Result<extern "system" fn()> {
         let raw_fn = self.egl.get_proc_address(func_name).ok_or(anyhow::anyhow!(
@@ -207,7 +222,10 @@ impl EGLData {
         }
     }
 
-    pub fn create_dmabuf_data(&self, egl_image: &khronos_egl::Image) -> anyhow::Result<DMAbufData> {
+    pub fn create_dmabuf_data(
+        &self,
+        egl_image: &khronos_egl::Image,
+    ) -> anyhow::Result<RenderDMAbufData> {
         use egl_ex::PFNEGLEXPORTDMABUFIMAGEMESAPROC as FUNC;
         unsafe {
             let egl_export_dmabuf_image_mesa =
@@ -235,7 +253,7 @@ impl EGLData {
 
             let mod_info = self.query_dmabuf_mod_info()?;
 
-            Ok(DMAbufData {
+            Ok(RenderDMAbufData {
                 fd: fds[0],
                 stride: strides[0],
                 offset: offsets[0],

src/backend/wayvr/mod.rs

@@ -90,11 +90,27 @@ pub enum WayVRSignal {
     BroadcastStateChanged(packet_server::WvrStateChanged),
 }
 
+pub enum BlitMethod {
+    Dmabuf,
+    Software,
+}
+
+impl BlitMethod {
+    pub fn from_string(str: &str) -> Option<BlitMethod> {
+        match str {
+            "dmabuf" => Some(BlitMethod::Dmabuf),
+            "software" => Some(BlitMethod::Software),
+            _ => None,
+        }
+    }
+}
+
 pub struct Config {
     pub click_freeze_time_ms: u32,
     pub keyboard_repeat_delay_ms: u32,
     pub keyboard_repeat_rate: u32,
     pub auto_hide_delay: Option<u32>, // if None, auto-hide is disabled
+    pub blit_method: BlitMethod,
 }
 
 pub struct WayVRState {
@@ -104,7 +120,7 @@ pub struct WayVRState {
     wm: Rc<RefCell<window::WindowManager>>,
     egl_data: Rc<egl_data::EGLData>,
     pub processes: process::ProcessVec,
-    config: Config,
+    pub config: Config,
     dashboard_display: Option<display::DisplayHandle>,
     pub tasks: SyncEventQueue<WayVRTask>,
     pub signals: SyncEventQueue<WayVRSignal>,
@@ -256,7 +272,7 @@ impl WayVR {
         Ok(Self { state, ipc_server })
     }
 
-    pub fn tick_display(&mut self, display: display::DisplayHandle) -> anyhow::Result<()> {
+    pub fn tick_display(&mut self, display: display::DisplayHandle) -> anyhow::Result<bool> {
         // millis since the start of wayvr
         let display = self
             .state
@@ -266,12 +282,12 @@ impl WayVR {
 
         if !display.wants_redraw {
             // Nothing changed, do not render
-            return Ok(());
+            return Ok(false);
         }
 
         if !display.visible {
             // Display is invisible, do not render
-            return Ok(());
+            return Ok(false);
         }
 
         let time_ms = get_millis() - self.state.time_start;
@@ -279,7 +295,7 @@ impl WayVR {
         display.tick_render(&mut self.state.manager.state.gles_renderer, time_ms)?;
         display.wants_redraw = false;
 
-        Ok(())
+        Ok(true)
     }
 
     pub fn tick_events(&mut self, app: &AppState) -> anyhow::Result<Vec<TickTask>> {
@@ -537,10 +553,13 @@ impl WayVRState {
         }
     }
 
-    pub fn get_dmabuf_data(&self, display: display::DisplayHandle) -> Option<egl_data::DMAbufData> {
+    pub fn get_render_data(
+        &self,
+        display: display::DisplayHandle,
+    ) -> Option<&egl_data::RenderData> {
         self.displays
             .get(&display)
-            .map(|display| display.dmabuf_data.clone())
+            .map(|display| &display.render_data)
     }
 
     pub fn create_display(
@@ -555,12 +574,12 @@ impl WayVRState {
             egl_data: self.egl_data.clone(),
             renderer: &mut self.manager.state.gles_renderer,
             wayland_env: self.manager.wayland_env.clone(),
+            config: &self.config,
             width,
             height,
             name,
             primary,
         })?;
-
         let handle = self.displays.add(display);
 
         self.signals.send(WayVRSignal::BroadcastStateChanged(

src/config_wayvr.rs

@@ -112,8 +112,8 @@ fn def_keyboard_repeat_rate() -> u32 {
     50
 }
 
-fn def_version() -> u32 {
+fn def_blit_method() -> String {
-    1
+    String::from("dmabuf")
 }
 
 #[derive(Deserialize, Serialize)]
@@ -125,8 +125,6 @@ pub struct WayVRDashboard {
 
 #[derive(Deserialize, Serialize)]
 pub struct WayVRConfig {
-    #[serde(default = "def_version")]
-    pub version: u32,
     #[serde(default = "def_false")]
     pub run_compositor_at_start: bool,
 
@@ -150,6 +148,9 @@ pub struct WayVRConfig {
 
     #[serde(default = "def_keyboard_repeat_rate")]
     pub keyboard_repeat_rate: u32,
+
+    #[serde(default = "def_blit_method")]
+    pub blit_method: String,
 }
 
 impl WayVRConfig {
@@ -173,17 +174,19 @@ impl WayVRConfig {
     pub fn get_wayvr_config(
         config_general: &crate::config::GeneralConfig,
         config_wayvr: &crate::config_wayvr::WayVRConfig,
-    ) -> wayvr::Config {
+    ) -> anyhow::Result<wayvr::Config> {
-        wayvr::Config {
+        Ok(wayvr::Config {
             click_freeze_time_ms: config_general.click_freeze_time_ms,
             keyboard_repeat_delay_ms: config_wayvr.keyboard_repeat_delay,
             keyboard_repeat_rate: config_wayvr.keyboard_repeat_rate,
+            blit_method: wayvr::BlitMethod::from_string(&config_wayvr.blit_method)
+                .ok_or(anyhow::anyhow!("Unknown blit method"))?,
             auto_hide_delay: if config_wayvr.auto_hide {
                 Some(config_wayvr.auto_hide_delay)
             } else {
                 None
             },
-        }
+        })
     }
 
     pub fn post_load(
@@ -221,7 +224,7 @@ impl WayVRConfig {
         if self.run_compositor_at_start {
             // Start Wayland server instantly
             Ok(Some(Rc::new(RefCell::new(WayVRData::new(
-                Self::get_wayvr_config(config, self),
+                Self::get_wayvr_config(config, self)?,
             )?))))
         } else {
             // Lazy-init WayVR later if the user requested
@@ -238,10 +241,5 @@ pub fn load_wayvr() -> WayVRConfig {
         config_io::ConfigRoot::WayVR.get_conf_d_path()
     );
 
-    let config = load_config_with_conf_d::<WayVRConfig>("wayvr.yaml", config_io::ConfigRoot::WayVR);
+    load_config_with_conf_d::<WayVRConfig>("wayvr.yaml", config_io::ConfigRoot::WayVR)
-
-    if config.version != def_version() {
-        panic!("WayVR config version {} is not supported", config.version);
-    }
-    config
 }

src/graphics/mod.rs

@@ -802,10 +802,9 @@ impl WlxGraphics {
         frame: DmabufFrame,
         tiling: ImageTiling,
         layouts: Vec<SubresourceLayout>,
-        modifiers: Vec<u64>,
+        modifiers: &[u64],
     ) -> anyhow::Result<Arc<Image>> {
         let extent = [frame.format.width, frame.format.height, 1];
-
         let format = fourcc_to_vk(frame.format.fourcc)?;
 
         let image = unsafe {
@@ -817,7 +816,7 @@ impl WlxGraphics {
                     usage: ImageUsage::SAMPLED,
                     external_memory_handle_types: ExternalMemoryHandleTypes::DMA_BUF,
                     tiling,
-                    drm_format_modifiers: modifiers,
+                    drm_format_modifiers: modifiers.to_owned(),
                     drm_format_modifier_plane_layouts: layouts,
                     ..Default::default()
                 },
@@ -894,7 +893,7 @@ impl WlxGraphics {
             tiling = ImageTiling::DrmFormatModifier;
         };
 
-        self.dmabuf_texture_ex(frame, tiling, layouts, modifiers)
+        self.dmabuf_texture_ex(frame, tiling, layouts, &modifiers)
     }
 
     pub fn render_texture(

src/overlays/wayvr.rs

@@ -1,6 +1,9 @@
 use glam::{vec3a, Affine2, Vec3, Vec3A};
 use std::{cell::RefCell, collections::HashMap, rc::Rc, sync::Arc};
-use vulkano::image::SubresourceLayout;
+use vulkano::{
+    command_buffer::CommandBufferUsage,
+    image::{view::ImageView, SubresourceLayout},
+};
 use wayvr_ipc::packet_server::{self, PacketServer, WvrStateChanged};
 use wlx_capture::frame::{DmabufFrame, FourCC, FrameFormat, FramePlane};
 
@@ -170,8 +173,8 @@ impl InteractionHandler for WayVRInteractionHandler {
 }
 
 pub struct WayVRRenderer {
-    dmabuf_image: Option<Arc<vulkano::image::Image>>,
+    vk_image: Option<Arc<vulkano::image::Image>>,
-    view: Option<Arc<vulkano::image::view::ImageView>>,
+    vk_image_view: Option<Arc<vulkano::image::view::ImageView>>,
     context: Rc<RefCell<WayVRContext>>,
     graphics: Arc<WlxGraphics>,
 }
@@ -184,8 +187,8 @@ impl WayVRRenderer {
     ) -> anyhow::Result<Self> {
         Ok(Self {
             context: Rc::new(RefCell::new(WayVRContext::new(wvr, display)?)),
-            dmabuf_image: None,
+            vk_image: None,
-            view: None,
+            vk_image_view: None,
             graphics: app.graphics.clone(),
         })
     }
@@ -551,54 +554,82 @@ where
 }
 
 impl WayVRRenderer {
-    fn ensure_dmabuf(&mut self, data: wayvr::egl_data::DMAbufData) -> anyhow::Result<()> {
+    fn ensure_software_data(
-        if self.dmabuf_image.is_none() {
+        &mut self,
-            // First init
+        data: &wayvr::egl_data::RenderSoftwarePixelsData,
-            let mut planes = [FramePlane::default(); 4];
+    ) -> anyhow::Result<()> {
-            planes[0].fd = Some(data.fd);
+        let mut upload = self
-            planes[0].offset = data.offset as u32;
+            .graphics
-            planes[0].stride = data.stride;
+            .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
 
-            let ctx = self.context.borrow_mut();
+        let tex = upload.texture2d_raw(
-            let wayvr = ctx.wayvr.borrow_mut();
+            data.width as u32,
-            if let Some(disp) = wayvr.data.state.displays.get(&ctx.display) {
+            data.height as u32,
-                let frame = DmabufFrame {
+            vulkano::format::Format::R8G8B8A8_UNORM,
-                    format: FrameFormat {
+            &data.data,
-                        width: disp.width as u32,
+        )?;
-                        height: disp.height as u32,
-                        fourcc: FourCC {
-                            value: data.mod_info.fourcc,
-                        },
-                        modifier: data.mod_info.modifiers[0], /* possibly not proper? */
-                        ..Default::default()
-                    },
-                    num_planes: 1,
-                    planes,
-                };
 
-                drop(wayvr);
+        upload.build_and_execute_now()?;
 
-                let layouts: Vec<SubresourceLayout> = vec![SubresourceLayout {
+        self.vk_image = Some(tex.clone());
-                    offset: data.offset as _,
+        self.vk_image_view = Some(ImageView::new_default(tex).unwrap());
-                    size: 0,
-                    row_pitch: data.stride as _,
-                    array_pitch: None,
-                    depth_pitch: None,
-                }];
 
-                let tex = self.graphics.dmabuf_texture_ex(
+        Ok(())
-                    frame,
+    }
-                    vulkano::image::ImageTiling::DrmFormatModifier,
+
-                    layouts,
+    fn ensure_dmabuf_data(
-                    data.mod_info.modifiers,
+        &mut self,
-                )?;
+        data: &wayvr::egl_data::RenderDMAbufData,
-                self.dmabuf_image = Some(tex.clone());
+    ) -> anyhow::Result<()> {
-                self.view = Some(vulkano::image::view::ImageView::new_default(tex).unwrap());
+        if self.vk_image.is_some() {
-            } else {
+            return Ok(()); // already initialized and automatically updated due to direct zero-copy textue access
-                anyhow::bail!("Failed to fetch WayVR display")
-            }
         }
 
+        // First init
+        let mut planes = [FramePlane::default(); 4];
+        planes[0].fd = Some(data.fd);
+        planes[0].offset = data.offset as u32;
+        planes[0].stride = data.stride;
+
+        let ctx = self.context.borrow_mut();
+        let wayvr = ctx.wayvr.borrow_mut();
+        let Some(disp) = wayvr.data.state.displays.get(&ctx.display) else {
+            anyhow::bail!("Failed to fetch WayVR display")
+        };
+
+        let frame = DmabufFrame {
+            format: FrameFormat {
+                width: disp.width as u32,
+                height: disp.height as u32,
+                fourcc: FourCC {
+                    value: data.mod_info.fourcc,
+                },
+                modifier: data.mod_info.modifiers[0], /* possibly not proper? */
+                ..Default::default()
+            },
+            num_planes: 1,
+            planes,
+        };
+
+        drop(wayvr);
+
+        let layouts: Vec<SubresourceLayout> = vec![SubresourceLayout {
+            offset: data.offset as _,
+            size: 0,
+            row_pitch: data.stride as _,
+            array_pitch: None,
+            depth_pitch: None,
+        }];
+
+        let tex = self.graphics.dmabuf_texture_ex(
+            frame,
+            vulkano::image::ImageTiling::DrmFormatModifier,
+            layouts,
+            &data.mod_info.modifiers,
+        )?;
+
+        self.vk_image = Some(tex.clone());
+        self.vk_image_view = Some(vulkano::image::view::ImageView::new_default(tex).unwrap());
         Ok(())
     }
 }
@@ -626,34 +657,48 @@ impl OverlayRenderer for WayVRRenderer {
         let ctx = self.context.borrow();
         let mut wayvr = ctx.wayvr.borrow_mut();
 
-        match wayvr.data.tick_display(ctx.display) {
+        let redrawn = match wayvr.data.tick_display(ctx.display) {
-            Ok(_) => {}
+            Ok(r) => r,
             Err(e) => {
                 log::error!("tick_display failed: {}", e);
                 return Ok(()); // do not proceed further
             }
+        };
+
+        if !redrawn {
+            return Ok(());
         }
 
-        let dmabuf_data = wayvr
+        let data = wayvr
             .data
             .state
-            .get_dmabuf_data(ctx.display)
+            .get_render_data(ctx.display)
-            .ok_or(anyhow::anyhow!("Failed to fetch dmabuf data"))?
+            .ok_or(anyhow::anyhow!("Failed to fetch render data"))?
             .clone();
 
         drop(wayvr);
         drop(ctx);
-        self.ensure_dmabuf(dmabuf_data.clone())?;
+
+        match data {
+            wayvr::egl_data::RenderData::Dmabuf(data) => {
+                self.ensure_dmabuf_data(&data)?;
+            }
+            wayvr::egl_data::RenderData::Software(data) => {
+                if let Some(new_frame) = &data {
+                    self.ensure_software_data(new_frame)?;
+                }
+            }
+        }
 
         Ok(())
     }
 
     fn view(&mut self) -> Option<Arc<vulkano::image::view::ImageView>> {
-        self.view.clone()
+        self.vk_image_view.clone()
     }
 
     fn frame_transform(&mut self) -> Option<FrameTransform> {
-        self.view.as_ref().map(|view| FrameTransform {
+        self.vk_image_view.as_ref().map(|view| FrameTransform {
             extent: view.image().extent(),
             ..Default::default()
         })

src/res/wayvr.yaml

@@ -5,6 +5,13 @@
 
 version: 1
 
+# If your gpu has some issues with zero-copy textures, you can set this option to "software".
+#
+# Possible options:
+# "dmabuf": Use zero-copy texture access (from EGL to Vulkan) - no performance impact
+# "software": Read pixel data to memory via glReadPixels() every time a content has been updated. Minor performance impact on large resolutions
+blit_method: "dmabuf"
+
 # Set to true if you want to make Wyland server instantly available.
 # By default, WayVR starts only when it's needed.
 # (this option is primarily used for remote starting external processes and development purposes)

src/state.rs

@@ -139,7 +139,7 @@ impl AppState {
             Ok(wvr.clone())
         } else {
             let wayvr = Rc::new(RefCell::new(WayVRData::new(
-                WayVRConfig::get_wayvr_config(&self.session.config, &self.session.wayvr_config),
+                WayVRConfig::get_wayvr_config(&self.session.config, &self.session.wayvr_config)?,
             )?));
             self.wayvr = Some(wayvr.clone());
             Ok(wayvr)