From 6d39380ebc010dc2eabb11351564c4f00546f022 Mon Sep 17 00:00:00 2001 From: Aleksander Date: Tue, 11 Mar 2025 23:42:37 +0100 Subject: [PATCH] WayVR: Implement software texture blitting as an alternative to dmabuf (Closes #174) (#178) --- src/backend/wayvr/display.rs | 49 +++++++++-- src/backend/wayvr/egl_data.rs | 24 +++++- src/backend/wayvr/mod.rs | 35 ++++++-- src/config_wayvr.rs | 26 +++--- src/graphics/mod.rs | 7 +- src/overlays/wayvr.rs | 155 ++++++++++++++++++++++------------ src/res/wayvr.yaml | 7 ++ src/state.rs | 2 +- 8 files changed, 214 insertions(+), 91 deletions(-) diff --git a/src/backend/wayvr/display.rs b/src/backend/wayvr/display.rs index 01f1445..67804ed 100644 --- a/src/backend/wayvr/display.rs +++ b/src/backend/wayvr/display.rs @@ -1,4 +1,4 @@ -use std::{cell::RefCell, rc::Rc}; +use std::{cell::RefCell, rc::Rc, sync::Arc}; use smithay::{ backend::renderer::{ @@ -23,7 +23,7 @@ use crate::{ use super::{ client::WayVRCompositor, comp::send_frames_surface_tree, egl_data, event_queue::SyncEventQueue, - process, smithay_wrapper, time, window, WayVRSignal, + process, smithay_wrapper, time, window, BlitMethod, WayVRSignal, }; fn generate_auth_key() -> String { @@ -71,7 +71,8 @@ pub struct Display { gles_texture: GlesTexture, // TODO: drop texture egl_image: khronos_egl::Image, egl_data: Rc, - pub dmabuf_data: egl_data::DMAbufData, + + pub render_data: egl_data::RenderData, pub tasks: SyncEventQueue, } @@ -87,6 +88,7 @@ impl Drop for Display { pub struct DisplayInitParams<'a> { pub wm: Rc>, + pub config: &'a super::Config, pub renderer: &'a mut GlesRenderer, pub egl_data: Rc, pub wayland_env: super::WaylandEnv, @@ -128,7 +130,13 @@ impl Display { })?; let egl_image = params.egl_data.create_egl_image(tex_id)?; - let dmabuf_data = params.egl_data.create_dmabuf_data(&egl_image)?; + + let render_data = match params.config.blit_method { + BlitMethod::Dmabuf => { + egl_data::RenderData::Dmabuf(params.egl_data.create_dmabuf_data(&egl_image)?) + } + BlitMethod::Software => egl_data::RenderData::Software(None), + }; let opaque = false; let size = (params.width as i32, params.height as i32).into(); @@ -145,7 +153,7 @@ impl Display { wayland_env: params.wayland_env, wm: params.wm, displayed_windows: Vec::new(), - dmabuf_data, + render_data, egl_image, gles_texture, last_pressed_time_ms: 0, @@ -292,7 +300,7 @@ impl Display { } } - pub fn tick_render(&self, renderer: &mut GlesRenderer, time_ms: u64) -> anyhow::Result<()> { + pub fn tick_render(&mut self, renderer: &mut GlesRenderer, time_ms: u64) -> anyhow::Result<()> { renderer.bind(self.gles_texture.clone())?; let size = Size::from((self.width as i32, self.height as i32)); @@ -340,6 +348,35 @@ impl Display { send_frames_surface_tree(window.toplevel.wl_surface(), time_ms as u32); } + if let egl_data::RenderData::Software(_) = &self.render_data { + // Read OpenGL texture into memory. Slow! + let pixel_data = renderer.with_context(|gl| unsafe { + gl.BindTexture(ffi::TEXTURE_2D, self.gles_texture.tex_id()); + + let len = self.width as usize * self.height as usize * 4; + let mut data: Box<[u8]> = Box::new_uninit_slice(len).assume_init(); + gl.ReadPixels( + 0, + 0, + self.width as i32, + self.height as i32, + ffi::RGBA, + ffi::UNSIGNED_BYTE, + data.as_mut_ptr().cast(), + ); + + let data: Arc<[u8]> = Arc::from(data); + data + })?; + + self.render_data = + egl_data::RenderData::Software(Some(egl_data::RenderSoftwarePixelsData { + data: pixel_data, + width: self.width, + height: self.height, + })); + } + Ok(()) } diff --git a/src/backend/wayvr/egl_data.rs b/src/backend/wayvr/egl_data.rs index 8653778..0ad1046 100644 --- a/src/backend/wayvr/egl_data.rs +++ b/src/backend/wayvr/egl_data.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use super::egl_ex; use anyhow::anyhow; @@ -23,13 +25,26 @@ pub struct DMAbufModifierInfo { } #[derive(Debug, Clone)] -pub struct DMAbufData { +pub struct RenderDMAbufData { pub fd: i32, pub stride: i32, pub offset: i32, pub mod_info: DMAbufModifierInfo, } +#[derive(Debug, Clone)] +pub struct RenderSoftwarePixelsData { + pub data: Arc<[u8]>, + pub width: u16, + pub height: u16, +} + +#[derive(Debug, Clone)] +pub enum RenderData { + Dmabuf(RenderDMAbufData), + Software(Option), // will be set if the next image data is available +} + impl EGLData { pub fn load_func(&self, func_name: &str) -> anyhow::Result { let raw_fn = self.egl.get_proc_address(func_name).ok_or(anyhow::anyhow!( @@ -207,7 +222,10 @@ impl EGLData { } } - pub fn create_dmabuf_data(&self, egl_image: &khronos_egl::Image) -> anyhow::Result { + pub fn create_dmabuf_data( + &self, + egl_image: &khronos_egl::Image, + ) -> anyhow::Result { use egl_ex::PFNEGLEXPORTDMABUFIMAGEMESAPROC as FUNC; unsafe { let egl_export_dmabuf_image_mesa = @@ -235,7 +253,7 @@ impl EGLData { let mod_info = self.query_dmabuf_mod_info()?; - Ok(DMAbufData { + Ok(RenderDMAbufData { fd: fds[0], stride: strides[0], offset: offsets[0], diff --git a/src/backend/wayvr/mod.rs b/src/backend/wayvr/mod.rs index cd0ab47..c6a6ff7 100644 --- a/src/backend/wayvr/mod.rs +++ b/src/backend/wayvr/mod.rs @@ -90,11 +90,27 @@ pub enum WayVRSignal { BroadcastStateChanged(packet_server::WvrStateChanged), } +pub enum BlitMethod { + Dmabuf, + Software, +} + +impl BlitMethod { + pub fn from_string(str: &str) -> Option { + match str { + "dmabuf" => Some(BlitMethod::Dmabuf), + "software" => Some(BlitMethod::Software), + _ => None, + } + } +} + pub struct Config { pub click_freeze_time_ms: u32, pub keyboard_repeat_delay_ms: u32, pub keyboard_repeat_rate: u32, pub auto_hide_delay: Option, // if None, auto-hide is disabled + pub blit_method: BlitMethod, } pub struct WayVRState { @@ -104,7 +120,7 @@ pub struct WayVRState { wm: Rc>, egl_data: Rc, pub processes: process::ProcessVec, - config: Config, + pub config: Config, dashboard_display: Option, pub tasks: SyncEventQueue, pub signals: SyncEventQueue, @@ -256,7 +272,7 @@ impl WayVR { Ok(Self { state, ipc_server }) } - pub fn tick_display(&mut self, display: display::DisplayHandle) -> anyhow::Result<()> { + pub fn tick_display(&mut self, display: display::DisplayHandle) -> anyhow::Result { // millis since the start of wayvr let display = self .state @@ -266,12 +282,12 @@ impl WayVR { if !display.wants_redraw { // Nothing changed, do not render - return Ok(()); + return Ok(false); } if !display.visible { // Display is invisible, do not render - return Ok(()); + return Ok(false); } let time_ms = get_millis() - self.state.time_start; @@ -279,7 +295,7 @@ impl WayVR { display.tick_render(&mut self.state.manager.state.gles_renderer, time_ms)?; display.wants_redraw = false; - Ok(()) + Ok(true) } pub fn tick_events(&mut self, app: &AppState) -> anyhow::Result> { @@ -537,10 +553,13 @@ impl WayVRState { } } - pub fn get_dmabuf_data(&self, display: display::DisplayHandle) -> Option { + pub fn get_render_data( + &self, + display: display::DisplayHandle, + ) -> Option<&egl_data::RenderData> { self.displays .get(&display) - .map(|display| display.dmabuf_data.clone()) + .map(|display| &display.render_data) } pub fn create_display( @@ -555,12 +574,12 @@ impl WayVRState { egl_data: self.egl_data.clone(), renderer: &mut self.manager.state.gles_renderer, wayland_env: self.manager.wayland_env.clone(), + config: &self.config, width, height, name, primary, })?; - let handle = self.displays.add(display); self.signals.send(WayVRSignal::BroadcastStateChanged( diff --git a/src/config_wayvr.rs b/src/config_wayvr.rs index 5c62ff5..871a633 100644 --- a/src/config_wayvr.rs +++ b/src/config_wayvr.rs @@ -112,8 +112,8 @@ fn def_keyboard_repeat_rate() -> u32 { 50 } -fn def_version() -> u32 { - 1 +fn def_blit_method() -> String { + String::from("dmabuf") } #[derive(Deserialize, Serialize)] @@ -125,8 +125,6 @@ pub struct WayVRDashboard { #[derive(Deserialize, Serialize)] pub struct WayVRConfig { - #[serde(default = "def_version")] - pub version: u32, #[serde(default = "def_false")] pub run_compositor_at_start: bool, @@ -150,6 +148,9 @@ pub struct WayVRConfig { #[serde(default = "def_keyboard_repeat_rate")] pub keyboard_repeat_rate: u32, + + #[serde(default = "def_blit_method")] + pub blit_method: String, } impl WayVRConfig { @@ -173,17 +174,19 @@ impl WayVRConfig { pub fn get_wayvr_config( config_general: &crate::config::GeneralConfig, config_wayvr: &crate::config_wayvr::WayVRConfig, - ) -> wayvr::Config { - wayvr::Config { + ) -> anyhow::Result { + Ok(wayvr::Config { click_freeze_time_ms: config_general.click_freeze_time_ms, keyboard_repeat_delay_ms: config_wayvr.keyboard_repeat_delay, keyboard_repeat_rate: config_wayvr.keyboard_repeat_rate, + blit_method: wayvr::BlitMethod::from_string(&config_wayvr.blit_method) + .ok_or(anyhow::anyhow!("Unknown blit method"))?, auto_hide_delay: if config_wayvr.auto_hide { Some(config_wayvr.auto_hide_delay) } else { None }, - } + }) } pub fn post_load( @@ -221,7 +224,7 @@ impl WayVRConfig { if self.run_compositor_at_start { // Start Wayland server instantly Ok(Some(Rc::new(RefCell::new(WayVRData::new( - Self::get_wayvr_config(config, self), + Self::get_wayvr_config(config, self)?, )?)))) } else { // Lazy-init WayVR later if the user requested @@ -238,10 +241,5 @@ pub fn load_wayvr() -> WayVRConfig { config_io::ConfigRoot::WayVR.get_conf_d_path() ); - let config = load_config_with_conf_d::("wayvr.yaml", config_io::ConfigRoot::WayVR); - - if config.version != def_version() { - panic!("WayVR config version {} is not supported", config.version); - } - config + load_config_with_conf_d::("wayvr.yaml", config_io::ConfigRoot::WayVR) } diff --git a/src/graphics/mod.rs b/src/graphics/mod.rs index 49a64fb..a03b280 100644 --- a/src/graphics/mod.rs +++ b/src/graphics/mod.rs @@ -802,10 +802,9 @@ impl WlxGraphics { frame: DmabufFrame, tiling: ImageTiling, layouts: Vec, - modifiers: Vec, + modifiers: &[u64], ) -> anyhow::Result> { let extent = [frame.format.width, frame.format.height, 1]; - let format = fourcc_to_vk(frame.format.fourcc)?; let image = unsafe { @@ -817,7 +816,7 @@ impl WlxGraphics { usage: ImageUsage::SAMPLED, external_memory_handle_types: ExternalMemoryHandleTypes::DMA_BUF, tiling, - drm_format_modifiers: modifiers, + drm_format_modifiers: modifiers.to_owned(), drm_format_modifier_plane_layouts: layouts, ..Default::default() }, @@ -894,7 +893,7 @@ impl WlxGraphics { tiling = ImageTiling::DrmFormatModifier; }; - self.dmabuf_texture_ex(frame, tiling, layouts, modifiers) + self.dmabuf_texture_ex(frame, tiling, layouts, &modifiers) } pub fn render_texture( diff --git a/src/overlays/wayvr.rs b/src/overlays/wayvr.rs index 064fda5..27791cc 100644 --- a/src/overlays/wayvr.rs +++ b/src/overlays/wayvr.rs @@ -1,6 +1,9 @@ use glam::{vec3a, Affine2, Vec3, Vec3A}; use std::{cell::RefCell, collections::HashMap, rc::Rc, sync::Arc}; -use vulkano::image::SubresourceLayout; +use vulkano::{ + command_buffer::CommandBufferUsage, + image::{view::ImageView, SubresourceLayout}, +}; use wayvr_ipc::packet_server::{self, PacketServer, WvrStateChanged}; use wlx_capture::frame::{DmabufFrame, FourCC, FrameFormat, FramePlane}; @@ -170,8 +173,8 @@ impl InteractionHandler for WayVRInteractionHandler { } pub struct WayVRRenderer { - dmabuf_image: Option>, - view: Option>, + vk_image: Option>, + vk_image_view: Option>, context: Rc>, graphics: Arc, } @@ -184,8 +187,8 @@ impl WayVRRenderer { ) -> anyhow::Result { Ok(Self { context: Rc::new(RefCell::new(WayVRContext::new(wvr, display)?)), - dmabuf_image: None, - view: None, + vk_image: None, + vk_image_view: None, graphics: app.graphics.clone(), }) } @@ -551,54 +554,82 @@ where } impl WayVRRenderer { - fn ensure_dmabuf(&mut self, data: wayvr::egl_data::DMAbufData) -> anyhow::Result<()> { - if self.dmabuf_image.is_none() { - // First init - let mut planes = [FramePlane::default(); 4]; - planes[0].fd = Some(data.fd); - planes[0].offset = data.offset as u32; - planes[0].stride = data.stride; + fn ensure_software_data( + &mut self, + data: &wayvr::egl_data::RenderSoftwarePixelsData, + ) -> anyhow::Result<()> { + let mut upload = self + .graphics + .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?; - let ctx = self.context.borrow_mut(); - let wayvr = ctx.wayvr.borrow_mut(); - if let Some(disp) = wayvr.data.state.displays.get(&ctx.display) { - let frame = DmabufFrame { - format: FrameFormat { - width: disp.width as u32, - height: disp.height as u32, - fourcc: FourCC { - value: data.mod_info.fourcc, - }, - modifier: data.mod_info.modifiers[0], /* possibly not proper? */ - ..Default::default() - }, - num_planes: 1, - planes, - }; + let tex = upload.texture2d_raw( + data.width as u32, + data.height as u32, + vulkano::format::Format::R8G8B8A8_UNORM, + &data.data, + )?; - drop(wayvr); + upload.build_and_execute_now()?; - let layouts: Vec = vec![SubresourceLayout { - offset: data.offset as _, - size: 0, - row_pitch: data.stride as _, - array_pitch: None, - depth_pitch: None, - }]; + self.vk_image = Some(tex.clone()); + self.vk_image_view = Some(ImageView::new_default(tex).unwrap()); - let tex = self.graphics.dmabuf_texture_ex( - frame, - vulkano::image::ImageTiling::DrmFormatModifier, - layouts, - data.mod_info.modifiers, - )?; - self.dmabuf_image = Some(tex.clone()); - self.view = Some(vulkano::image::view::ImageView::new_default(tex).unwrap()); - } else { - anyhow::bail!("Failed to fetch WayVR display") - } + Ok(()) + } + + fn ensure_dmabuf_data( + &mut self, + data: &wayvr::egl_data::RenderDMAbufData, + ) -> anyhow::Result<()> { + if self.vk_image.is_some() { + return Ok(()); // already initialized and automatically updated due to direct zero-copy textue access } + // First init + let mut planes = [FramePlane::default(); 4]; + planes[0].fd = Some(data.fd); + planes[0].offset = data.offset as u32; + planes[0].stride = data.stride; + + let ctx = self.context.borrow_mut(); + let wayvr = ctx.wayvr.borrow_mut(); + let Some(disp) = wayvr.data.state.displays.get(&ctx.display) else { + anyhow::bail!("Failed to fetch WayVR display") + }; + + let frame = DmabufFrame { + format: FrameFormat { + width: disp.width as u32, + height: disp.height as u32, + fourcc: FourCC { + value: data.mod_info.fourcc, + }, + modifier: data.mod_info.modifiers[0], /* possibly not proper? */ + ..Default::default() + }, + num_planes: 1, + planes, + }; + + drop(wayvr); + + let layouts: Vec = vec![SubresourceLayout { + offset: data.offset as _, + size: 0, + row_pitch: data.stride as _, + array_pitch: None, + depth_pitch: None, + }]; + + let tex = self.graphics.dmabuf_texture_ex( + frame, + vulkano::image::ImageTiling::DrmFormatModifier, + layouts, + &data.mod_info.modifiers, + )?; + + self.vk_image = Some(tex.clone()); + self.vk_image_view = Some(vulkano::image::view::ImageView::new_default(tex).unwrap()); Ok(()) } } @@ -626,34 +657,48 @@ impl OverlayRenderer for WayVRRenderer { let ctx = self.context.borrow(); let mut wayvr = ctx.wayvr.borrow_mut(); - match wayvr.data.tick_display(ctx.display) { - Ok(_) => {} + let redrawn = match wayvr.data.tick_display(ctx.display) { + Ok(r) => r, Err(e) => { log::error!("tick_display failed: {}", e); return Ok(()); // do not proceed further } + }; + + if !redrawn { + return Ok(()); } - let dmabuf_data = wayvr + let data = wayvr .data .state - .get_dmabuf_data(ctx.display) - .ok_or(anyhow::anyhow!("Failed to fetch dmabuf data"))? + .get_render_data(ctx.display) + .ok_or(anyhow::anyhow!("Failed to fetch render data"))? .clone(); drop(wayvr); drop(ctx); - self.ensure_dmabuf(dmabuf_data.clone())?; + + match data { + wayvr::egl_data::RenderData::Dmabuf(data) => { + self.ensure_dmabuf_data(&data)?; + } + wayvr::egl_data::RenderData::Software(data) => { + if let Some(new_frame) = &data { + self.ensure_software_data(new_frame)?; + } + } + } Ok(()) } fn view(&mut self) -> Option> { - self.view.clone() + self.vk_image_view.clone() } fn frame_transform(&mut self) -> Option { - self.view.as_ref().map(|view| FrameTransform { + self.vk_image_view.as_ref().map(|view| FrameTransform { extent: view.image().extent(), ..Default::default() }) diff --git a/src/res/wayvr.yaml b/src/res/wayvr.yaml index a1062aa..0c0f13e 100644 --- a/src/res/wayvr.yaml +++ b/src/res/wayvr.yaml @@ -5,6 +5,13 @@ version: 1 +# If your gpu has some issues with zero-copy textures, you can set this option to "software". +# +# Possible options: +# "dmabuf": Use zero-copy texture access (from EGL to Vulkan) - no performance impact +# "software": Read pixel data to memory via glReadPixels() every time a content has been updated. Minor performance impact on large resolutions +blit_method: "dmabuf" + # Set to true if you want to make Wyland server instantly available. # By default, WayVR starts only when it's needed. # (this option is primarily used for remote starting external processes and development purposes) diff --git a/src/state.rs b/src/state.rs index ff80ca9..1cab906 100644 --- a/src/state.rs +++ b/src/state.rs @@ -139,7 +139,7 @@ impl AppState { Ok(wvr.clone()) } else { let wayvr = Rc::new(RefCell::new(WayVRData::new( - WayVRConfig::get_wayvr_config(&self.session.config, &self.session.wayvr_config), + WayVRConfig::get_wayvr_config(&self.session.config, &self.session.wayvr_config)?, )?)); self.wayvr = Some(wayvr.clone()); Ok(wayvr)