From 335652ecb1a6ae60174f23de8006b44f41f92b5e Mon Sep 17 00:00:00 2001 From: galister <22305755+galister@users.noreply.github.com> Date: Sun, 28 Jan 2024 13:08:57 +0100 Subject: [PATCH] fix openxr screen freeze --- src/backend/openvr/lines.rs | 3 ++ src/backend/openxr/overlay.rs | 90 ++++++++++++++++++----------------- src/backend/overlay.rs | 7 +++ src/gui/mod.rs | 3 ++ src/overlays/screen.rs | 42 +++++++++++----- src/state.rs | 5 +- 6 files changed, 91 insertions(+), 59 deletions(-) diff --git a/src/backend/openvr/lines.rs b/src/backend/openvr/lines.rs index af1f386..f7373b4 100644 --- a/src/backend/openvr/lines.rs +++ b/src/backend/openvr/lines.rs @@ -127,4 +127,7 @@ impl OverlayRenderer for StaticRenderer { fn view(&mut self) -> Option> { Some(self.view.clone()) } + fn extent(&self) -> [u32; 3] { + self.view.image().extent().clone() + } } diff --git a/src/backend/openxr/overlay.rs b/src/backend/openxr/overlay.rs index 1cf55fd..041c116 100644 --- a/src/backend/openxr/overlay.rs +++ b/src/backend/openxr/overlay.rs @@ -1,11 +1,7 @@ use std::sync::Arc; use super::XrState; -use crate::{ - backend::overlay::OverlayData, - graphics::{WlxPass, WlxPipeline}, - state::AppState, -}; +use crate::{backend::overlay::OverlayData, graphics::WlxPipeline, state::AppState}; use ash::vk::{self}; use openxr as xr; use vulkano::{ @@ -17,25 +13,30 @@ use vulkano::{ #[derive(Default)] pub struct OpenXrOverlayData { + last_view: Option>, inner: Option, pub(super) init: bool, } impl OverlayData { - pub(super) fn initialize(&mut self, xr: &XrState, state: &mut AppState) { - let Some(my_view) = self.view() else { - log::error!("Failed to get view for overlay"); - return; + pub(super) fn present_xr( + &mut self, + xr: &XrState, + state: &mut AppState, + ) -> Option<(xr::SwapchainSubImage, xr::Extent2Df)> { + if let Some(new_view) = self.view() { + self.data.last_view = Some(new_view); + } + + let my_view = if let Some(view) = self.data.last_view.as_ref() { + view.clone() + } else { + log::warn!("{}: Will not show - image not ready", self.state.name); + return None; }; - self.data.inner = { - let extent = my_view.image().extent(); - log::info!( - "{}: Create swapchain {}x{}", - self.state.name, - extent[0], - extent[1] - ); + let data = self.data.inner.get_or_insert_with(|| { + let extent = self.backend.extent(); let swapchain = xr .session @@ -53,7 +54,7 @@ impl OverlayData { }) .unwrap(); - let framebuffers = swapchain + let framebuffers: Vec = swapchain .enumerate_images() .unwrap() .into_iter() @@ -87,13 +88,6 @@ impl OverlayData { shaders.get("frag_srgb").unwrap().clone(), state.graphics.native_format, ); - let set = pipeline.uniform_sampler(0, my_view.clone(), Filter::Linear); - let pass = pipeline.create_pass( - [view.image().extent()[0] as _, view.image().extent()[1] as _], - state.graphics.quad_verts.clone(), - state.graphics.quad_indices.clone(), - vec![set], - ); let inner = Framebuffer::new( pipeline.render_pass.clone(), @@ -110,30 +104,25 @@ impl OverlayData { inner, view, pipeline, - pass, } }) .collect(); - Some(XrOverlayData { + log::info!( + "{}: Created swapchain {}x{} depth: {}, {} MB", + self.state.name, + extent[0], + extent[1], + framebuffers.len(), + extent[0] * extent[1] * 4 * framebuffers.len() as u32 / 1024 / 1024 + ); + + XrOverlayData { swapchain, framebuffers, extent, - }) - }; - } - - pub(super) fn present_xr( - &mut self, - xr: &XrState, - state: &mut AppState, - ) -> Option<(xr::SwapchainSubImage, xr::Extent2Df)> { - if self.data.inner.is_none() { - self.initialize(xr, state); - return None; - } - - let data = self.data.inner.as_mut().unwrap(); + } + }); let idx = data.swapchain.acquire_image().unwrap(); @@ -144,7 +133,21 @@ impl OverlayData { .graphics .create_command_buffer(CommandBufferUsage::OneTimeSubmit) .begin_render_pass(&frame.pipeline); - command_buffer.run_ref(&frame.pass); + + let set = frame + .pipeline + .uniform_sampler(0, my_view.clone(), Filter::Linear); + let pass = frame.pipeline.create_pass( + [ + my_view.image().extent()[0] as _, + my_view.image().extent()[1] as _, + ], + state.graphics.quad_verts.clone(), + state.graphics.quad_indices.clone(), + vec![set], + ); + + command_buffer.run_ref(&pass); command_buffer.end_render_pass().build_and_execute_now(); data.swapchain.release_image().unwrap(); @@ -180,5 +183,4 @@ struct XrFramebuffer { inner: Arc, view: Arc, pipeline: Arc, - pass: WlxPass, } diff --git a/src/backend/overlay.rs b/src/backend/overlay.rs index fc7d0e6..042475a 100644 --- a/src/backend/overlay.rs +++ b/src/backend/overlay.rs @@ -150,6 +150,7 @@ pub trait OverlayRenderer { fn resume(&mut self, app: &mut AppState); fn render(&mut self, app: &mut AppState); fn view(&mut self) -> Option>; + fn extent(&self) -> [u32; 3]; } pub struct FallbackRenderer; @@ -162,6 +163,9 @@ impl OverlayRenderer for FallbackRenderer { fn view(&mut self) -> Option> { None } + fn extent(&self) -> [u32; 3] { + [0, 0, 0] + } } // Boilerplate and dummies @@ -204,6 +208,9 @@ impl OverlayRenderer for SplitOverlayBackend { fn view(&mut self) -> Option> { self.renderer.view() } + fn extent(&self) -> [u32; 3] { + self.renderer.extent() + } } impl InteractionHandler for SplitOverlayBackend { fn on_left(&mut self, app: &mut AppState, pointer: usize) { diff --git a/src/gui/mod.rs b/src/gui/mod.rs index fb3fbc3..7cf1f67 100644 --- a/src/gui/mod.rs +++ b/src/gui/mod.rs @@ -462,6 +462,9 @@ impl OverlayRenderer for Canvas { fn view(&mut self) -> Option> { Some(self.view_final.clone()) } + fn extent(&self) -> [u32; 3] { + self.view_final.image().extent().clone() + } } impl OverlayBackend for Canvas {} diff --git a/src/overlays/screen.rs b/src/overlays/screen.rs index 42cb008..ccfcd37 100644 --- a/src/overlays/screen.rs +++ b/src/overlays/screen.rs @@ -112,17 +112,13 @@ struct ScreenPipeline { } impl ScreenPipeline { - fn new(graphics: Arc, image: &Image) -> Self { + fn new(graphics: Arc, image: &Image, extent: &[u32; 3]) -> Self { let dim = image.extent(); let vertex_buffer = graphics.upload_verts(dim[0] as _, dim[1] as _, 0.0, 0.0, dim[0] as _, dim[1] as _); - // TODO make this a setting - let render_w = dim[0].min(1920); - let render_h = (dim[1] as f32 / dim[0] as f32 * render_w as f32) as u32; - - let render_texture = graphics.render_texture(render_w, render_h, Format::R8G8B8A8_UNORM); + let render_texture = graphics.render_texture(extent[0], extent[1], Format::R8G8B8A8_UNORM); let view = ImageView::new_default(render_texture).unwrap(); @@ -204,10 +200,12 @@ impl ScreenPipeline { } pub struct ScreenRenderer { + name: Arc, capture: Box, receiver: Option>, pipeline: Option, last_image: Option>, + extent: [u32; 3], } impl ScreenRenderer { @@ -216,11 +214,14 @@ impl ScreenRenderer { return None; }; let capture = WlrDmabufCapture::new(client, output.id); + Some(ScreenRenderer { + name: output.name.clone(), capture: Box::new(capture), receiver: None, pipeline: None, last_image: None, + extent: extent_from_res(output.size), }) } @@ -235,16 +236,21 @@ impl ScreenRenderer { let capture = PipewireCapture::new(name, node_id, 60); Some(ScreenRenderer { + name: output.name.clone(), capture: Box::new(capture), receiver: None, pipeline: None, last_image: None, + extent: extent_from_res(output.size), }) } } impl OverlayRenderer for ScreenRenderer { - fn init(&mut self, _app: &mut AppState) {} + fn init(&mut self, app: &mut AppState) { + let images = app.graphics.shared_images.read().unwrap(); + self.last_image = Some(images.get("fallback").unwrap().clone()); + } fn render(&mut self, app: &mut AppState) { let receiver = self.receiver.get_or_insert_with(|| self.capture.init()); @@ -256,10 +262,10 @@ impl OverlayRenderer for ScreenRenderer { continue; } if let Some(new) = app.graphics.dmabuf_texture(frame) { - let pipeline = self - .pipeline - .get_or_insert_with(|| ScreenPipeline::new(app.graphics.clone(), &new)); - log::info!("New frame"); + let pipeline = self.pipeline.get_or_insert_with(|| { + ScreenPipeline::new(app.graphics.clone(), &new, &self.extent) + }); + log::debug!("{}: New DMA-buf frame", self.name); pipeline.render(new); self.last_image = Some(pipeline.view()); } @@ -273,6 +279,7 @@ impl OverlayRenderer for ScreenRenderer { log::error!("No fd"); continue; }; + log::debug!("{}: New MemFd frame", self.name); let format = fourcc_to_vk(frame.format.fourcc); let len = frame.plane.stride as usize * frame.format.height as usize; @@ -300,6 +307,7 @@ impl OverlayRenderer for ScreenRenderer { self.last_image = Some(ImageView::new_default(image).unwrap()); } WlxFrame::MemPtr(frame) => { + log::debug!("{}: New MemPtr frame", self.name); let mut upload = app .graphics .create_command_buffer(CommandBufferUsage::OneTimeSubmit); @@ -328,6 +336,9 @@ impl OverlayRenderer for ScreenRenderer { fn view(&mut self) -> Option> { self.last_image.take() } + fn extent(&self) -> [u32; 3] { + self.extent.clone() + } } fn try_create_screen(wl: &WlxClient, id: u32, session: &AppSession) -> Option> @@ -441,3 +452,12 @@ where { todo!() } + +fn extent_from_res(res: (i32, i32)) -> [u32; 3] { + // screens above a certain resolution will have severe aliasing + + // TODO make dynamic. maybe don't go above HMD resolution? + let w = res.0.min(1920) as u32; + let h = (res.1 as f32 / res.0 as f32 * w as f32) as u32; + [w, h, 1] +} diff --git a/src/state.rs b/src/state.rs index 824b712..822b038 100644 --- a/src/state.rs +++ b/src/state.rs @@ -1,10 +1,7 @@ use std::{env::VarError, path::Path, sync::Arc}; use glam::{Quat, Vec3}; -use vulkano::{ - command_buffer::CommandBufferUsage, format::Format, image::view::ImageView, - shader::ShaderModule, -}; +use vulkano::{command_buffer::CommandBufferUsage, format::Format, image::view::ImageView}; use crate::{ backend::{common::TaskContainer, input::InputState},