apply pipewire transforms to screens

Cargo.toml

@@ -55,7 +55,7 @@ sysinfo = { version = "0.31.2" }
 thiserror = "1.0.61"
 vulkano = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
 vulkano-shaders = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
-wlx-capture = { git = "https://github.com/galister/wlx-capture", tag = "v0.3.13", default-features = false }
+wlx-capture = { git = "https://github.com/galister/wlx-capture", tag = "v0.3.15", default-features = false }
 libmonado-rs = { git = "https://github.com/technobaboo/libmonado-rs", rev = "a495f6d", optional = true }
 winit = { version = "0.30.0", optional = true }
 xdg = "2.5.2"

src/backend/input.rs

@@ -417,7 +417,9 @@ where
             .grabbed
             .is_some_and(|x| x.grabbed_id == hit.overlay)
         {
-            let can_curve = hovered.extent().is_some_and(|e| e[0] >= e[1]);
+            let can_curve = hovered
+                .frame_transform()
+                .is_some_and(|e| e.extent[0] >= e.extent[1]);
 
             if can_curve {
                 let cur = hovered.state.curvature.unwrap_or(0.0);

src/backend/openvr/lines.rs

@@ -11,7 +11,9 @@ use vulkano::format::Format;
 use vulkano::image::view::ImageView;
 use vulkano::image::ImageLayout;
 
-use crate::backend::overlay::{OverlayData, OverlayRenderer, OverlayState, SplitOverlayBackend};
+use crate::backend::overlay::{
+    FrameTransform, OverlayData, OverlayRenderer, OverlayState, SplitOverlayBackend,
+};
 use crate::graphics::WlxGraphics;
 use crate::state::AppState;
 
@@ -189,7 +191,10 @@ impl OverlayRenderer for StaticRenderer {
     fn view(&mut self) -> Option<Arc<ImageView>> {
         Some(self.view.clone())
     }
-    fn extent(&mut self) -> Option<[u32; 3]> {
+    fn frame_transform(&mut self) -> Option<FrameTransform> {
-        Some(self.view.image().extent())
+        Some(FrameTransform {
+            extent: self.view.image().extent(),
+            ..Default::default()
+        })
     }
 }

src/backend/openvr/overlay.rs

@@ -188,7 +188,7 @@ impl OverlayData<OpenVrOverlayData> {
     }
 
     pub(super) fn upload_transform(
-        &self,
+        &mut self,
         universe: ETrackingUniverseOrigin,
         overlay: &mut OverlayManager,
     ) {
@@ -197,7 +197,14 @@ impl OverlayData<OpenVrOverlayData> {
             return;
         };
 
-        let transform = Matrix3x4::from_affine(&self.state.transform);
+        let effective = self.state.transform
+            * self
+                .backend
+                .frame_transform()
+                .map(|f| f.transform)
+                .unwrap_or_default();
+
+        let transform = Matrix3x4::from_affine(&effective);
 
         if let Err(e) = overlay.set_transform_absolute(handle, universe, &transform) {
             log::error!(

src/backend/openxr/overlay.rs

@@ -39,7 +39,8 @@ impl OverlayData<OpenXrOverlayData> {
             return Ok(CompositionLayer::None);
         };
 
-        let extent = self.extent().unwrap(); // want panic
+        let frame_transform = self.frame_transform().unwrap(); // want panic
+        let extent = frame_transform.extent;
 
         let data = match self.data.swapchain {
             Some(ref mut data) => data,
@@ -74,10 +75,12 @@ impl OverlayData<OpenXrOverlayData> {
             (major / aspect_ratio, major)
         };
 
+        let transform = self.state.transform * frame_transform.transform;
+
         if let Some(curvature) = self.state.curvature {
             let radius = scale_x / (2.0 * PI * curvature);
-            let quat = helpers::transform_to_norm_quat(&self.state.transform);
+            let quat = helpers::transform_to_norm_quat(&transform);
-            let center_point = self.state.transform.translation + quat.mul_vec3a(Vec3A::Z * radius);
+            let center_point = transform.translation + quat.mul_vec3a(Vec3A::Z * radius);
 
             let posef = helpers::translation_rotation_to_posef(center_point, quat);
             let angle = 2.0 * (scale_x / (2.0 * radius));
@@ -93,7 +96,7 @@ impl OverlayData<OpenXrOverlayData> {
                 .aspect_ratio(aspect_ratio);
             Ok(CompositionLayer::Cylinder(cylinder))
         } else {
-            let posef = helpers::transform_to_posef(&self.state.transform);
+            let posef = helpers::transform_to_posef(&transform);
             let quad = xr::CompositionLayerQuad::new()
                 .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA)
                 .pose(posef)

src/backend/overlay.rs

@@ -233,8 +233,8 @@ where
     pub fn view(&mut self) -> Option<Arc<ImageView>> {
         self.backend.view()
     }
-    pub fn extent(&mut self) -> Option<[u32; 3]> {
+    pub fn frame_transform(&mut self) -> Option<FrameTransform> {
-        self.backend.extent()
+        self.backend.frame_transform()
     }
     pub fn set_visible(&mut self, app: &mut AppState, visible: bool) -> anyhow::Result<()> {
         let old_visible = self.state.want_visible;
@@ -250,6 +250,12 @@ where
     }
 }
 
+#[derive(Default)]
+pub struct FrameTransform {
+    pub extent: [u32; 3],
+    pub transform: Affine3A,
+}
+
 pub trait OverlayRenderer {
     /// Called once, before the first frame is rendered
     fn init(&mut self, app: &mut AppState) -> anyhow::Result<()>;
@@ -263,7 +269,7 @@ pub trait OverlayRenderer {
     /// Used for creating swapchains.
     ///
     /// Muse not be None if view() is also not None
-    fn extent(&mut self) -> Option<[u32; 3]>;
+    fn frame_transform(&mut self) -> Option<FrameTransform>;
 }
 
 pub struct FallbackRenderer;
@@ -284,7 +290,7 @@ impl OverlayRenderer for FallbackRenderer {
     fn view(&mut self) -> Option<Arc<ImageView>> {
         None
     }
-    fn extent(&mut self) -> Option<[u32; 3]> {
+    fn frame_transform(&mut self) -> Option<FrameTransform> {
         None
     }
 }
@@ -341,8 +347,8 @@ impl OverlayRenderer for SplitOverlayBackend {
     fn view(&mut self) -> Option<Arc<ImageView>> {
         self.renderer.view()
     }
-    fn extent(&mut self) -> Option<[u32; 3]> {
+    fn frame_transform(&mut self) -> Option<FrameTransform> {
-        self.renderer.extent()
+        self.renderer.frame_transform()
     }
 }
 impl InteractionHandler for SplitOverlayBackend {

src/gui/canvas/mod.rs

@@ -13,7 +13,7 @@ use vulkano::{
 use crate::{
     backend::{
         input::{Haptics, InteractionHandler, PointerHit},
-        overlay::{OverlayBackend, OverlayRenderer},
+        overlay::{FrameTransform, OverlayBackend, OverlayRenderer},
     },
     graphics::{WlxGraphics, WlxPass, WlxPipeline, WlxPipelineLegacy, BLEND_ALPHA},
     state::AppState,
@@ -375,8 +375,11 @@ impl<D, S> OverlayRenderer for Canvas<D, S> {
         Some(self.view_final.clone())
     }
 
-    fn extent(&mut self) -> Option<[u32; 3]> {
+    fn frame_transform(&mut self) -> Option<FrameTransform> {
-        Some(self.view_final.image().extent())
+        Some(FrameTransform {
+            extent: self.view_final.image().extent(),
+            ..Default::default()
+        })
     }
 }
 

src/overlays/keyboard.rs

@@ -7,7 +7,7 @@ use std::{
 use crate::{
     backend::{
         input::{InteractionHandler, PointerMode},
-        overlay::{OverlayBackend, OverlayData, OverlayRenderer, OverlayState},
+        overlay::{FrameTransform, OverlayBackend, OverlayData, OverlayRenderer, OverlayState},
     },
     config::{self, ConfigType},
     gui::{
@@ -533,8 +533,8 @@ impl OverlayRenderer for KeyboardBackend {
     fn render(&mut self, app: &mut AppState) -> anyhow::Result<()> {
         self.canvas.render(app)
     }
-    fn extent(&mut self) -> Option<[u32; 3]> {
+    fn frame_transform(&mut self) -> Option<FrameTransform> {
-        self.canvas.extent()
+        self.canvas.frame_transform()
     }
     fn view(&mut self) -> Option<std::sync::Arc<vulkano::image::view::ImageView>> {
         self.canvas.view()

src/overlays/mirror.rs

@@ -10,7 +10,8 @@ use crate::{
     backend::{
         common::OverlaySelector,
         overlay::{
-            ui_transform, OverlayBackend, OverlayRenderer, OverlayState, SplitOverlayBackend,
+            ui_transform, FrameTransform, OverlayBackend, OverlayRenderer, OverlayState,
+            SplitOverlayBackend,
         },
         task::TaskType,
     },
@@ -121,8 +122,11 @@ impl OverlayRenderer for MirrorRenderer {
         self.renderer.as_mut().and_then(|r| r.view())
     }
 
-    fn extent(&mut self) -> Option<[u32; 3]> {
+    fn frame_transform(&mut self) -> Option<FrameTransform> {
-        Some(self.last_extent)
+        Some(FrameTransform {
+            extent: self.last_extent,
+            ..Default::default()
+        })
     }
 }
 

src/overlays/screen.rs

@@ -13,6 +13,8 @@ use vulkano::{
     image::{sampler::Filter, view::ImageView, Image},
     pipeline::graphics::color_blend::AttachmentBlend,
 };
+use wlx_capture::frame as wlx_frame;
+
 use wlx_capture::{
     frame::{
         DrmFormat, FrameFormat, MouseMeta, WlxFrame, DRM_FORMAT_ABGR2101010, DRM_FORMAT_ABGR8888,
@@ -46,12 +48,12 @@ use {
 #[cfg(feature = "x11")]
 use wlx_capture::xshm::{XshmCapture, XshmScreen};
 
-use glam::{vec2, vec3a, Affine2, Quat, Vec2, Vec3};
+use glam::{vec2, vec3a, Affine2, Affine3A, Quat, Vec2, Vec3};
 
 use crate::{
     backend::{
         input::{Haptics, InteractionHandler, PointerHit, PointerMode},
-        overlay::{OverlayRenderer, OverlayState, SplitOverlayBackend},
+        overlay::{FrameTransform, OverlayRenderer, OverlayState, SplitOverlayBackend},
     },
     config::{def_pw_tokens, GeneralConfig, PwTokenMap},
     graphics::{
@@ -284,6 +286,7 @@ pub struct ScreenRenderer {
     capture: Box<dyn WlxCapture>,
     pipeline: Option<ScreenPipeline>,
     last_view: Option<Arc<ImageView>>,
+    transform: Affine3A,
     extent: Option<[u32; 3]>,
 }
 
@@ -295,6 +298,7 @@ impl ScreenRenderer {
             capture,
             pipeline: None,
             last_view: None,
+            transform: Affine3A::IDENTITY,
             extent: None,
         }
     }
@@ -309,6 +313,7 @@ impl ScreenRenderer {
             capture: Box::new(capture),
             pipeline: None,
             last_view: None,
+            transform: Affine3A::IDENTITY,
             extent: None,
         })
     }
@@ -323,6 +328,7 @@ impl ScreenRenderer {
             capture: Box::new(capture),
             pipeline: None,
             last_view: None,
+            transform: Affine3A::IDENTITY,
             extent: None,
         })
     }
@@ -365,6 +371,7 @@ impl ScreenRenderer {
                 capture: Box::new(capture),
                 pipeline: None,
                 last_view: None,
+                transform: Affine3A::IDENTITY,
                 extent: None,
             },
             select_screen_result.restore_token,
@@ -380,6 +387,7 @@ impl ScreenRenderer {
             capture: Box::new(capture),
             pipeline: None,
             last_view: None,
+            transform: Affine3A::IDENTITY,
             extent: None,
         }
     }
@@ -469,6 +477,7 @@ impl OverlayRenderer for ScreenRenderer {
                         continue;
                     }
                     self.extent.get_or_insert_with(|| {
+                        self.transform = affine_from_format(&frame.format);
                         extent_from_format(frame.format, &app.session.config)
                     });
                     match app.graphics.dmabuf_texture(frame) {
@@ -511,6 +520,7 @@ impl OverlayRenderer for ScreenRenderer {
                         continue;
                     };
                     self.extent.get_or_insert_with(|| {
+                        self.transform = affine_from_format(&frame.format);
                         extent_from_format(frame.format, &app.session.config)
                     });
                     log::debug!("{}: New MemFd frame", self.name);
@@ -601,8 +611,11 @@ impl OverlayRenderer for ScreenRenderer {
     fn view(&mut self) -> Option<Arc<ImageView>> {
         self.last_view.clone()
     }
-    fn extent(&mut self) -> Option<[u32; 3]> {
+    fn frame_transform(&mut self) -> Option<FrameTransform> {
-        self.extent
+        self.extent.map(|extent| FrameTransform {
+            extent,
+            transform: self.transform,
+        })
     }
 }
 
@@ -771,16 +784,10 @@ pub fn create_screens_wayland(
     wl: &mut WlxClientAlias,
     app: &mut AppState,
 ) -> anyhow::Result<ScreenCreateData> {
-    use crate::config::AStrMap;
-
     let mut screens = vec![];
 
     // Load existing Pipewire tokens from file
-    let mut pw_tokens: PwTokenMap = if let Ok(conf) = load_pw_token_config() {
+    let mut pw_tokens: PwTokenMap = load_pw_token_config().unwrap_or_default();
-        conf
-    } else {
-        AStrMap::new()
-    };
 
     let pw_tokens_copy = pw_tokens.clone();
     let has_wlr_dmabuf = wl.maybe_wlr_dmabuf_mgr.is_some();
@@ -857,15 +864,10 @@ pub fn create_screens_x11pw(_app: &mut AppState) -> anyhow::Result<ScreenCreateD
 
 #[cfg(all(feature = "x11", feature = "pipewire"))]
 pub fn create_screens_x11pw(app: &mut AppState) -> anyhow::Result<ScreenCreateData> {
-    use crate::config::{AStrMap, AStrMapExt};
     use anyhow::bail;
 
     // Load existing Pipewire tokens from file
-    let mut pw_tokens: PwTokenMap = if let Ok(conf) = load_pw_token_config() {
+    let mut pw_tokens: PwTokenMap = load_pw_token_config().unwrap_or_default();
-        conf
-    } else {
-        AStrMap::new()
-    };
     let pw_tokens_copy = pw_tokens.clone();
     let token = pw_tokens.arc_get("x11").map(|s| s.as_str());
     let embed_mouse = !app.session.config.double_cursor_fix;
@@ -931,6 +933,7 @@ pub fn create_screens_x11pw(app: &mut AppState) -> anyhow::Result<ScreenCreateDa
                 capture: Box::new(PipewireCapture::new(m.name.clone(), s.node_id)),
                 pipeline: None,
                 last_view: None,
+                transform: Affine3A::IDENTITY,
                 extent: Some(extent_from_res(
                     size.0 as _,
                     size.1 as _,
@@ -1050,6 +1053,31 @@ fn extent_from_res(width: u32, height: u32, config: &GeneralConfig) -> [u32; 3]
     [w, h, 1]
 }
 
+fn affine_from_format(format: &FrameFormat) -> Affine3A {
+    const FLIP_X: Vec3 = Vec3 {
+        x: -1.0,
+        y: 1.0,
+        z: 1.0,
+    };
+
+    match format.transform {
+        wlx_frame::Transform::None => Affine3A::IDENTITY,
+        wlx_frame::Transform::Rotated90 => Affine3A::from_rotation_z(PI / 2.0),
+        wlx_frame::Transform::Rotated180 => Affine3A::from_rotation_z(PI),
+        wlx_frame::Transform::Rotated270 => Affine3A::from_rotation_z(-PI / 2.0),
+        wlx_frame::Transform::Flipped => Affine3A::from_scale(FLIP_X),
+        wlx_frame::Transform::Flipped90 => {
+            Affine3A::from_scale(FLIP_X) * Affine3A::from_rotation_z(PI / 2.0)
+        }
+        wlx_frame::Transform::Flipped180 => {
+            Affine3A::from_scale(FLIP_X) * Affine3A::from_rotation_z(PI)
+        }
+        wlx_frame::Transform::Flipped270 => {
+            Affine3A::from_scale(FLIP_X) * Affine3A::from_rotation_z(-PI / 2.0)
+        }
+    }
+}
+
 #[cfg(all(feature = "pipewire", feature = "x11"))]
 fn best_match<'a>(
     stream: &PipewireStream,

src/overlays/wayvr.rs

@@ -9,7 +9,7 @@ use crate::{
         common::{OverlayContainer, OverlaySelector},
         input::{self, InteractionHandler},
         overlay::{
-            ui_transform, OverlayData, OverlayID, OverlayRenderer, OverlayState,
+            ui_transform, FrameTransform, OverlayData, OverlayID, OverlayRenderer, OverlayState,
             SplitOverlayBackend,
         },
         task::TaskType,
@@ -299,6 +299,7 @@ impl WayVRRenderer {
                             value: data.mod_info.fourcc,
                         },
                         modifier: data.mod_info.modifiers[0], /* possibly not proper? */
+                        ..Default::default()
                     },
                     num_planes: 1,
                     planes,
@@ -373,8 +374,11 @@ impl OverlayRenderer for WayVRRenderer {
         self.view.clone()
     }
 
-    fn extent(&mut self) -> Option<[u32; 3]> {
+    fn frame_transform(&mut self) -> Option<FrameTransform> {
-        self.view.as_ref().map(|view| view.image().extent())
+        self.view.as_ref().map(|view| FrameTransform {
+            extent: view.image().extent(),
+            ..Default::default()
+        })
     }
 }