feat: handle screen changes at runtime

Cargo.lock

@@ -3603,8 +3603,8 @@ dependencies = [
 
 [[package]]
 name = "wlx-capture"
-version = "0.3.1"
+version = "0.3.8"
-source = "git+https://github.com/galister/wlx-capture?tag=v0.3.1#f9c9c98e0752bfbdaa2af69b59376c697c223915"
+source = "git+https://github.com/galister/wlx-capture?tag=v0.3.8#e88ff6c0ceb360c61c5f26ede6a339a2daa09d8b"
 dependencies = [
  "ashpd",
  "drm-fourcc",

Cargo.toml

@@ -43,7 +43,7 @@ strum = { version = "0.25.0", features = ["derive"] }
 thiserror = "1.0.56"
 vulkano = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
 vulkano-shaders = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
-wlx-capture = { git = "https://github.com/galister/wlx-capture", tag = "v0.3.1", default-features = false }
+wlx-capture = { git = "https://github.com/galister/wlx-capture", tag = "v0.3.8", default-features = false }
 xdg = "2.5.2"
 
 [features]

src/backend/common.rs

@@ -8,17 +8,19 @@ use std::{
 #[cfg(feature = "openxr")]
 use openxr as xr;
 
-use glam::{Affine3A, Vec2, Vec3A, Vec3Swizzles};
+use glam::{vec2, Affine3A, Vec2, Vec3A, Vec3Swizzles};
 use idmap::IdMap;
 use serde::Deserialize;
 use thiserror::Error;
+use wlx_capture::wayland::{OutputChangeEvent, WlxClient};
 
 use crate::{
     overlays::{
         keyboard::create_keyboard,
-        watch::{create_watch, WATCH_NAME},
+        screen::{create_screen_interaction, create_screen_renderer_wl, load_pw_token_config},
+        watch::{create_watch, create_watch_canvas, WATCH_NAME},
     },
-    state::{AppState, ScreenMeta},
+    state::AppState,
 };
 
 use super::overlay::{OverlayBackend, OverlayData, OverlayState};
@@ -43,7 +45,7 @@ where
     T: Default,
 {
     overlays: IdMap<usize, OverlayData<T>>,
-    pub extent: Vec2,
+    wl: Option<WlxClient>,
 }
 
 impl<T> OverlayContainer<T>
@@ -52,18 +54,36 @@ where
 {
     pub fn new(app: &mut AppState) -> anyhow::Result<Self> {
         let mut overlays = IdMap::new();
-        let (screens, extent) = if std::env::var("WAYLAND_DISPLAY").is_ok() {
+        let mut wl = WlxClient::new();
-            crate::overlays::screen::get_screens_wayland(&app.session)?
-        } else {
-            crate::overlays::screen::get_screens_x11(&app.session)?
-        };
 
         app.screens.clear();
-        for screen in screens.iter() {
+        let data = if let Some(wl) = wl.as_mut() {
-            app.screens.push(ScreenMeta {
+            crate::overlays::screen::create_screens_wayland(wl, app)?
-                name: screen.state.name.clone(),
+        } else {
-                id: screen.state.id,
+            crate::overlays::screen::create_screens_x11(app)?
-            });
+        };
+
+        let mut show_screens = app.session.config.show_screens.clone();
+        if show_screens.is_empty() {
+            if let Some((_, s, _)) = data.screens.first() {
+                show_screens.push(s.name.clone());
+            }
+        }
+
+        for (meta, mut state, backend) in data.screens {
+            if show_screens.contains(&state.name) {
+                state.show_hide = true;
+                state.want_visible = false;
+            }
+            overlays.insert(
+                state.id,
+                OverlayData::<T> {
+                    state,
+                    backend,
+                    ..Default::default()
+                },
+            );
+            app.screens.push(meta);
         }
 
         let mut watch = create_watch::<T>(app)?;
@@ -75,21 +95,134 @@ where
         keyboard.state.want_visible = false;
         overlays.insert(keyboard.state.id, keyboard);
 
-        let mut show_screens = app.session.config.show_screens.clone();
+        Ok(Self { overlays, wl })
-        if show_screens.is_empty() {
+    }
-            if let Some(s) = screens.first() {
+
-                show_screens.push(s.state.name.clone());
+    pub fn update(&mut self, app: &mut AppState) -> anyhow::Result<Vec<OverlayData<T>>> {
+        let mut removed_overlays = vec![];
+        let Some(wl) = self.wl.as_mut() else {
+            return Ok(removed_overlays);
+        };
+
+        wl.dispatch_pending();
+
+        let mut create_ran = false;
+        let mut extent_dirty = false;
+        let mut watch_dirty = false;
+
+        let mut maybe_token_store = None;
+
+        for ev in wl.iter_events().collect::<Vec<_>>() {
+            match ev {
+                OutputChangeEvent::Create(_) => {
+                    if create_ran {
+                        continue;
+                    }
+                    let data = crate::overlays::screen::create_screens_wayland(wl, app)?;
+                    create_ran = true;
+                    for (meta, state, backend) in data.screens {
+                        self.overlays.insert(
+                            state.id,
+                            OverlayData::<T> {
+                                state,
+                                backend,
+                                ..Default::default()
+                            },
+                        );
+                        app.screens.push(meta);
+                        watch_dirty = true;
+                    }
+                }
+                OutputChangeEvent::Destroy(id) => {
+                    let Some(idx) = app.screens.iter().position(|s| s.native_handle == id) else {
+                        continue;
+                    };
+
+                    let meta = &app.screens[idx];
+                    let removed = self.overlays.remove(meta.id).unwrap();
+                    removed_overlays.push(removed);
+                    log::info!("{}: Destroyed", meta.name);
+                    app.screens.remove(idx);
+                    watch_dirty = true;
+                    extent_dirty = true;
+                }
+                OutputChangeEvent::Logical(id) => {
+                    let Some(meta) = app.screens.iter().find(|s| s.native_handle == id) else {
+                        continue;
+                    };
+                    let output = wl.outputs.get(id).unwrap();
+                    let Some(overlay) = self.overlays.get_mut(meta.id) else {
+                        continue;
+                    };
+                    let logical_pos =
+                        vec2(output.logical_pos.0 as f32, output.logical_pos.1 as f32);
+                    let logical_size =
+                        vec2(output.logical_size.0 as f32, output.logical_size.1 as f32);
+                    let transform = output.transform.into();
+                    overlay
+                        .backend
+                        .set_interaction(Box::new(create_screen_interaction(
+                            logical_pos,
+                            logical_size,
+                            transform,
+                        )));
+                    extent_dirty = true;
+                }
+                OutputChangeEvent::Physical(id) => {
+                    let Some(meta) = app.screens.iter().find(|s| s.native_handle == id) else {
+                        continue;
+                    };
+                    let output = wl.outputs.get(id).unwrap();
+                    let Some(overlay) = self.overlays.get_mut(meta.id) else {
+                        continue;
+                    };
+
+                    let has_wlr_dmabuf = wl.maybe_wlr_dmabuf_mgr.is_some();
+                    let has_wlr_screencopy = wl.maybe_wlr_screencopy_mgr.is_some();
+
+                    let pw_token_store = maybe_token_store.get_or_insert_with(|| {
+                        load_pw_token_config().unwrap_or_else(|e| {
+                            log::warn!("Failed to load PipeWire token config: {:?}", e);
+                            Default::default()
+                        })
+                    });
+
+                    if let Some(renderer) = create_screen_renderer_wl(
+                        output,
+                        has_wlr_dmabuf,
+                        has_wlr_screencopy,
+                        pw_token_store,
+                        &app.session,
+                    ) {
+                        overlay.backend.set_renderer(Box::new(renderer));
+                    }
+                    extent_dirty = true;
+                }
             }
         }
 
-        for mut screen in screens {
+        if extent_dirty && !create_ran {
-            if show_screens.contains(&screen.state.name) {
+            let extent = wl.get_desktop_extent();
-                screen.state.show_hide = true;
+            let origin = wl.get_desktop_origin();
-                screen.state.want_visible = false;
+            app.hid_provider
-            }
+                .set_desktop_extent(vec2(extent.0 as f32, extent.1 as f32));
-            overlays.insert(screen.state.id, screen);
+            app.hid_provider
+                .set_desktop_origin(vec2(origin.0 as f32, origin.1 as f32));
         }
-        Ok(Self { overlays, extent })
+
+        if watch_dirty {
+            let watch = self.mut_by_name(WATCH_NAME).unwrap(); // want panic
+            match create_watch_canvas(None, app) {
+                Ok(canvas) => {
+                    watch.backend = Box::new(canvas);
+                }
+                Err(e) => {
+                    log::error!("Failed to create watch canvas: {}", e);
+                }
+            }
+        }
+
+        Ok(removed_overlays)
     }
 
     pub fn mut_by_selector(&mut self, selector: &OverlaySelector) -> Option<&mut OverlayData<T>> {

src/backend/openvr/mod.rs

@@ -118,8 +118,6 @@ pub fn openvr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
     let mut osc_sender =
         crate::backend::osc::OscSender::new(state.session.config.osc_out_port).ok();
 
-    state.hid_provider.set_desktop_extent(overlays.extent);
-
     set_action_manifest(&mut input_mgr)?;
 
     let mut input_source = OpenVrInputSource::new(&mut input_mgr)?;
@@ -203,6 +201,12 @@ pub fn openvr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
         notifications.submit_pending(&mut state);
 
         state.tasks.retrieve_due(&mut due_tasks);
+
+        let mut removed_overlays = overlays.update(&mut state)?;
+        for o in removed_overlays.iter_mut() {
+            o.destroy(&mut overlay_mgr);
+        }
+
         while let Some(task) = due_tasks.pop_front() {
             match task {
                 TaskType::Global(f) => f(&mut state),

src/backend/openxr/mod.rs

@@ -78,8 +78,6 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
     let mut osc_sender =
         crate::backend::osc::OscSender::new(app_state.session.config.osc_out_port).ok();
 
-    app_state.hid_provider.set_desktop_extent(overlays.extent);
-
     let (session, mut frame_wait, mut frame_stream) = unsafe {
         let raw_session = helpers::create_overlay_session(
             &xr_instance,
@@ -322,6 +320,8 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
             &frame_ref,
         )?;
 
+        let _ = overlays.update(&mut app_state)?;
+
         notifications.submit_pending(&mut app_state);
 
         app_state.tasks.retrieve_due(&mut due_tasks);

src/backend/overlay.rs

@@ -16,7 +16,10 @@ use super::input::{DummyInteractionHandler, Haptics, InteractionHandler, Pointer
 
 static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(0);
 
-pub trait OverlayBackend: OverlayRenderer + InteractionHandler {}
+pub trait OverlayBackend: OverlayRenderer + InteractionHandler {
+    fn set_renderer(&mut self, renderer: Box<dyn OverlayRenderer>);
+    fn set_interaction(&mut self, interaction: Box<dyn InteractionHandler>);
+}
 
 pub struct OverlayState {
     pub id: usize,
@@ -258,7 +261,14 @@ impl Default for SplitOverlayBackend {
     }
 }
 
-impl OverlayBackend for SplitOverlayBackend {}
+impl OverlayBackend for SplitOverlayBackend {
+    fn set_renderer(&mut self, renderer: Box<dyn OverlayRenderer>) {
+        self.renderer = renderer;
+    }
+    fn set_interaction(&mut self, interaction: Box<dyn InteractionHandler>) {
+        self.interaction = interaction;
+    }
+}
 impl OverlayRenderer for SplitOverlayBackend {
     fn init(&mut self, app: &mut AppState) -> anyhow::Result<()> {
         self.renderer.init(app)

src/gui/mod.rs

@@ -514,7 +514,10 @@ impl<D, S> OverlayRenderer for Canvas<D, S> {
     }
 }
 
-impl<D, S> OverlayBackend for Canvas<D, S> {}
+impl<D, S> OverlayBackend for Canvas<D, S> {
+    fn set_renderer(&mut self, _renderer: Box<dyn OverlayRenderer>) {}
+    fn set_interaction(&mut self, _interaction: Box<dyn InteractionHandler>) {}
+}
 
 pub struct Control<D, S> {
     pub state: Option<S>,

src/hid.rs

@@ -35,12 +35,14 @@ pub trait HidProvider {
     fn set_modifiers(&mut self, mods: u8);
     fn send_key(&self, key: u16, down: bool);
     fn set_desktop_extent(&mut self, extent: Vec2);
+    fn set_desktop_origin(&mut self, origin: Vec2);
     fn on_new_frame(&mut self);
 }
 
 pub struct UInputProvider {
     handle: UInputHandle<File>,
     desktop_extent: Vec2,
+    desktop_origin: Vec2,
     mouse_moved: bool,
     cur_modifiers: u8,
 }
@@ -135,6 +137,7 @@ impl UInputProvider {
                 return Some(UInputProvider {
                     handle,
                     desktop_extent: Vec2::ZERO,
+                    desktop_origin: Vec2::ZERO,
                     mouse_moved: false,
                     cur_modifiers: 0,
                 });
@@ -154,7 +157,7 @@ impl HidProvider for UInputProvider {
         #[cfg(debug_assertions)]
         log::trace!("Mouse move: {:?}", pos);
 
-        let pos = pos * (MOUSE_EXTENT / self.desktop_extent);
+        let pos = (pos - self.desktop_origin) * (MOUSE_EXTENT / self.desktop_extent);
 
         let time = get_time();
         let events = [
@@ -209,9 +212,11 @@ impl HidProvider for UInputProvider {
         }
     }
     fn set_desktop_extent(&mut self, extent: Vec2) {
-        log::info!("Desktop extent: {:?}", extent);
         self.desktop_extent = extent;
     }
+    fn set_desktop_origin(&mut self, origin: Vec2) {
+        self.desktop_origin = origin;
+    }
     fn on_new_frame(&mut self) {
         self.mouse_moved = false;
     }
@@ -224,6 +229,7 @@ impl HidProvider for DummyProvider {
     fn set_modifiers(&mut self, _modifiers: u8) {}
     fn send_key(&self, _key: u16, _down: bool) {}
     fn set_desktop_extent(&mut self, _extent: Vec2) {}
+    fn set_desktop_origin(&mut self, _origin: Vec2) {}
     fn on_new_frame(&mut self) {}
 }
 

src/overlays/screen.rs

@@ -40,12 +40,12 @@ use glam::{vec2, vec3a, Affine2, Quat, Vec2};
 use crate::{
     backend::{
         input::{Haptics, InteractionHandler, PointerHit, PointerMode},
-        overlay::{OverlayData, OverlayRenderer, OverlayState, SplitOverlayBackend},
+        overlay::{OverlayRenderer, OverlayState, SplitOverlayBackend},
     },
     config::def_pw_tokens,
     graphics::{fourcc_to_vk, WlxCommandBuffer, WlxPipeline, WlxPipelineLegacy},
     hid::{MOUSE_LEFT, MOUSE_MIDDLE, MOUSE_RIGHT},
-    state::{AppSession, AppState},
+    state::{AppSession, AppState, ScreenMeta},
 };
 
 const CURSOR_SIZE: f32 = 16. / 1440.;
@@ -130,6 +130,7 @@ impl InteractionHandler for ScreenInteractionHandler {
     fn on_left(&mut self, _app: &mut AppState, _hand: usize) {}
 }
 
+#[derive(Clone)]
 struct ScreenPipeline {
     view: Arc<ImageView>,
     mouse: Option<Arc<ImageView>>,
@@ -500,36 +501,22 @@ impl OverlayRenderer for ScreenRenderer {
 }
 
 #[cfg(feature = "wayland")]
-/// Panics if id is not a valid output id
+pub fn create_screen_renderer_wl(
-fn try_create_screen<O>(
+    output: &WlxOutput,
-    wl: &WlxClient,
+    has_wlr_dmabuf: bool,
-    id: u32,
+    has_wlr_screencopy: bool,
     pw_token_store: &mut HashMap<String, String>,
     session: &AppSession,
-) -> Option<OverlayData<O>>
+) -> Option<ScreenRenderer> {
-where
-    O: Default,
-{
-    let output = &wl.outputs.get(id).unwrap(); // safe due to contract
-    log::info!(
-        "{}: Res {}x{} Size {:?} Pos {:?}",
-        output.name,
-        output.size.0,
-        output.size.1,
-        output.logical_size,
-        output.logical_pos,
-    );
-
     let mut capture: Option<ScreenRenderer> = None;
-
     if (&*session.config.capture_method == "auto" || &*session.config.capture_method == "dmabuf")
-        && wl.maybe_wlr_dmabuf_mgr.is_some()
+        && has_wlr_dmabuf
     {
         log::info!("{}: Using Wlr DMA-Buf", &output.name);
         capture = ScreenRenderer::new_wlr_dmabuf(output);
     }
 
-    if &*session.config.capture_method == "screencopy" && wl.maybe_wlr_screencopy_mgr.is_some() {
+    if &*session.config.capture_method == "screencopy" && has_wlr_screencopy {
         log::info!("{}: Using Wlr Screencopy Wl-SHM", &output.name);
         capture = ScreenRenderer::new_wlr_screencopy(output);
     }
@@ -572,78 +559,67 @@ where
             }
         }
     }
-    if let Some(capture) = capture {
+    capture
-        let backend = Box::new(SplitOverlayBackend {
+}
-            renderer: Box::new(capture),
-            interaction: Box::new(ScreenInteractionHandler::new(
-                vec2(output.logical_pos.0 as f32, output.logical_pos.1 as f32),
-                vec2(output.logical_size.0 as f32, output.logical_size.1 as f32),
-                output.transform.into(),
-            )),
-        });
 
-        let axis = Vec3::new(0., 0., 1.);
+pub fn create_screen_interaction(
+    logical_pos: Vec2,
+    logical_size: Vec2,
+    transform: Transform,
+) -> ScreenInteractionHandler {
+    ScreenInteractionHandler::new(logical_pos, logical_size, transform)
+}
 
-        let transform = output.transform.into();
+fn create_screen_state(
-
+    name: Arc<str>,
-        let angle = if session.config.upright_screen_fix {
+    res: (i32, i32),
-            match transform {
+    transform: Transform,
-                Transform::_90 | Transform::Flipped90 => PI / 2.,
+    session: &AppSession,
-                Transform::_180 | Transform::Flipped180 => PI,
+) -> OverlayState {
-                Transform::_270 | Transform::Flipped270 => -PI / 2.,
+    let angle = if session.config.upright_screen_fix {
-                _ => 0.,
+        match transform {
-            }
+            Transform::_90 | Transform::Flipped90 => PI / 2.,
-        } else {
+            Transform::_180 | Transform::Flipped180 => PI,
-            0.
+            Transform::_270 | Transform::Flipped270 => -PI / 2.,
-        };
+            _ => 0.,
-
+        }
-        let center = Vec2 { x: 0.5, y: 0.5 };
-        let interaction_transform = match transform {
-            Transform::_90 | Transform::Flipped90 => Affine2::from_cols(
-                Vec2::NEG_Y * (output.size.0 as f32 / output.size.1 as f32),
-                Vec2::NEG_X,
-                center,
-            ),
-            Transform::_180 | Transform::Flipped180 => Affine2::from_cols(
-                Vec2::NEG_X,
-                Vec2::NEG_Y * (-output.size.0 as f32 / output.size.1 as f32),
-                center,
-            ),
-            Transform::_270 | Transform::Flipped270 => Affine2::from_cols(
-                Vec2::Y * (output.size.0 as f32 / output.size.1 as f32),
-                Vec2::X,
-                center,
-            ),
-            _ => Affine2::from_cols(
-                Vec2::X,
-                Vec2::Y * (-output.size.0 as f32 / output.size.1 as f32),
-                center,
-            ),
-        };
-
-        Some(OverlayData {
-            state: OverlayState {
-                name: output.name.clone(),
-                show_hide: session
-                    .config
-                    .show_screens
-                    .iter()
-                    .any(|s| s.as_ref() == output.name.as_ref()),
-                grabbable: true,
-                recenter: true,
-                interactable: true,
-                spawn_scale: 1.5 * session.config.desktop_view_scale,
-                spawn_point: vec3a(0., 0.5, -1.),
-                spawn_rotation: Quat::from_axis_angle(axis, angle),
-                interaction_transform,
-                ..Default::default()
-            },
-            backend,
-            ..Default::default()
-        })
     } else {
-        log::warn!("{}: Will not be used", &output.name);
+        0.
-        None
+    };
+
+    let center = Vec2 { x: 0.5, y: 0.5 };
+    let interaction_transform = match transform {
+        Transform::_90 | Transform::Flipped90 => Affine2::from_cols(
+            Vec2::NEG_Y * (res.0 as f32 / res.1 as f32),
+            Vec2::NEG_X,
+            center,
+        ),
+        Transform::_180 | Transform::Flipped180 => Affine2::from_cols(
+            Vec2::NEG_X,
+            Vec2::NEG_Y * (-res.0 as f32 / res.1 as f32),
+            center,
+        ),
+        Transform::_270 | Transform::Flipped270 => {
+            Affine2::from_cols(Vec2::Y * (res.0 as f32 / res.1 as f32), Vec2::X, center)
+        }
+        _ => Affine2::from_cols(Vec2::X, Vec2::Y * (-res.0 as f32 / res.1 as f32), center),
+    };
+
+    OverlayState {
+        name: name.clone(),
+        show_hide: session
+            .config
+            .show_screens
+            .iter()
+            .any(|s| s.as_ref() == name.as_ref()),
+        grabbable: true,
+        recenter: true,
+        interactable: true,
+        spawn_scale: 1.5 * session.config.desktop_view_scale,
+        spawn_point: vec3a(0., 0.5, -1.),
+        spawn_rotation: Quat::from_axis_angle(Vec3::Z, angle),
+        interaction_transform,
+        ..Default::default()
     }
 }
 
@@ -688,21 +664,24 @@ pub fn load_pw_token_config() -> Result<HashMap<String, String>, Box<dyn Error>>
     Ok(map)
 }
 
+pub(crate) struct ScreenCreateData {
+    pub screens: Vec<(ScreenMeta, OverlayState, Box<SplitOverlayBackend>)>,
+}
+
 #[cfg(not(feature = "wayland"))]
-pub fn get_screens_wayland<O>(_session: &AppSession) -> anyhow::Result<(Vec<OverlayData<O>>, Vec2)>
+pub fn create_screens_wayland(
-where
+    wl: &mut WlxClient,
-    O: Default,
+    app: &AppState,
-{
+) -> anyhow::Result<ScreenCreateData> {
     anyhow::bail!("Wayland support not enabled")
 }
 
 #[cfg(feature = "wayland")]
-pub fn get_screens_wayland<O>(session: &AppSession) -> anyhow::Result<(Vec<OverlayData<O>>, Vec2)>
+pub fn create_screens_wayland(
-where
+    wl: &mut WlxClient,
-    O: Default,
+    app: &mut AppState,
-{
+) -> anyhow::Result<ScreenCreateData> {
-    let mut overlays = vec![];
+    let mut screens = vec![];
-    let mut wl = WlxClient::new().ok_or_else(|| anyhow::anyhow!("Failed to connect to Wayland"))?;
 
     // Load existing Pipewire tokens from file
     let mut pw_tokens: HashMap<String, String> = if let Ok(conf) = load_pw_token_config() {
@@ -712,24 +691,47 @@ where
     };
 
     let pw_tokens_copy = pw_tokens.clone();
+    let has_wlr_dmabuf = wl.maybe_wlr_dmabuf_mgr.is_some();
+    let has_wlr_screencopy = wl.maybe_wlr_screencopy_mgr.is_some();
 
-    let mut origin = (i32::MAX, i32::MAX);
+    for (id, output) in wl.outputs.iter() {
-    for output in wl.outputs.values() {
+        if app.screens.iter().any(|s| s.name == output.name) {
-        origin.0 = origin.0.min(output.logical_pos.0);
+            continue;
-        origin.1 = origin.1.min(output.logical_pos.1);
+        }
-    }
 
-    log::info!("Desktop origin: {:?}", origin);
+        log::info!(
+            "{}: Init screen of res {:?}, logical {:?} at {:?}",
+            output.name,
+            output.size,
+            output.logical_size,
+            output.logical_pos,
+        );
 
-    // adjust all outputs so that the top-left corner is at (0, 0)
+        if let Some(renderer) = create_screen_renderer_wl(
-    for output in wl.outputs.values_mut() {
+            output,
-        output.logical_pos.0 -= origin.0;
+            has_wlr_dmabuf,
-        output.logical_pos.1 -= origin.1;
+            has_wlr_screencopy,
-    }
+            &mut pw_tokens,
+            &app.session,
+        ) {
+            let logical_pos = vec2(output.logical_pos.0 as f32, output.logical_pos.1 as f32);
+            let logical_size = vec2(output.logical_size.0 as f32, output.logical_size.1 as f32);
+            let transform = output.transform.into();
+            let interaction = create_screen_interaction(logical_pos, logical_size, transform);
+            let state =
+                create_screen_state(output.name.clone(), output.size, transform, &app.session);
 
-    for id in wl.outputs.keys() {
+            let meta = ScreenMeta {
-        if let Some(overlay) = try_create_screen(&wl, *id, &mut pw_tokens, session) {
+                name: wl.outputs[id].name.clone(),
-            overlays.push(overlay);
+                id: state.id,
+                native_handle: *id,
+            };
+
+            let backend = Box::new(SplitOverlayBackend {
+                renderer: Box::new(renderer),
+                interaction: Box::new(interaction),
+            });
+            screens.push((meta, state, backend));
         }
     }
 
@@ -741,22 +743,23 @@ where
     }
 
     let extent = wl.get_desktop_extent();
-    Ok((overlays, Vec2::new(extent.0 as f32, extent.1 as f32)))
+    let origin = wl.get_desktop_origin();
+
+    app.hid_provider
+        .set_desktop_extent(vec2(extent.0 as f32, extent.1 as f32));
+    app.hid_provider
+        .set_desktop_origin(vec2(origin.0 as f32, origin.1 as f32));
+
+    Ok(ScreenCreateData { screens })
 }
 
 #[cfg(not(feature = "x11"))]
-pub fn get_screens_x11<O>(_session: &AppSession) -> anyhow::Result<(Vec<OverlayData<O>>, Vec2)>
+pub fn create_screens_x11(session: &AppSession) -> anyhow::Result<ScreenCreateData> {
-where
-    O: Default,
-{
     anyhow::bail!("X11 support not enabled")
 }
 
 #[cfg(feature = "x11")]
-pub fn get_screens_x11<O>(session: &AppSession) -> anyhow::Result<(Vec<OverlayData<O>>, Vec2)>
+pub fn create_screens_x11(app: &mut AppState) -> anyhow::Result<ScreenCreateData> {
-where
-    O: Default,
-{
     use anyhow::bail;
 
     let mut extent = vec2(0., 0.);
@@ -768,64 +771,52 @@ where
         }
     };
 
-    let overlays = monitors
+    let screens = monitors
         .into_iter()
         .map(|s| {
-            log::info!(
-                "{}: Res {}x{}, Pos {}x{}",
-                s.name,
-                s.monitor.width(),
-                s.monitor.height(),
-                s.monitor.x(),
-                s.monitor.y()
-            );
-            let size = (s.monitor.width(), s.monitor.height());
-            let capture: ScreenRenderer = ScreenRenderer::new_xshm(s.clone());
-
-            let backend = Box::new(SplitOverlayBackend {
-                renderer: Box::new(capture),
-                interaction: Box::new(ScreenInteractionHandler::new(
-                    vec2(s.monitor.x() as f32, s.monitor.y() as f32),
-                    vec2(s.monitor.width() as f32, s.monitor.height() as f32),
-                    Transform::Normal,
-                )),
-            });
-
-            let interaction_transform = Affine2::from_translation(Vec2 { x: 0.5, y: 0.5 })
-                * Affine2::from_scale(Vec2 {
-                    x: 1.,
-                    y: -size.0 as f32 / size.1 as f32,
-                });
-
             extent.x = extent.x.max((s.monitor.x() + s.monitor.width()) as f32);
             extent.y = extent.y.max((s.monitor.y() + s.monitor.height()) as f32);
-            OverlayData {
+
-                state: OverlayState {
+            let size = (s.monitor.width(), s.monitor.height());
-                    name: s.name.clone(),
+            let pos = (s.monitor.x(), s.monitor.y());
-                    show_hide: session
+            let renderer = ScreenRenderer::new_xshm(s.clone());
-                        .config
+
-                        .show_screens
+            log::info!(
-                        .iter()
+                "{}: Init screen of res {:?} at {:?}",
-                        .any(|x| x.as_ref() == s.name.as_ref()),
+                s.name.clone(),
-                    grabbable: true,
+                size,
-                    recenter: true,
+                pos,
-                    interactable: true,
+            );
-                    spawn_scale: 1.5 * session.config.desktop_view_scale,
+
-                    spawn_point: vec3a(0., 0.5, -1.),
+            let interaction = create_screen_interaction(
-                    spawn_rotation: Quat::IDENTITY,
+                vec2(s.monitor.x() as f32, s.monitor.y() as f32),
-                    interaction_transform,
+                vec2(size.0 as f32, size.1 as f32),
-                    ..Default::default()
+                Transform::Normal,
-                },
+            );
-                backend,
+
-                ..Default::default()
+            let state = create_screen_state(s.name.clone(), size, Transform::Normal, &app.session);
-            }
+
+            let meta = ScreenMeta {
+                name: s.name.clone(),
+                id: state.id,
+                native_handle: 0,
+            };
+
+            let backend = Box::new(SplitOverlayBackend {
+                renderer: Box::new(renderer),
+                interaction: Box::new(interaction),
+            });
+            (meta, state, backend)
         })
         .collect();
 
-    Ok((overlays, extent))
+    app.hid_provider.set_desktop_extent(extent);
+
+    Ok(ScreenCreateData { screens })
 }
 
 #[allow(unused)]
+#[derive(Clone, Copy)]
 pub enum Transform {
     Normal,
     _90,

src/overlays/watch.rs

@@ -8,7 +8,10 @@ use crate::{
         def_half, def_left, def_point7, def_watch_pos, def_watch_rot, load_known_yaml, ConfigType,
     },
     config_io,
-    gui::modular::{modular_canvas, ModularUiConfig},
+    gui::{
+        modular::{modular_canvas, ModularData, ModularUiConfig},
+        Canvas,
+    },
     state::{AppState, LeftRight},
 };
 
@@ -20,8 +23,6 @@ where
 {
     let config = load_known_yaml::<ModularUiConfig>(ConfigType::Watch);
 
-    let canvas = modular_canvas(&config.size, &config.elements, state)?;
-
     let relative_to = RelativeTo::Hand(state.session.config.watch_hand as usize);
 
     Ok(OverlayData {
@@ -36,11 +37,20 @@ where
             relative_to,
             ..Default::default()
         },
-        backend: Box::new(canvas),
+        backend: Box::new(create_watch_canvas(Some(config), state)?),
         ..Default::default()
     })
 }
 
+pub fn create_watch_canvas(
+    config: Option<ModularUiConfig>,
+    state: &AppState,
+) -> anyhow::Result<Canvas<(), ModularData>> {
+    let config = config.unwrap_or_else(|| load_known_yaml::<ModularUiConfig>(ConfigType::Watch));
+
+    modular_canvas(&config.size, &config.elements, state)
+}
+
 pub fn watch_fade<D>(app: &mut AppState, watch: &mut OverlayData<D>)
 where
     D: Default,

src/state.rs

@@ -172,6 +172,7 @@ impl AudioOutput {
 pub struct ScreenMeta {
     pub name: Arc<str>,
     pub id: usize,
+    pub native_handle: u32,
 }
 
 #[derive(Serialize, Deserialize, Clone, Copy, Default)]