Integrate WayVR into wlx directly

src/overlays/keyboard.rs

@@ -18,7 +18,7 @@ use crate::{
         get_key_type, KeyModifier, KeyType, VirtualKey, XkbKeymap, ALT, CTRL, KEYS_TO_MODS, META,
         NUM_LOCK, SHIFT, SUPER,
     },
-    state::AppState,
+    state::{AppState, KeyboardFocus},
 };
 use glam::{vec2, vec3a, Affine2, Vec4};
 use once_cell::sync::Lazy;
@@ -31,6 +31,36 @@ const AUTO_RELEASE_MODS: [KeyModifier; 5] = [SHIFT, CTRL, ALT, SUPER, META];
 
 pub const KEYBOARD_NAME: &str = "kbd";
 
+fn send_key(app: &mut AppState, key: VirtualKey, down: bool) {
+    log::info!(
+        "Sending key {:?} to {:?} (down: {})",
+        key,
+        app.keyboard_focus,
+        down
+    );
+    match app.keyboard_focus {
+        KeyboardFocus::PhysicalScreen => {
+            app.hid_provider.send_key(key, down);
+        }
+        KeyboardFocus::WayVR =>
+        {
+            #[cfg(feature = "wayvr")]
+            if let Some(wayvr) = &app.wayvr {
+                wayvr.borrow_mut().send_key(key as u32, down);
+            }
+        }
+    }
+}
+
+fn set_modifiers(app: &mut AppState, mods: u8) {
+    match app.keyboard_focus {
+        KeyboardFocus::PhysicalScreen => {
+            app.hid_provider.set_modifiers(mods);
+        }
+        KeyboardFocus::WayVR => {}
+    }
+}
+
 pub fn create_keyboard<O>(
     app: &AppState,
     keymap: Option<XkbKeymap>,
@@ -197,22 +227,22 @@ fn key_press(
 
             if let PointerMode::Right = mode {
                 data.modifiers |= SHIFT;
-                app.hid_provider.set_modifiers(data.modifiers);
+                set_modifiers(app, data.modifiers);
             }
 
-            app.hid_provider.send_key(*vk, true);
+            send_key(app, *vk, true);
             *pressed = true;
         }
         Some(KeyButtonData::Modifier { modifier, sticky }) => {
             *sticky = data.modifiers & *modifier == 0;
             data.modifiers |= *modifier;
             data.key_click(app);
-            app.hid_provider.set_modifiers(data.modifiers);
+            set_modifiers(app, data.modifiers);
         }
         Some(KeyButtonData::Macro { verbs }) => {
             data.key_click(app);
             for (vk, press) in verbs {
-                app.hid_provider.send_key(*vk, *press);
+                send_key(app, *vk, *press);
             }
         }
         Some(KeyButtonData::Exec { program, args, .. }) => {
@@ -236,20 +266,20 @@ fn key_release(
 ) {
     match control.state.as_mut() {
         Some(KeyButtonData::Key { vk, pressed }) => {
-            app.hid_provider.send_key(*vk, false);
+            send_key(app, *vk, false);
             *pressed = false;
 
             for m in AUTO_RELEASE_MODS.iter() {
                 if data.modifiers & *m != 0 {
                     data.modifiers &= !*m;
-                    app.hid_provider.set_modifiers(data.modifiers);
+                    set_modifiers(app, data.modifiers);
                 }
             }
         }
         Some(KeyButtonData::Modifier { modifier, sticky }) => {
             if !*sticky {
                 data.modifiers &= !*modifier;
-                app.hid_provider.set_modifiers(data.modifiers);
+                set_modifiers(app, data.modifiers);
             }
         }
         Some(KeyButtonData::Exec {
@@ -493,7 +523,7 @@ impl OverlayRenderer for KeyboardBackend {
     }
     fn pause(&mut self, app: &mut AppState) -> anyhow::Result<()> {
         self.canvas.data_mut().modifiers = 0;
-        app.hid_provider.set_modifiers(0);
+        set_modifiers(app, 0);
         self.canvas.pause(app)
     }
     fn resume(&mut self, app: &mut AppState) -> anyhow::Result<()> {

src/overlays/mod.rs

@@ -6,3 +6,6 @@ pub mod mirror;
 pub mod screen;
 pub mod toast;
 pub mod watch;
+
+#[cfg(feature = "wayvr")]
+pub mod wayvr;

src/overlays/screen.rs

@@ -58,7 +58,7 @@ use crate::{
         fourcc_to_vk, WlxCommandBuffer, WlxPipeline, WlxPipelineLegacy, DRM_FORMAT_MOD_INVALID,
     },
     hid::{MOUSE_LEFT, MOUSE_MIDDLE, MOUSE_RIGHT},
-    state::{AppSession, AppState, ScreenMeta},
+    state::{AppSession, AppState, KeyboardFocus, ScreenMeta},
 };
 
 #[cfg(feature = "wayland")]
@@ -712,6 +712,7 @@ fn create_screen_state(
 
     OverlayState {
         name: name.clone(),
+        keyboard_focus: Some(KeyboardFocus::PhysicalScreen),
         grabbable: true,
         recenter: true,
         anchored: true,

src/overlays/wayvr.rs

@@ -0,0 +1,272 @@
+use glam::{vec3a, Affine2};
+use std::{cell::RefCell, rc::Rc, sync::Arc};
+use vulkano::image::SubresourceLayout;
+use wlx_capture::frame::{DmabufFrame, FourCC, FrameFormat, FramePlane};
+
+use crate::{
+    backend::{
+        input::{self, InteractionHandler},
+        overlay::{ui_transform, OverlayData, OverlayRenderer, OverlayState, SplitOverlayBackend},
+        wayvr,
+    },
+    graphics::WlxGraphics,
+    state::{self, KeyboardFocus},
+};
+
+pub struct WayVRContext {
+    wayvr: Rc<RefCell<wayvr::WayVR>>,
+    display: wayvr::display::DisplayHandle,
+    width: u32,
+    height: u32,
+}
+
+#[derive(Default)]
+pub struct WayVRProcess<'a> {
+    pub exec_path: &'a str,
+    pub args: &'a [&'a str],
+    pub env: &'a [(&'a str, &'a str)],
+}
+
+impl WayVRContext {
+    pub fn new(
+        wvr: Rc<RefCell<wayvr::WayVR>>,
+        width: u32,
+        height: u32,
+        processes: &[WayVRProcess],
+    ) -> anyhow::Result<Self> {
+        let mut wayvr = wvr.borrow_mut();
+
+        let display = wayvr.create_display(width, height)?;
+
+        for process in processes {
+            wayvr.spawn_process(display, process.exec_path, process.args, process.env)?;
+        }
+
+        Ok(Self {
+            wayvr: wvr.clone(),
+            display,
+            width,
+            height,
+        })
+    }
+}
+
+pub struct WayVRInteractionHandler {
+    context: Rc<RefCell<WayVRContext>>,
+    mouse_transform: Affine2,
+}
+
+impl WayVRInteractionHandler {
+    pub fn new(context: Rc<RefCell<WayVRContext>>, mouse_transform: Affine2) -> Self {
+        Self {
+            context,
+            mouse_transform,
+        }
+    }
+}
+
+impl InteractionHandler for WayVRInteractionHandler {
+    fn on_hover(
+        &mut self,
+        _app: &mut state::AppState,
+        hit: &input::PointerHit,
+    ) -> Option<input::Haptics> {
+        let ctx = self.context.borrow();
+
+        let pos = self.mouse_transform.transform_point2(hit.uv);
+        let x = ((pos.x * ctx.width as f32) as i32).max(0);
+        let y = ((pos.y * ctx.height as f32) as i32).max(0);
+
+        let ctx = self.context.borrow();
+        ctx.wayvr
+            .borrow_mut()
+            .send_mouse_move(ctx.display, x as u32, y as u32);
+
+        None
+    }
+
+    fn on_left(&mut self, _app: &mut state::AppState, _pointer: usize) {
+        // Ignore event
+    }
+
+    fn on_pointer(&mut self, _app: &mut state::AppState, hit: &input::PointerHit, pressed: bool) {
+        if let Some(index) = match hit.mode {
+            input::PointerMode::Left => Some(wayvr::MouseIndex::Left),
+            input::PointerMode::Middle => Some(wayvr::MouseIndex::Center),
+            input::PointerMode::Right => Some(wayvr::MouseIndex::Right),
+            _ => {
+                // Unknown pointer event, ignore
+                None
+            }
+        } {
+            let ctx = self.context.borrow();
+            let mut wayvr = ctx.wayvr.borrow_mut();
+            if pressed {
+                wayvr.send_mouse_down(ctx.display, index);
+            } else {
+                wayvr.send_mouse_up(ctx.display, index);
+            }
+        }
+    }
+
+    fn on_scroll(&mut self, _app: &mut state::AppState, _hit: &input::PointerHit, delta: f32) {
+        let ctx = self.context.borrow();
+        ctx.wayvr.borrow_mut().send_mouse_scroll(ctx.display, delta);
+    }
+}
+
+pub struct WayVRRenderer {
+    dmabuf_image: Option<Arc<vulkano::image::Image>>,
+    view: Option<Arc<vulkano::image::view::ImageView>>,
+    context: Rc<RefCell<WayVRContext>>,
+    graphics: Arc<WlxGraphics>,
+    width: u32,
+    height: u32,
+}
+
+impl WayVRRenderer {
+    pub fn new(
+        app: &mut state::AppState,
+        wvr: Rc<RefCell<wayvr::WayVR>>,
+        width: u32,
+        height: u32,
+        processes: &[WayVRProcess],
+    ) -> anyhow::Result<Self> {
+        Ok(Self {
+            context: Rc::new(RefCell::new(WayVRContext::new(
+                wvr, width, height, processes,
+            )?)),
+            width,
+            height,
+            dmabuf_image: None,
+            view: None,
+            graphics: app.graphics.clone(),
+        })
+    }
+}
+
+impl WayVRRenderer {
+    fn ensure_dmabuf(&mut self, data: wayvr::egl_data::DMAbufData) -> anyhow::Result<()> {
+        if self.dmabuf_image.is_none() {
+            // First init
+            let mut planes = [FramePlane::default(); 4];
+            planes[0].fd = Some(data.fd);
+            planes[0].offset = data.offset as u32;
+            planes[0].stride = data.stride;
+
+            let frame = DmabufFrame {
+                format: FrameFormat {
+                    width: self.width,
+                    height: self.height,
+                    fourcc: FourCC {
+                        value: data.mod_info.fourcc,
+                    },
+                    modifier: data.mod_info.modifiers[0], /* possibly not proper? */
+                },
+                num_planes: 1,
+                planes,
+            };
+
+            let layouts: Vec<SubresourceLayout> = vec![SubresourceLayout {
+                offset: data.offset as _,
+                size: 0,
+                row_pitch: data.stride as _,
+                array_pitch: None,
+                depth_pitch: None,
+            }];
+
+            let tex = self.graphics.dmabuf_texture_ex(
+                frame,
+                vulkano::image::ImageTiling::DrmFormatModifier,
+                layouts,
+                data.mod_info.modifiers,
+            )?;
+            self.dmabuf_image = Some(tex.clone());
+            self.view = Some(vulkano::image::view::ImageView::new_default(tex).unwrap());
+        }
+
+        Ok(())
+    }
+}
+
+impl OverlayRenderer for WayVRRenderer {
+    fn init(&mut self, _app: &mut state::AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+
+    fn pause(&mut self, _app: &mut state::AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+
+    fn resume(&mut self, _app: &mut state::AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+
+    fn render(&mut self, _app: &mut state::AppState) -> anyhow::Result<()> {
+        let ctx = self.context.borrow();
+        let mut wayvr = ctx.wayvr.borrow_mut();
+
+        wayvr.tick_display(ctx.display)?;
+
+        let dmabuf_data = wayvr
+            .get_dmabuf_data(ctx.display)
+            .ok_or(anyhow::anyhow!("Failed to fetch dmabuf data"))?
+            .clone();
+
+        drop(wayvr);
+        drop(ctx);
+        self.ensure_dmabuf(dmabuf_data.clone())?;
+
+        Ok(())
+    }
+
+    fn view(&mut self) -> Option<Arc<vulkano::image::view::ImageView>> {
+        self.view.clone()
+    }
+
+    fn extent(&mut self) -> Option<[u32; 3]> {
+        self.view.as_ref().map(|view| view.image().extent())
+    }
+}
+
+#[allow(dead_code)]
+pub fn create_wayvr<O>(
+    app: &mut state::AppState,
+    width: u32,
+    height: u32,
+    processes: &[WayVRProcess],
+) -> anyhow::Result<OverlayData<O>>
+where
+    O: Default,
+{
+    let transform = ui_transform(&[width, height]);
+
+    let state = OverlayState {
+        name: format!("WayVR Screen ({}x{})", width, height).into(),
+        keyboard_focus: Some(KeyboardFocus::WayVR),
+        want_visible: true,
+        interactable: true,
+        recenter: true,
+        grabbable: true,
+        spawn_scale: 1.0,
+        spawn_point: vec3a(0.0, -0.5, 0.0),
+        interaction_transform: transform,
+        ..Default::default()
+    };
+
+    let wayvr = app.get_wayvr()?;
+
+    let renderer = WayVRRenderer::new(app, wayvr, width, height, processes)?;
+    let context = renderer.context.clone();
+
+    let backend = Box::new(SplitOverlayBackend {
+        renderer: Box::new(renderer),
+        interaction: Box::new(WayVRInteractionHandler::new(context, Affine2::IDENTITY)),
+    });
+
+    Ok(OverlayData {
+        state,
+        backend,
+        ..Default::default()
+    })
+}