feat: PipeWire capture on X11

Co-authored-by: Yuxuan Shui <yshuiv7@gmail.com>

Cargo.lock

@@ -4277,7 +4277,7 @@ dependencies = [
 [[package]]
 name = "wlx-capture"
 version = "0.3.9"
-source = "git+https://github.com/galister/wlx-capture?tag=v0.3.9#0552577c45ce135a2f42d11399bcca0034762e2a"
+source = "git+https://github.com/galister/wlx-capture?tag=v0.3.10#dacce353e4a0a4778323e96540c053ffefe82bfe"
 dependencies = [
  "ashpd",
  "drm-fourcc",

Cargo.toml

@@ -53,26 +53,24 @@ sysinfo = { version = "0.30.0", optional = true }
 thiserror = "1.0.56"
 vulkano = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
 vulkano-shaders = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
-wlx-capture = { git = "https://github.com/galister/wlx-capture", tag = "v0.3.9", default-features = false }
+wlx-capture = { git = "https://github.com/galister/wlx-capture", tag = "v0.3.10", default-features = false }
 winit = { version = "0.29.15", optional = true }
 xdg = "2.5.2"
 log-panics = { version = "2.1.0", features = ["with-backtrace"] }
 serde_json5 = "0.1.0"
 xkbcommon = { version = "0.7.0" }
-xcb = { version = "1.4.0", optional = true, features = ["as-raw-xcb-connection"] }
+xcb = { version = "1.4.0", optional = true, features = [
+  "as-raw-xcb-connection",
+] }
 
 [features]
 default = ["openvr", "openxr", "osc", "x11", "wayland"]
 openvr = ["dep:ovr_overlay", "dep:json"]
 openxr = ["dep:openxr", "dep:sysinfo"]
 osc = ["dep:rosc"]
-x11 = [
+x11 = ["dep:xcb", "wlx-capture/xshm", "xkbcommon/x11"]
-  "dep:xcb",
+wayland = ["pipewire", "wlx-capture/wlr", "xkbcommon/wayland"]
-
+pipewire = ["wlx-capture/pipewire"]
-  "wlx-capture/xshm",
-  "xkbcommon/x11",
-]
-wayland = ["wlx-capture/pipewire", "wlx-capture/wlr", "xkbcommon/wayland"]
 uidev = ["dep:winit"]
 no-dmabuf = []
 xcb = ["dep:xcb"]

src/backend/common.rs

@@ -72,7 +72,13 @@ where
             crate::overlays::screen::create_screens_wayland(wl, app)?
         } else {
             keymap = get_keymap_x11().ok();
-            crate::overlays::screen::create_screens_x11(app)?
+            match crate::overlays::screen::create_screens_x11pw(app) {
+                Ok(data) => data,
+                Err(e) => {
+                    log::info!("Will not use PipeWire capture: {:?}", e);
+                    crate::overlays::screen::create_screens_xshm(app)?
+                }
+            }
         };
 
         let mut show_screens = app.session.config.show_screens.clone();

src/hid/mod.rs

@@ -584,10 +584,14 @@ impl XkbKeymap {
 pub use wayland::get_keymap_wl;
 
 #[cfg(not(feature = "wayland"))]
-pub fn get_keymap_wl() -> anyhow::Result<XkbKeymap> {}
+pub fn get_keymap_wl() -> anyhow::Result<XkbKeymap> {
+    anyhow::bail!("Wayland support not enabled.")
+}
 
 #[cfg(feature = "x11")]
 pub use x11::get_keymap_x11;
 
 #[cfg(not(feature = "x11"))]
-pub fn get_keymap_x11() -> anyhow::Result<XkbKeymap> {}
+pub fn get_keymap_x11() -> anyhow::Result<XkbKeymap> {
+    anyhow::bail!("X11 support not enabled.")
+}

src/overlays/mirror.rs

@@ -31,7 +31,7 @@ pub struct MirrorRenderer {
 }
 impl MirrorRenderer {
     pub fn new(name: Arc<str>) -> Self {
-        let selector = Box::pin(pipewire_select_screen(None, false, false, false));
+        let selector = Box::pin(pipewire_select_screen(None, false, false, false, false));
         Self {
             name,
             renderer: None,
@@ -58,12 +58,9 @@ impl OverlayRenderer for MirrorRenderer {
             };
 
             if let Ok(pw_result) = maybe_pw_result {
-                log::info!(
+                let node_id = pw_result.streams.first().unwrap().node_id; // streams guaranteed to have at least one element
-                    "{}: PipeWire node selected: {}",
+                log::info!("{}: PipeWire node selected: {}", self.name.clone(), node_id,);
-                    self.name.clone(),
+                let capture = PipewireCapture::new(self.name.clone(), node_id);
-                    pw_result.node_id
-                );
-                let capture = PipewireCapture::new(self.name.clone(), pw_result.node_id);
                 self.renderer = Some(ScreenRenderer::new_raw(
                     self.name.clone(),
                     Box::new(capture),

src/overlays/screen.rs

@@ -21,13 +21,18 @@ use wlx_capture::{
     WlxCapture,
 };
 
+#[cfg(feature = "pipewire")]
+use {
+    crate::config_io,
+    std::error::Error,
+    std::{ops::Deref, path::PathBuf},
+    wlx_capture::pipewire::{pipewire_select_screen, PipewireCapture, PipewireStream},
+};
+
 #[cfg(feature = "wayland")]
 use {
     crate::config::AStrMapExt,
-    crate::config_io,
-    std::{error::Error, ops::Deref, path::PathBuf},
     wlx_capture::{
-        pipewire::{pipewire_select_screen, PipewireCapture},
         wayland::{wayland_client::protocol::wl_output, WlxClient, WlxOutput},
         wlr_dmabuf::WlrDmabufCapture,
         wlr_screencopy::WlrScreencopyCapture,
@@ -327,10 +332,17 @@ impl ScreenRenderer {
     )> {
         let name = output.name.clone();
         let embed_mouse = !session.config.double_cursor_fix;
-        let select_screen_result =
+        let select_screen_result = futures::executor::block_on(pipewire_select_screen(
-            futures::executor::block_on(pipewire_select_screen(token, embed_mouse, true, true))?;
+            token,
+            embed_mouse,
+            true,
+            true,
+            false,
+        ))?;
 
-        let capture = PipewireCapture::new(name, select_screen_result.node_id);
+        let node_id = select_screen_result.streams.first().unwrap().node_id; // streams guaranteed to have at least one element
+
+        let capture = PipewireCapture::new(name, node_id);
 
         Ok((
             ScreenRenderer {
@@ -667,14 +679,14 @@ pub struct TokenConf {
     pub pw_tokens: PwTokenMap,
 }
 
-#[cfg(feature = "wayland")]
+#[cfg(feature = "pipewire")]
 fn get_pw_token_path() -> PathBuf {
     let mut path = config_io::get_conf_d_path();
     path.push("pw_tokens.yaml");
     path
 }
 
-#[cfg(feature = "wayland")]
+#[cfg(feature = "pipewire")]
 pub fn save_pw_token_config(tokens: PwTokenMap) -> Result<(), Box<dyn Error>> {
     let conf = TokenConf { pw_tokens: tokens };
     let yaml = serde_yaml::to_string(&conf)?;
@@ -683,7 +695,7 @@ pub fn save_pw_token_config(tokens: PwTokenMap) -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
-#[cfg(feature = "wayland")]
+#[cfg(feature = "pipewire")]
 pub fn load_pw_token_config() -> Result<PwTokenMap, Box<dyn Error>> {
     let yaml = std::fs::read_to_string(get_pw_token_path())?;
     let conf: TokenConf = serde_yaml::from_str(yaml.as_str())?;
@@ -782,12 +794,102 @@ pub fn create_screens_wayland(
 }
 
 #[cfg(not(feature = "x11"))]
-pub fn create_screens_x11(_app: &mut AppState) -> anyhow::Result<ScreenCreateData> {
+pub fn create_screens_xshm(_app: &mut AppState) -> anyhow::Result<ScreenCreateData> {
     anyhow::bail!("X11 support not enabled")
 }
 
+#[cfg(not(all(feature = "x11", feature = "pipewire")))]
+pub fn create_screens_x11pw(_app: &mut AppState) -> anyhow::Result<ScreenCreateData> {
+    anyhow::bail!("Pipewire support not enabled")
+}
+
+#[cfg(all(feature = "x11", feature = "pipewire"))]
+pub fn create_screens_x11pw(app: &mut AppState) -> anyhow::Result<ScreenCreateData> {
+    use crate::config::{AStrMap, AStrMapExt};
+    use anyhow::bail;
+
+    // Load existing Pipewire tokens from file
+    let mut pw_tokens: PwTokenMap = if let Ok(conf) = load_pw_token_config() {
+        conf
+    } else {
+        AStrMap::new()
+    };
+    let pw_tokens_copy = pw_tokens.clone();
+    let token = pw_tokens.arc_get("x11").map(|s| s.as_str());
+    let embed_mouse = !app.session.config.double_cursor_fix;
+    let select_screen_result =
+        futures::executor::block_on(pipewire_select_screen(token, embed_mouse, true, true, true))?;
+    if let Some(restore_token) = select_screen_result.restore_token {
+        if pw_tokens.arc_set("x11".into(), restore_token.clone()) {
+            log::info!("Adding Pipewire token {}", restore_token);
+        }
+    }
+    if pw_tokens_copy != pw_tokens {
+        // Token list changed, re-create token config file
+        if let Err(err) = save_pw_token_config(pw_tokens) {
+            log::error!("Failed to save Pipewire token config: {}", err);
+        }
+    }
+
+    let monitors = match XshmCapture::get_monitors() {
+        Ok(m) => m,
+        Err(e) => {
+            bail!(e.to_string());
+        }
+    };
+    log::info!("Got {} monitors", monitors.len());
+    log::info!("Got {} streams", select_screen_result.streams.len());
+
+    let mut extent = vec2(0., 0.);
+    let screens = select_screen_result
+        .streams
+        .into_iter()
+        .enumerate()
+        .map(|(i, s)| {
+            let m = best_match(&s, monitors.iter().map(AsRef::as_ref)).unwrap();
+            log::info!("Stream {i} is {}", m.name);
+            extent.x = extent.x.max((m.monitor.x() + m.monitor.width()) as f32);
+            extent.y = extent.y.max((m.monitor.y() + m.monitor.height()) as f32);
+
+            let size = (m.monitor.width(), m.monitor.height());
+            let interaction = create_screen_interaction(
+                vec2(m.monitor.x() as f32, m.monitor.y() as f32),
+                vec2(m.monitor.width() as f32, m.monitor.height() as f32),
+                Transform::Normal,
+            );
+
+            let state = create_screen_state(m.name.clone(), size, Transform::Normal, &app.session);
+
+            let meta = ScreenMeta {
+                name: m.name.clone(),
+                id: state.id,
+                native_handle: 0,
+            };
+
+            let renderer = ScreenRenderer {
+                name: m.name.clone(),
+                capture: Box::new(PipewireCapture::new(m.name.clone(), s.node_id)),
+                pipeline: None,
+                last_view: None,
+                extent: extent_from_res(size),
+            };
+
+            let backend = Box::new(SplitOverlayBackend {
+                renderer: Box::new(renderer),
+                interaction: Box::new(interaction),
+            });
+            (meta, state, backend)
+        })
+        .collect();
+
+    app.hid_provider.set_desktop_extent(extent);
+    app.hid_provider.set_desktop_origin(vec2(0.0, 0.0));
+
+    Ok(ScreenCreateData { screens })
+}
+
 #[cfg(feature = "x11")]
-pub fn create_screens_x11(app: &mut AppState) -> anyhow::Result<ScreenCreateData> {
+pub fn create_screens_xshm(app: &mut AppState) -> anyhow::Result<ScreenCreateData> {
     use anyhow::bail;
 
     let mut extent = vec2(0., 0.);
@@ -839,6 +941,7 @@ pub fn create_screens_x11(app: &mut AppState) -> anyhow::Result<ScreenCreateData
         .collect();
 
     app.hid_provider.set_desktop_extent(extent);
+    app.hid_provider.set_desktop_origin(vec2(0.0, 0.0));
 
     Ok(ScreenCreateData { screens })
 }
@@ -880,3 +983,31 @@ fn extent_from_res(res: (i32, i32)) -> [u32; 3] {
     let h = (res.1 as f32 / res.0 as f32 * w as f32) as u32;
     [w, h, 1]
 }
+
+#[cfg(all(feature = "pipewire", feature = "x11"))]
+fn best_match<'a>(
+    stream: &PipewireStream,
+    mut streams: impl Iterator<Item = &'a XshmScreen>,
+) -> Option<&'a XshmScreen> {
+    let mut best = streams.next();
+    log::debug!("stream: {:?}", stream.position);
+    log::debug!("first: {:?}", best.map(|b| &b.monitor));
+    let Some(position) = stream.position else {
+        return best;
+    };
+
+    let mut best_dist = best
+        .map(|b| (b.monitor.x() - position.0).abs() + (b.monitor.y() - position.1).abs())
+        .unwrap_or(i32::MAX);
+    for stream in streams {
+        log::debug!("checking: {:?}", stream.monitor);
+        let dist =
+            (stream.monitor.x() - position.0).abs() + (stream.monitor.y() - position.1).abs();
+        if dist < best_dist {
+            best = Some(stream);
+            best_dist = dist;
+        }
+    }
+    log::debug!("best: {:?}", best.map(|b| &b.monitor));
+    best
+}