omg: curved screens

src/backend/common.rs

@@ -1,5 +1,6 @@
 use std::{
     collections::{BinaryHeap, VecDeque},
+    f32::consts::PI,
     sync::Arc,
     time::Instant,
 };
@@ -7,7 +8,7 @@ use std::{
 #[cfg(feature = "openxr")]
 use openxr as xr;
 
-use glam::{Affine3A, Vec2, Vec3A};
+use glam::{Affine3A, Vec2, Vec3A, Vec3Swizzles};
 use idmap::IdMap;
 use serde::Deserialize;
 use thiserror::Error;
@@ -252,23 +253,79 @@ impl TaskContainer {
     }
 }
 
-pub fn raycast(
+pub fn raycast_plane(
     source: &Affine3A,
     source_fwd: Vec3A,
     plane: &Affine3A,
     plane_norm: Vec3A,
-) -> Option<(Vec3A, f32)> {
+) -> Option<(f32, Vec2)> {
     let plane_normal = plane.transform_vector3a(plane_norm);
     let ray_dir = source.transform_vector3a(source_fwd);
 
     let d = plane.translation.dot(-plane_normal);
     let dist = -(d + source.translation.dot(plane_normal)) / ray_dir.dot(plane_normal);
 
-    if dist < 0.0 {
+    let hit_local = plane
-        // plane is behind the caster
+        .inverse()
+        .transform_point3a(source.translation + ray_dir * dist)
+        .xy();
+
+    Some((dist, hit_local))
+}
+
+pub fn raycast_cylinder(
+    source: &Affine3A,
+    source_fwd: Vec3A,
+    plane: &Affine3A,
+    curvature: f32,
+) -> Option<(f32, Vec2)> {
+    // this is solved locally; (0,0) is the center of the cylinder, and the cylinder is aligned with the Y axis
+    let size = plane.x_axis.length();
+    let to_local = Affine3A {
+        matrix3: plane.matrix3.mul_scalar(1.0 / size),
+        translation: plane.translation,
+    }
+    .inverse();
+
+    let r = size / (2.0 * PI * curvature);
+
+    let ray_dir = to_local.transform_vector3a(source.transform_vector3a(source_fwd));
+    let ray_origin = to_local.transform_point3a(source.translation) + Vec3A::NEG_Z * r;
+
+    let d = ray_dir.xz();
+    let s = ray_origin.xz();
+
+    let a = d.dot(d);
+    let b = d.dot(s);
+    let c = s.dot(s) - r * r;
+
+    let d = (b * b) - (a * c);
+    if d < f32::EPSILON {
         return None;
     }
 
-    let hit_pos = source.translation + ray_dir * dist;
+    let sqrt_d = d.sqrt();
-    Some((hit_pos, dist))
+
+    let t1 = (-b - sqrt_d) / a;
+    let t2 = (-b + sqrt_d) / a;
+
+    let t = t1.max(t2);
+
+    if t < f32::EPSILON {
+        return None;
+    }
+
+    let mut hit_local = ray_origin + ray_dir * t;
+    if hit_local.z > 0.0 {
+        // hitting the opposite half of the cylinder
+        return None;
+    }
+
+    let max_angle = 2.0 * (size / (2.0 * r));
+    let x_angle = (hit_local.x / r).asin();
+
+    hit_local.x = x_angle / max_angle;
+    hit_local.y = hit_local.y / size;
+
+    Some((t, hit_local.xy()))
 }

src/backend/input.rs

@@ -9,7 +9,7 @@ use smallvec::{smallvec, SmallVec};
 use crate::state::AppState;
 
 use super::{
-    common::{raycast, OverlayContainer},
+    common::{raycast_cylinder, raycast_plane, OverlayContainer},
     overlay::OverlayData,
 };
 
@@ -234,7 +234,8 @@ impl InteractionHandler for DummyInteractionHandler {
 #[derive(Debug, Clone, Copy, Default)]
 struct RayHit {
     overlay: usize,
-    hit_pos: Vec3A,
+    global_pos: Vec3A,
+    local_pos: Vec2,
     dist: f32,
 }
 
@@ -362,7 +363,28 @@ where
 
     if pointer.now.scroll.abs() > 0.1 {
         let scroll = pointer.now.scroll;
-        hovered.backend.on_scroll(app, &hit, scroll);
+        if app.input_state.pointers[1 - idx]
+            .interaction
+            .grabbed
+            .is_some_and(|x| x.grabbed_id == hit.overlay)
+        {
+            let is_portrait = hovered.view().is_some_and(|v| {
+                let extent = v.image().extent();
+                extent[0] >= extent[1]
+            });
+
+            if is_portrait {
+                let cur = hovered.state.curvature.unwrap_or(0.0);
+                let new = (cur - scroll * 0.01).min(0.35);
+                if new <= f32::EPSILON {
+                    hovered.state.curvature = None;
+                } else {
+                    hovered.state.curvature = Some(new);
+                }
+            }
+        } else {
+            hovered.backend.on_scroll(app, &hit, scroll);
+        }
         pointer = &mut app.input_state.pointers[idx];
     }
 
@@ -393,7 +415,11 @@ impl Pointer {
                 continue;
             }
 
-            if let Some(hit) = self.ray_test(overlay.state.id, &overlay.state.transform) {
+            if let Some(hit) = self.ray_test(
+                overlay.state.id,
+                &overlay.state.transform,
+                &overlay.state.curvature,
+            ) {
                 if hit.dist.is_infinite() || hit.dist.is_nan() {
                     continue;
                 }
@@ -408,12 +434,8 @@ impl Pointer {
 
             let uv = overlay
                 .state
-                .transform
+                .interaction_transform
-                .inverse()
+                .transform_point2(hit.local_pos);
-                .transform_point3a(hit.hit_pos)
-                .truncate();
-
-            let uv = overlay.state.interaction_transform.transform_point2(uv);
 
             if uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0 {
                 continue;
@@ -489,14 +511,26 @@ impl Pointer {
         }
     }
 
-    fn ray_test(&self, overlay: usize, plane: &Affine3A) -> Option<RayHit> {
+    fn ray_test(
-        let Some((hit_pos, dist)) = raycast(&self.pose, Vec3A::NEG_Z, plane, Vec3A::NEG_Z) else {
+        &self,
+        overlay: usize,
+        transform: &Affine3A,
+        curvature: &Option<f32>,
+    ) -> Option<RayHit> {
+        let (dist, local_pos) = match curvature {
+            Some(curvature) => raycast_cylinder(&self.pose, Vec3A::NEG_Z, transform, *curvature),
+            _ => raycast_plane(&self.pose, Vec3A::NEG_Z, transform, Vec3A::NEG_Z),
+        }?;
+
+        if dist < 0.0 {
+            // hit is behind us
             return None;
-        };
+        }
 
         Some(RayHit {
             overlay,
-            hit_pos,
+            global_pos: self.pose.transform_point3a(Vec3A::NEG_Z * dist),
+            local_pos,
             dist,
         })
     }

src/backend/openvr/overlay.rs

@@ -1,3 +1,5 @@
+use core::f32;
+
 use glam::Vec4;
 use ovr_overlay::{
     overlay::{OverlayHandle, OverlayManager},
@@ -21,7 +23,6 @@ pub(super) struct OpenVrOverlayData {
     pub(super) last_image: Option<u64>,
     pub(super) visible: bool,
     pub(super) color: Vec4,
-    pub(super) curvature: f32,
     pub(super) sort_order: u32,
     pub(crate) width: f32,
     pub(super) override_width: bool,
@@ -88,6 +89,8 @@ impl OverlayData<OpenVrOverlayData> {
     ) {
         if self.data.visible {
             if self.state.dirty {
+                self.upload_curvature(overlay);
+
                 self.upload_transform(universe, overlay);
                 self.upload_alpha(overlay);
                 self.state.dirty = false;
@@ -172,7 +175,7 @@ impl OverlayData<OpenVrOverlayData> {
             log::debug!("{}: No overlay handle", self.state.name);
             return;
         };
-        if let Err(e) = overlay.set_curvature(handle, self.data.curvature) {
+        if let Err(e) = overlay.set_curvature(handle, self.state.curvature.unwrap_or(0.0)) {
             log::error!(
                 "{}: Failed to set overlay curvature: {}",
                 self.state.name,

src/backend/openxr/helpers.rs

@@ -1,5 +1,5 @@
 use anyhow::{bail, ensure};
-use glam::{Affine3A, Quat, Vec3};
+use glam::{Affine3A, Quat, Vec3, Vec3A};
 use openxr as xr;
 use xr::OverlaySessionCreateFlagsEXTX;
 
@@ -139,12 +139,14 @@ fn quat_lerp(a: Quat, mut b: Quat, t: f32) -> Quat {
     .normalize()
 }
 
-pub(super) fn transform_to_posef(transform: &Affine3A) -> xr::Posef {
+pub(super) fn transform_to_norm_quat(transform: &Affine3A) -> Quat {
-    let translation = transform.translation;
     let norm_mat3 = transform
         .matrix3
         .mul_scalar(1.0 / transform.matrix3.x_axis.length());
-    let mut rotation = Quat::from_mat3a(&norm_mat3).normalize();
+    Quat::from_mat3a(&norm_mat3).normalize()
+}
+
+pub(super) fn translation_rotation_to_posef(translation: Vec3A, mut rotation: Quat) -> xr::Posef {
     if !rotation.is_finite() {
         rotation = Quat::IDENTITY;
     }
@@ -163,3 +165,9 @@ pub(super) fn transform_to_posef(transform: &Affine3A) -> xr::Posef {
         },
     }
 }
+
+pub(super) fn transform_to_posef(transform: &Affine3A) -> xr::Posef {
+    let translation = transform.translation;
+    let rotation = transform_to_norm_quat(transform);
+    translation_rotation_to_posef(translation, rotation)
+}

src/backend/openxr/lines.rs

@@ -19,7 +19,7 @@ use crate::{
 
 use super::{
     swapchain::{create_swapchain_render_data, SwapchainRenderData},
-    XrState,
+    CompositionLayer, XrState,
 };
 
 static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(1);
@@ -135,7 +135,7 @@ impl LinePool {
         &'a mut self,
         xr: &'a XrState,
         command_buffer: &mut WlxCommandBuffer,
-    ) -> anyhow::Result<Vec<xr::CompositionLayerQuad<xr::Vulkan>>> {
+    ) -> anyhow::Result<Vec<CompositionLayer>> {
         let mut quads = Vec::new();
 
         for line in self.lines.values_mut() {
@@ -155,7 +155,7 @@ impl LinePool {
                         height: inner.length,
                     });
 
-                quads.push(quad);
+                quads.push(CompositionLayer::Quad(quad));
             }
         }
 

src/backend/openxr/mod.rs

@@ -275,13 +275,18 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
                 continue;
             }
 
-            if let Some(quad) = o.present_xr(&xr_state, &mut command_buffer)? {
+            let maybe_layer = o.present_xr(&xr_state, &mut command_buffer)?;
-                layers.push((dist_sq, quad));
+            if let CompositionLayer::None = maybe_layer {
-            };
+                continue;
+            }
+            layers.push((dist_sq, maybe_layer));
         }
 
-        for quad in lines.present_xr(&xr_state, &mut command_buffer)? {
+        for maybe_layer in lines.present_xr(&xr_state, &mut command_buffer)? {
-            layers.push((0.0, quad));
+            if let CompositionLayer::None = maybe_layer {
+                continue;
+            }
+            layers.push((0.0, maybe_layer));
         }
 
         command_buffer.build_and_execute_now()?;
@@ -290,7 +295,11 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
 
         let frame_ref = layers
             .iter()
-            .map(|f| &f.1 as &xr::CompositionLayerBase<xr::Vulkan>)
+            .map(|f| match f.1 {
+                CompositionLayer::Quad(ref l) => l as &xr::CompositionLayerBase<xr::Vulkan>,
+                CompositionLayer::Cylinder(ref l) => l as &xr::CompositionLayerBase<xr::Vulkan>,
+                CompositionLayer::None => unreachable!(),
+            })
             .collect::<Vec<_>>();
 
         frame_stream.end(
@@ -351,3 +360,9 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
 
     Ok(())
 }
+
+pub(super) enum CompositionLayer<'a> {
+    None,
+    Quad(xr::CompositionLayerQuad<'a, xr::Vulkan>),
+    Cylinder(xr::CompositionLayerCylinderKHR<'a, xr::Vulkan>),
+}

src/backend/openxr/overlay.rs

@@ -1,8 +1,9 @@
+use glam::Vec3A;
 use openxr as xr;
-use std::sync::Arc;
+use std::{f32::consts::PI, sync::Arc};
 use xr::{CompositionLayerFlags, EyeVisibility};
 
-use super::{helpers, swapchain::SwapchainRenderData, XrState};
+use super::{helpers, swapchain::SwapchainRenderData, CompositionLayer, XrState};
 use crate::{
     backend::{openxr::swapchain::create_swapchain_render_data, overlay::OverlayData},
     graphics::WlxCommandBuffer,
@@ -23,7 +24,7 @@ impl OverlayData<OpenXrOverlayData> {
         &'a mut self,
         xr: &'a XrState,
         command_buffer: &mut WlxCommandBuffer,
-    ) -> anyhow::Result<Option<xr::CompositionLayerQuad<xr::Vulkan>>> {
+    ) -> anyhow::Result<CompositionLayer> {
         if let Some(new_view) = self.view() {
             self.data.last_view = Some(new_view);
         }
@@ -32,7 +33,7 @@ impl OverlayData<OpenXrOverlayData> {
             view.clone()
         } else {
             log::warn!("{}: Will not show - image not ready", self.state.name);
-            return Ok(None);
+            return Ok(CompositionLayer::None);
         };
         let extent = my_view.image().extent();
 
@@ -55,10 +56,8 @@ impl OverlayData<OpenXrOverlayData> {
         };
 
         let sub_image = data.acquire_present_release(command_buffer, my_view, self.state.alpha)?;
-        let posef = helpers::transform_to_posef(&self.state.transform);
 
         let aspect_ratio = extent[1] as f32 / extent[0] as f32;
-
         let (scale_x, scale_y) = if aspect_ratio < 1.0 {
             let major = self.state.transform.matrix3.col(0).length();
             (major, major * aspect_ratio)
@@ -67,17 +66,38 @@ impl OverlayData<OpenXrOverlayData> {
             (major / aspect_ratio, major)
         };
 
-        let quad = xr::CompositionLayerQuad::new()
+        if let Some(curvature) = self.state.curvature {
-            .pose(posef)
+            let radius = scale_x / (2.0 * PI * curvature);
-            .sub_image(sub_image)
+            let quat = helpers::transform_to_norm_quat(&self.state.transform);
-            .eye_visibility(EyeVisibility::BOTH)
+            let center_point = self.state.transform.translation + quat.mul_vec3a(Vec3A::Z * radius);
-            .layer_flags(CompositionLayerFlags::CORRECT_CHROMATIC_ABERRATION)
+
-            .space(&xr.stage)
+            let posef = helpers::translation_rotation_to_posef(center_point, quat);
-            .size(xr::Extent2Df {
+            let angle = 2.0 * (scale_x / (2.0 * radius));
-                width: scale_x,
+
-                height: scale_y,
+            let cylinder = xr::CompositionLayerCylinderKHR::new()
-            });
+                .pose(posef)
-        Ok(Some(quad))
+                .sub_image(sub_image)
+                .eye_visibility(EyeVisibility::BOTH)
+                .layer_flags(CompositionLayerFlags::CORRECT_CHROMATIC_ABERRATION)
+                .space(&xr.stage)
+                .radius(radius)
+                .central_angle(angle)
+                .aspect_ratio(aspect_ratio);
+            Ok(CompositionLayer::Cylinder(cylinder))
+        } else {
+            let posef = helpers::transform_to_posef(&self.state.transform);
+            let quad = xr::CompositionLayerQuad::new()
+                .pose(posef)
+                .sub_image(sub_image)
+                .eye_visibility(EyeVisibility::BOTH)
+                .layer_flags(CompositionLayerFlags::CORRECT_CHROMATIC_ABERRATION)
+                .space(&xr.stage)
+                .size(xr::Extent2Df {
+                    width: scale_x,
+                    height: scale_y,
+                });
+            Ok(CompositionLayer::Quad(quad))
+        }
     }
 
     pub(super) fn after_input(&mut self, app: &mut AppState) -> anyhow::Result<()> {

src/backend/overlay.rs

@@ -35,6 +35,7 @@ pub struct OverlayState {
     pub spawn_point: Vec3A,
     pub spawn_rotation: Quat,
     pub relative_to: RelativeTo,
+    pub curvature: Option<f32>,
     pub primary_pointer: Option<usize>,
     pub interaction_transform: Affine2,
     pub birthframe: usize,
@@ -53,6 +54,7 @@ impl Default for OverlayState {
             dirty: true,
             alpha: 1.0,
             relative_to: RelativeTo::None,
+            curvature: None,
             saved_point: None,
             saved_scale: None,
             spawn_scale: 1.0,