refactor overlay windowing

This commit is contained in:
galister
2025-10-05 18:49:59 +09:00
parent 40cc27f7b0
commit aa64310d96
46 changed files with 1329 additions and 1342 deletions

View File

@@ -0,0 +1,70 @@
use glam::{Affine2, Affine3A, Vec2};
use std::sync::Arc;
use vulkano::{format::Format, image::view::ImageView};
use crate::{
backend::input::{Haptics, PointerHit},
graphics::CommandBuffers,
state::AppState,
};
#[derive(Default, Clone, Copy)]
pub struct FrameMeta {
pub extent: [u32; 3],
pub transform: Affine3A,
pub format: Format,
}
pub enum ShouldRender {
/// The overlay is dirty and needs to be rendered.
Should,
/// The overlay is not dirty but is ready to be rendered.
Can,
/// The overlay is not ready to be rendered.
Unable,
}
pub trait OverlayBackend {
/// Called once, before the first frame is rendered
fn init(&mut self, app: &mut AppState) -> anyhow::Result<()>;
fn pause(&mut self, app: &mut AppState) -> anyhow::Result<()>;
fn resume(&mut self, app: &mut AppState) -> anyhow::Result<()>;
/// Called when the presentation layer is ready to present a new frame
fn should_render(&mut self, app: &mut AppState) -> anyhow::Result<ShouldRender>;
/// Called when the contents need to be rendered to the swapchain
fn render(
&mut self,
app: &mut AppState,
tgt: Arc<ImageView>,
buf: &mut CommandBuffers,
alpha: f32,
) -> anyhow::Result<bool>;
/// Called to retrieve the effective extent of the image
/// Used for creating swapchains.
///
/// Must be true if should_render was also true on the same frame.
fn frame_meta(&mut self) -> Option<FrameMeta>;
fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> Option<Haptics>;
fn on_left(&mut self, app: &mut AppState, pointer: usize);
fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool);
fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta_y: f32, delta_x: f32);
fn get_interaction_transform(&mut self) -> Option<Affine2>;
}
pub fn ui_transform(extent: [u32; 2]) -> Affine2 {
let aspect = extent[0] as f32 / extent[1] as f32;
let scale = if aspect < 1.0 {
Vec2 {
x: 1.0 / aspect,
y: -1.0,
}
} else {
Vec2 { x: 1.0, y: -aspect }
};
let center = Vec2 { x: 0.5, y: 0.5 };
Affine2::from_scale_angle_translation(scale, 0.0, center)
}

View File

@@ -0,0 +1,219 @@
use glam::{Affine3A, Vec3, Vec3A};
use slotmap::{HopSlotMap, Key};
use crate::{
overlays::{
anchor::create_anchor, keyboard::builder::create_keyboard, screen::create_screens,
watch::create_watch,
},
state::AppState,
windowing::{
set::OverlayWindowSet, snap_upright, window::OverlayWindowData, OverlayID, OverlaySelector,
},
};
pub struct OverlayWindowManager<T> {
overlays: HopSlotMap<OverlayID, OverlayWindowData<T>>,
sets: Vec<OverlayWindowSet>,
current_set: Option<usize>,
last_set: usize,
anchor_local: Affine3A,
watch_id: OverlayID,
}
impl<T> OverlayWindowManager<T>
where
T: Default,
{
pub fn new(app: &mut AppState, headless: bool) -> anyhow::Result<Self> {
let mut maybe_keymap = None;
let mut me = Self {
overlays: HopSlotMap::with_key(),
current_set: Some(0),
last_set: 0,
sets: vec![OverlayWindowSet::default()],
anchor_local: Affine3A::from_translation(Vec3::NEG_Z),
watch_id: OverlayID::null(), // set down below
};
if headless {
log::info!("Running in headless mode; keyboard will be en-US");
} else {
// create one work set for each screen.
match create_screens(app) {
Ok((data, keymap)) => {
let last_idx = data.screens.len() - 1;
for (idx, (meta, mut config)) in data.screens.into_iter().enumerate() {
config.show_on_spawn = true;
me.add(OverlayWindowData::from_config(config), app);
if idx < last_idx {
me.sets.push(OverlayWindowSet::default());
me.switch_to_set(app, Some(me.current_set.unwrap() + 1));
}
app.screens.push(meta);
}
maybe_keymap = keymap;
}
Err(e) => log::error!("Unable to initialize screens: {e:?}"),
}
}
let mut keyboard = OverlayWindowData::from_config(create_keyboard(app, maybe_keymap)?);
keyboard.config.show_on_spawn = true;
let keyboard_id = me.add(keyboard, app);
me.switch_to_set(app, None);
// copy keyboard to all sets
let kbd_state = me
.sets
.last()
.and_then(|s| s.overlays.get(keyboard_id))
.unwrap()
.clone();
for set in me.sets.iter_mut() {
set.overlays.insert(keyboard_id, kbd_state.clone());
}
let anchor = OverlayWindowData::from_config(create_anchor(app)?);
me.add(anchor, app);
let watch = OverlayWindowData::from_config(create_watch(app, me.sets.len())?);
me.watch_id = me.add(watch, app);
me.switch_to_set(app, None);
Ok(me)
}
pub fn mut_by_selector(
&mut self,
selector: &OverlaySelector,
) -> Option<&mut OverlayWindowData<T>> {
match selector {
OverlaySelector::Id(id) => self.mut_by_id(*id),
OverlaySelector::Name(name) => self.lookup(name).and_then(|id| self.mut_by_id(id)),
}
}
pub fn remove_by_selector(
&mut self,
selector: &OverlaySelector,
) -> Option<OverlayWindowData<T>> {
match selector {
OverlaySelector::Id(id) => self.overlays.remove(*id),
OverlaySelector::Name(name) => {
self.lookup(name).and_then(|id| self.overlays.remove(id))
}
}
}
pub fn get_by_id(&mut self, id: OverlayID) -> Option<&OverlayWindowData<T>> {
self.overlays.get(id)
}
pub fn mut_by_id(&mut self, id: OverlayID) -> Option<&mut OverlayWindowData<T>> {
self.overlays.get_mut(id)
}
pub fn iter(&self) -> impl Iterator<Item = (OverlayID, &'_ OverlayWindowData<T>)> {
self.overlays.iter()
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = (OverlayID, &'_ mut OverlayWindowData<T>)> {
self.overlays.iter_mut()
}
pub fn values(&self) -> impl Iterator<Item = &'_ OverlayWindowData<T>> {
self.overlays.values()
}
pub fn values_mut(&mut self) -> impl Iterator<Item = &'_ mut OverlayWindowData<T>> {
self.overlays.values_mut()
}
pub fn lookup(&self, name: &str) -> Option<OverlayID> {
self.overlays
.iter()
.find(|(_, v)| v.config.name.as_ref() == name)
.map(|(k, _)| k)
}
pub fn add(&mut self, mut overlay: OverlayWindowData<T>, app: &mut AppState) -> OverlayID {
if overlay.config.show_on_spawn {
overlay.config.activate(app);
}
let id = self.overlays.insert(overlay);
id
}
pub fn switch_or_toggle_set(&mut self, app: &mut AppState, set: usize) {
let new_set = if self.current_set.iter().any(|cur| *cur == set) {
None
} else {
Some(set)
};
self.switch_to_set(app, new_set);
}
pub fn switch_to_set(&mut self, app: &mut AppState, new_set: Option<usize>) {
if new_set == self.current_set {
return;
}
if let Some(current_set) = self.current_set.as_ref() {
let ws = &mut self.sets[*current_set];
ws.overlays.clear();
for (id, data) in self.overlays.iter_mut().filter(|(_, d)| !d.config.global) {
if let Some(mut state) = data.config.active_state.take() {
if let Some(transform) = data.config.saved_transform.take() {
state.transform = transform;
} else {
state.transform = Affine3A::ZERO;
}
log::warn!("{}: active_state → ws{}", data.config.name, current_set);
ws.overlays.insert(id, state);
}
}
}
if let Some(new_set) = new_set {
if new_set >= self.sets.len() {
log::warn!("switch_to_set: new_set is out of range ({new_set:?})");
return;
}
let ws = &mut self.sets[new_set];
for (id, data) in self.overlays.iter_mut().filter(|(_, d)| !d.config.global) {
if let Some(mut state) = ws.overlays.remove(id) {
if state.transform.x_axis.length_squared() > f32::EPSILON {
data.config.saved_transform = Some(state.transform);
}
state.transform = Affine3A::IDENTITY;
log::warn!("{}: ws{} → active_state", data.config.name, new_set);
data.config.active_state = Some(state);
data.config.reset(app, false);
}
}
}
self.current_set = new_set;
}
pub fn show_hide(&mut self, app: &mut AppState) {
if self.current_set.is_none() {
let hmd = snap_upright(app.input_state.hmd, Vec3A::Y);
app.anchor = hmd * self.anchor_local;
self.switch_to_set(app, Some(self.last_set));
} else {
self.switch_to_set(app, None);
}
// toggle watch back on if it was hidden
self.mut_by_id(self.watch_id).unwrap().config.activate(app);
}
}

View File

@@ -0,0 +1,45 @@
use glam::{Affine3A, Vec3A};
use slotmap::new_key_type;
use std::sync::Arc;
pub mod backend;
pub mod manager;
mod set;
pub mod window;
new_key_type! {
pub struct OverlayID;
}
#[derive(Clone, Debug)]
pub enum OverlaySelector {
Id(OverlayID),
Name(Arc<str>),
}
pub const Z_ORDER_TOAST: u32 = 70;
pub const Z_ORDER_LINES: u32 = 69;
pub const Z_ORDER_WATCH: u32 = 68;
pub const Z_ORDER_ANCHOR: u32 = 67;
pub const Z_ORDER_DEFAULT: u32 = 0;
pub const Z_ORDER_DASHBOARD: u32 = Z_ORDER_DEFAULT;
pub fn snap_upright(transform: Affine3A, up_dir: Vec3A) -> Affine3A {
if transform.x_axis.dot(up_dir).abs() < 0.2 {
let scale = transform.x_axis.length();
let col_z = transform.z_axis.normalize();
let col_y = up_dir;
let col_x = col_y.cross(col_z);
let col_y = col_z.cross(col_x).normalize();
let col_x = col_x.normalize();
Affine3A::from_cols(
col_x * scale,
col_y * scale,
col_z * scale,
transform.translation,
)
} else {
transform
}
}

View File

@@ -0,0 +1,8 @@
use slotmap::SecondaryMap;
use crate::windowing::{window::OverlayWindowState, OverlayID};
#[derive(Default)]
pub struct OverlayWindowSet {
pub(super) overlays: SecondaryMap<OverlayID, OverlayWindowState>,
}

View File

@@ -0,0 +1,311 @@
use glam::{Affine3A, Mat3A, Quat, Vec3, Vec3A};
use std::{f32::consts::PI, sync::Arc};
use vulkano::image::view::ImageView;
use crate::{
graphics::CommandBuffers,
state::AppState,
subsystem::input::KeyboardFocus,
windowing::{
backend::{FrameMeta, OverlayBackend, ShouldRender},
snap_upright,
},
};
#[derive(Clone, Copy, Debug, Default)]
pub enum Positioning {
/// Stays in place, recenters relative to HMD
#[default]
Floating,
/// Stays in place, recenters relative to anchor
Anchored,
/// Stays in place, no recentering
Static,
/// Following HMD
FollowHead { lerp: f32 },
/// Normally follows HMD, but paused due to interaction
FollowHeadPaused { lerp: f32 },
/// Following hand
FollowHand { hand: usize, lerp: f32 },
/// Normally follows hand, but paused due to interaction
FollowHandPaused { hand: usize, lerp: f32 },
/// Follow another overlay
FollowOverlay { id: usize },
}
impl Positioning {
pub fn moves_with_space(&self) -> bool {
matches!(
self,
Positioning::Floating | Positioning::Anchored | Positioning::Static
)
}
}
pub struct OverlayWindowData<T> {
pub config: OverlayWindowConfig,
pub data: T,
pub birthframe: usize,
pub primary_pointer: Option<usize>,
}
impl<T> OverlayWindowData<T>
where
T: Default,
{
pub fn from_config(config: OverlayWindowConfig) -> Self {
Self {
data: T::default(),
config,
primary_pointer: None,
birthframe: 0,
}
}
}
impl<T> OverlayWindowData<T>
where
T: Default,
{
pub fn init(&mut self, app: &mut AppState) -> anyhow::Result<()> {
//TODO: load state?
self.config.backend.init(app)
}
pub fn should_render(&mut self, app: &mut AppState) -> anyhow::Result<ShouldRender> {
self.config.backend.should_render(app)
}
pub fn render(
&mut self,
app: &mut AppState,
tgt: Arc<ImageView>,
buf: &mut CommandBuffers,
alpha: f32,
) -> anyhow::Result<bool> {
self.config.backend.render(app, tgt, buf, alpha)
}
pub fn frame_meta(&mut self) -> Option<FrameMeta> {
self.config.backend.frame_meta()
}
}
pub struct OverlayWindowConfig {
pub name: Arc<str>,
pub backend: Box<dyn OverlayBackend>,
/// The default state to show when the overlay is newly spawned.
pub default_state: OverlayWindowState,
/// The current state to show. None if the overlay is hidden.
pub active_state: Option<OverlayWindowState>,
/// Order to draw overlays in. Overlays with higher numbers will be drawn over ones with lower numbers.
pub z_order: u32,
/// If set, hovering this overlay will cause the HID provider to switch focus.
pub keyboard_focus: Option<KeyboardFocus>,
/// Should the overlay be displayed on the next frame?
pub show_on_spawn: bool,
/// Does not belong to any set; switching sets does not affect this overlay.
pub global: bool,
/// True if transform, curvature, alpha has changed. Only used by OpenVR.
pub dirty: bool,
pub saved_transform: Option<Affine3A>,
}
impl OverlayWindowConfig {
pub fn from_backend(backend: Box<dyn OverlayBackend>) -> Self {
Self {
name: "".into(),
backend,
default_state: OverlayWindowState {
transform: Affine3A::from_translation(Vec3::NEG_Z),
..OverlayWindowState::default()
},
active_state: None,
saved_transform: None,
z_order: 0,
keyboard_focus: None,
show_on_spawn: false,
global: false,
dirty: true,
}
}
pub fn activate(&mut self, app: &mut AppState) {
log::warn!("activate {}", self.name.as_ref());
self.dirty = true;
self.active_state = Some(self.default_state.clone());
self.reset(app, true);
}
pub fn deactivate(&mut self) {
log::warn!("deactivate {}", self.name.as_ref());
self.active_state = None;
}
pub fn toggle(&mut self, app: &mut AppState) {
if self.active_state.take().is_none() {
self.activate(app);
} else {
log::warn!("deactivate {}", self.name.as_ref());
}
}
pub fn auto_movement(&mut self, app: &mut AppState) {
let Some(state) = self.active_state.as_mut() else {
return;
};
let cur_transform = self.saved_transform.unwrap_or(self.default_state.transform);
let (target_transform, lerp) = match state.positioning {
Positioning::FollowHead { lerp } => (app.input_state.hmd * cur_transform, lerp),
Positioning::FollowHand { hand, lerp } => {
(app.input_state.pointers[hand].pose * cur_transform, lerp)
}
_ => return,
};
state.transform = match lerp {
1.0 => target_transform,
lerp => {
let scale = target_transform.matrix3.x_axis.length();
let rot_from = Quat::from_mat3a(&state.transform.matrix3.div_scalar(scale));
let rot_to = Quat::from_mat3a(&target_transform.matrix3.div_scalar(scale));
let rotation = rot_from.slerp(rot_to, lerp);
let translation = state
.transform
.translation
.slerp(target_transform.translation, lerp);
Affine3A::from_scale_rotation_translation(
Vec3::ONE * scale,
rotation,
translation.into(),
)
}
};
self.dirty = true;
}
/// Returns true if changes were saved.
pub fn save_transform(&mut self, app: &mut AppState) -> bool {
let Some(state) = self.active_state.as_mut() else {
return false;
};
let parent_transform = match state.positioning {
Positioning::Floating => snap_upright(app.input_state.hmd, Vec3A::Y),
Positioning::FollowHead { .. } | Positioning::FollowHeadPaused { .. } => {
app.input_state.hmd
}
Positioning::FollowHand { hand, .. } | Positioning::FollowHandPaused { hand, .. } => {
app.input_state.pointers[hand].pose
}
Positioning::Anchored => snap_upright(app.anchor, Vec3A::Y),
Positioning::FollowOverlay { .. } | Positioning::Static => return false,
};
self.saved_transform = Some(parent_transform.inverse() * state.transform);
true
}
pub fn reset(&mut self, app: &mut AppState, hard_reset: bool) {
let Some(state) = self.active_state.as_mut() else {
return;
};
let cur_transform = self.saved_transform.unwrap_or(self.default_state.transform);
let parent_transform = match state.positioning {
Positioning::Floating
| Positioning::FollowHead { .. }
| Positioning::FollowHeadPaused { .. } => app.input_state.hmd,
Positioning::FollowHand { hand, .. } | Positioning::FollowHandPaused { hand, .. } => {
app.input_state.pointers[hand].pose
}
Positioning::Anchored => app.anchor,
Positioning::FollowOverlay { .. } | Positioning::Static => return,
};
if hard_reset {
self.saved_transform = None;
}
state.transform = parent_transform * cur_transform;
if state.grabbable && hard_reset {
self.realign(&app.input_state.hmd);
}
self.dirty = true;
}
pub fn realign(&mut self, hmd: &Affine3A) {
let Some(state) = self.active_state.as_mut() else {
return;
};
let to_hmd = hmd.translation - state.transform.translation;
let up_dir: Vec3A;
if hmd.x_axis.dot(Vec3A::Y).abs() > 0.2 {
// Snap upright
up_dir = hmd.y_axis;
} else {
let dot = to_hmd.normalize().dot(hmd.z_axis);
let z_dist = to_hmd.length();
let y_dist = (state.transform.translation.y - hmd.translation.y).abs();
let x_angle = (y_dist / z_dist).asin();
if dot < -f32::EPSILON {
// facing down
let up_point = hmd.translation + z_dist / x_angle.cos() * Vec3A::Y;
up_dir = (up_point - state.transform.translation).normalize();
} else if dot > f32::EPSILON {
// facing up
let dn_point = hmd.translation + z_dist / x_angle.cos() * Vec3A::NEG_Y;
up_dir = (state.transform.translation - dn_point).normalize();
} else {
// perfectly upright
up_dir = Vec3A::Y;
}
}
let scale = state.transform.x_axis.length();
let col_z = (state.transform.translation - hmd.translation).normalize();
let col_y = up_dir;
let col_x = col_y.cross(col_z);
let col_y = col_z.cross(col_x).normalize();
let col_x = col_x.normalize();
let rot = Mat3A::from_quat(Quat::from_axis_angle(Vec3::Y, PI));
state.transform.matrix3 = Mat3A::from_cols(col_x, col_y, col_z).mul_scalar(scale) * rot;
self.dirty = true;
}
}
// Contains the window state for a given set
#[derive(Clone)]
pub struct OverlayWindowState {
pub transform: Affine3A,
pub alpha: f32,
pub grabbable: bool,
pub interactable: bool,
pub positioning: Positioning,
pub curvature: Option<f32>,
}
impl Default for OverlayWindowState {
fn default() -> Self {
Self {
grabbable: false,
interactable: false,
alpha: 1.0,
positioning: Positioning::Floating,
curvature: None,
transform: Affine3A::IDENTITY,
}
}
}