Jayden/wayvr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

screens & basic interactions

Browse Source
This commit is contained in:
galister
2023-12-07 02:07:13 +01:00
parent cb039de409
commit a3b60b9607
19 changed files with 518 additions and 1599 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
src/backend/input.rs
View File

@@ -1,7 +1,6 @@
use std::{collections::VecDeque, time::Instant};
use glam::{Affine3A, Vec2, Vec3A};
use log::warn;
use ovr_overlay::TrackedDeviceIndex;
use tinyvec::array_vec;
@@ -42,6 +41,39 @@ impl<TState, THand> InputState<TState, THand> {
pub fn post_update(&mut self) {
for hand in &mut self.pointers {
#[cfg(debug_assertions)]
{
if hand.now.click != hand.before.click {
log::debug!("Hand {}: click {}", hand.idx, hand.now.click);
}
if hand.now.grab != hand.before.grab {
log::debug!("Hand {}: grab {}", hand.idx, hand.now.grab);
}
if hand.now.alt_click != hand.before.alt_click {
log::debug!("Hand {}: alt_click {}", hand.idx, hand.now.alt_click);
}
if hand.now.show_hide != hand.before.show_hide {
log::debug!("Hand {}: show_hide {}", hand.idx, hand.now.show_hide);
}
if hand.now.space_drag != hand.before.space_drag {
log::debug!("Hand {}: space_drag {}", hand.idx, hand.now.space_drag);
}
if hand.now.click_modifier_right != hand.before.click_modifier_right {
log::debug!(
"Hand {}: click_modifier_right {}",
hand.idx,
hand.now.click_modifier_right
);
}
if hand.now.click_modifier_middle != hand.before.click_modifier_middle {
log::debug!(
"Hand {}: click_modifier_middle {}",
hand.idx,
hand.now.click_modifier_middle
);
}
}
if hand.now.click_modifier_right {
hand.interaction.mode = PointerMode::Right;
continue;
@@ -176,7 +208,7 @@ pub enum PointerMode {
}
impl<THand> Pointer<THand> {
pub fn interact<O>(&mut self, overlays: &mut OverlayContainer<O>, app: &mut AppState)
pub fn interact<O>(&mut self, overlays: &mut OverlayContainer<O>, app: &mut AppState) -> f32
where
O: Default,
{
@@ -184,10 +216,10 @@ impl<THand> Pointer<THand> {
if let Some(grabbed) = overlays.mut_by_id(grab_data.grabbed_id) {
self.handle_grabbed(grabbed, grab_data.offset);
} else {
warn!("Grabbed overlay {} does not exist", grab_data.grabbed_id);
log::warn!("Grabbed overlay {} does not exist", grab_data.grabbed_id);
self.interaction.grabbed = None;
}
return;
return grab_data.offset.length(); // grab interaction
}
let Some(mut hit) = self.get_nearest_hit(overlays) else {
@@ -208,7 +240,7 @@ impl<THand> Pointer<THand> {
clicked.backend.on_pointer(app, &hit, false);
}
}
return;
return 0.0; // no hit
};
if let Some(hovered_id) = self.interaction.hovered_id {
@@ -222,14 +254,14 @@ impl<THand> Pointer<THand> {
}
}
let Some(hovered) = overlays.mut_by_id(hit.overlay) else {
warn!("Hit overlay {} does not exist", hit.overlay);
return;
log::warn!("Hit overlay {} does not exist", hit.overlay);
return 0.0; // no hit
};
self.interaction.hovered_id = Some(hit.overlay);
if let Some(primary_pointer) = hovered.primary_pointer {
if hit.pointer < primary_pointer {
if hit.pointer <= primary_pointer {
hovered.primary_pointer = Some(hit.pointer);
hit.primary = true;
}
@@ -237,6 +269,14 @@ impl<THand> Pointer<THand> {
hovered.primary_pointer = Some(hit.pointer);
hit.primary = true;
}
log::debug!("Hit: {} {:?}", hovered.state.name, hit);
if self.now.grab && !self.before.grab {
self.start_grab(hovered);
return hit.dist;
}
hovered.backend.on_hover(app, &hit);
if self.now.scroll.abs() > 0.1 {
@@ -255,6 +295,7 @@ impl<THand> Pointer<THand> {
hovered.backend.on_pointer(app, &hit, false);
}
}
hit.dist
}
fn get_nearest_hit<O>(&mut self, overlays: &mut OverlayContainer<O>) -> Option<PointerHit>
@@ -276,15 +317,17 @@ impl<THand> Pointer<THand> {
hits.sort_by(|a, b| a.dist.partial_cmp(&b.dist).unwrap());
for hit in hits.iter() {
let uv = overlays
.get_by_id(hit.overlay)
.unwrap() // this is safe
let overlay = overlays.get_by_id(hit.overlay).unwrap(); // this is safe
let uv = overlay
.state
.transform
.inverse()
.transform_point3a(hit.hit_pos)
.truncate();
let uv = overlay.state.interaction_transform.transform_point2(uv);
if uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0 {
continue;
}
@@ -315,7 +358,9 @@ impl<THand> Pointer<THand> {
offset,
grabbed_id: overlay.state.id,
});
log::info!("Hand {}: grabbed {}", self.idx, overlay.state.name);
}
fn handle_grabbed<O>(&mut self, overlay: &mut OverlayData<O>, offset: Vec3A)
where
O: Default,
@@ -324,7 +369,7 @@ impl<THand> Pointer<THand> {
overlay.state.transform.translation = self.pose.transform_point3a(offset);
if self.now.click && !self.before.click {
warn!("todo: click-while-grabbed");
log::warn!("todo: click-while-grabbed");
}
match self.interaction.mode {
@@ -339,11 +384,14 @@ impl<THand> Pointer<THand> {
.mul_scalar(1.0 + 0.01 * self.now.scroll);
}
}
overlay.state.dirty = true;
} else {
overlay.state.spawn_point = overlay.state.transform.translation;
self.interaction.grabbed = None;
log::info!("Hand {}: dropped {}", self.idx, overlay.state.name);
}
}
fn ray_test(&self, overlay: usize, plane: &Affine3A) -> Option<RayHit> {
let plane_normal = plane.transform_vector3a(Vec3A::NEG_Z);
let ray_dir = self.pose.transform_vector3a(Vec3A::NEG_Z);

2
src/backend/openvr/input.rs
View File

@@ -59,6 +59,7 @@ pub(super) struct OpenVrInputState {
}
pub(super) struct OpenVrHandState {
pub(super) line_id: usize,
has_pose: bool,
input_hnd: InputValueHandle,
pose_hnd: ActionHandle,
@@ -103,6 +104,7 @@ impl InputState<OpenVrInputState, OpenVrHandState> {
pose: Affine3A::IDENTITY,
interaction: InteractionState::default(),
data: OpenVrHandState {
line_id: 0,
has_pose: false,
input_hnd: input_hnd[i],
pose_hnd: pose_hnd[i],

130
src/backend/openvr/lines.rs Normal file
View File

@@ -0,0 +1,130 @@
use std::f32::consts::PI;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use glam::{Affine3A, Vec3, Vec3A, Vec4};
use idmap::IdMap;
use ovr_overlay::overlay::OverlayManager;
use vulkano::command_buffer::CommandBufferUsage;
use vulkano::format::Format;
use vulkano::image::view::ImageView;
use vulkano::image::{ImageAccess, ImageLayout, ImageViewAbstract, ImmutableImage};
use crate::backend::overlay::{OverlayData, OverlayRenderer, OverlayState, SplitOverlayBackend};
use crate::graphics::WlxGraphics;
use crate::state::AppState;
use super::overlay::OpenVrOverlayData;
static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(1);
pub(super) struct LinePool {
lines: IdMap<usize, OverlayData<OpenVrOverlayData>>,
view: Arc<ImageView<ImmutableImage>>,
}
impl LinePool {
pub fn new(graphics: Arc<WlxGraphics>) -> Self {
let mut command_buffer = graphics.create_command_buffer(CommandBufferUsage::OneTimeSubmit);
let buf = vec![255; 16];
let texture = command_buffer.texture2d(2, 2, Format::R8G8B8A8_UNORM, buf);
command_buffer.build_and_execute_now();
graphics
.transition_layout(
texture.inner().image.clone(),
ImageLayout::ShaderReadOnlyOptimal,
ImageLayout::TransferSrcOptimal,
)
.wait(None)
.unwrap();
let view = ImageView::new_default(texture).unwrap();
LinePool {
lines: IdMap::new(),
view,
}
}
pub fn allocate(&mut self, overlay: &mut OverlayManager, app: &mut AppState) -> usize {
let id = AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed);
let mut data = OverlayData::<OpenVrOverlayData> {
state: OverlayState {
name: Arc::from(format!("wlx-line{}", id)),
show_hide: true,
width: 0.002,
size: (0, 0),
..Default::default()
},
backend: Box::new(SplitOverlayBackend {
renderer: Box::new(StaticRenderer {
view: self.view.clone(),
}),
..Default::default()
}),
..Default::default()
};
data.data.sort_order = 69;
data.initialize(overlay, app);
data.upload_texture(overlay, &app.graphics);
self.lines.insert(id, data);
id
}
pub fn draw_from(&mut self, id: usize, mut from: Affine3A, len: f32, color: Vec4) {
let rotation = Affine3A::from_axis_angle(Vec3::X, -PI * 0.5);
from.translation = from.translation + from.transform_vector3a(Vec3A::NEG_Z) * (len * 0.5);
let transform = from * rotation * Affine3A::from_scale(Vec3::new(1., len / 0.002, 1.));
self.draw_transform(id, transform, color);
}
fn draw_transform(&mut self, id: usize, transform: Affine3A, color: Vec4) {
if let Some(data) = self.lines.get_mut(id) {
data.state.want_visible = true;
data.state.transform = transform;
data.data.color = color;
} else {
log::warn!("Line {} does not exist", id);
}
}
pub fn hide(&mut self, id: usize) {
if let Some(data) = self.lines.get_mut(id) {
data.state.want_visible = false;
} else {
log::warn!("Line {} does not exist", id);
}
}
pub fn update(&mut self, overlay: &mut OverlayManager, app: &mut AppState) {
for data in self.lines.values_mut() {
data.after_input(overlay, app);
if data.state.want_visible {
data.upload_transform(overlay);
data.upload_color(overlay);
}
}
}
}
struct StaticRenderer {
view: Arc<ImageView<ImmutableImage>>,
}
impl OverlayRenderer for StaticRenderer {
fn init(&mut self, _app: &mut AppState) {}
fn pause(&mut self, _app: &mut AppState) {}
fn resume(&mut self, _app: &mut AppState) {}
fn render(&mut self, _app: &mut AppState) {}
fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>> {
Some(self.view.clone())
}
}

137
src/backend/openvr/mod.rs
View File

@@ -1,10 +1,9 @@
use glam::Vec4;
use std::{
collections::VecDeque,
path::Path,
time::{Duration, Instant},
};
use log::{error, info};
use ovr_overlay::{
sys::{ETrackedDeviceProperty, EVRApplicationType, EVREventType},
TrackedDeviceIndex,
@@ -15,7 +14,7 @@ use vulkano::{
Handle, VulkanObject,
};
use crate::state::AppState;
use crate::{backend::openvr::lines::LinePool, state::AppState};
use self::{input::action_manifest_path, overlay::OpenVrOverlayData};
@@ -25,12 +24,13 @@ use super::{
};
pub mod input;
pub mod lines;
pub mod overlay;
pub fn openvr_run() {
let app_type = EVRApplicationType::VRApplication_Overlay;
let Ok(context) = ovr_overlay::Context::init(app_type) else {
error!("Failed to initialize OpenVR");
log::error!("Failed to initialize OpenVR");
return;
};
@@ -56,12 +56,12 @@ pub fn openvr_run() {
let mut overlays = OverlayContainer::<OpenVrOverlayData>::new(&mut state);
if let Err(e) = input_mngr.set_action_manifest(action_manifest_path()) {
error!("Failed to set action manifest: {}", e.description());
log::error!("Failed to set action manifest: {}", e.description());
return;
};
let Ok(mut input) = InputState::new(&mut input_mngr) else {
error!("Failed to initialize input");
log::error!("Failed to initialize input");
return;
};
@@ -69,21 +69,25 @@ pub fn openvr_run() {
TrackedDeviceIndex::HMD,
ETrackedDeviceProperty::Prop_DisplayFrequency_Float,
) else {
error!("Failed to get display refresh rate");
log::error!("Failed to get display refresh rate");
return;
};
info!("HMD running @ {} Hz", refresh_rate);
log::info!("HMD running @ {} Hz", refresh_rate);
let frame_time = (1000.0 / refresh_rate).floor() * 0.001;
let mut next_device_update = Instant::now();
let mut due_tasks = VecDeque::with_capacity(4);
let mut lines = LinePool::new(state.graphics.clone());
input.pointers[0].data.line_id = lines.allocate(&mut overlay_mngr, &mut state);
input.pointers[1].data.line_id = lines.allocate(&mut overlay_mngr, &mut state);
loop {
while let Some(event) = system_mngr.poll_next_event() {
match event.event_type {
EVREventType::VREvent_Quit => {
info!("Received quit event, shutting down.");
log::info!("Received quit event, shutting down.");
return;
}
EVREventType::VREvent_TrackedDeviceActivated
@@ -93,64 +97,73 @@ pub fn openvr_run() {
}
_ => {}
}
}
if next_device_update <= Instant::now() {
input.update_devices(&mut system_mngr);
next_device_update = Instant::now() + Duration::from_secs(30);
}
if next_device_update <= Instant::now() {
input.update_devices(&mut system_mngr);
next_device_update = Instant::now() + Duration::from_secs(30);
}
state.tasks.retrieve_due(&mut due_tasks);
while let Some(task) = due_tasks.pop_front() {
match task {
TaskType::Global(f) => f(&mut state),
TaskType::Overlay(sel, f) => {
if let Some(o) = overlays.mut_by_selector(&sel) {
f(&mut state, &mut o.state);
}
state.tasks.retrieve_due(&mut due_tasks);
while let Some(task) = due_tasks.pop_front() {
match task {
TaskType::Global(f) => f(&mut state),
TaskType::Overlay(sel, f) => {
if let Some(o) = overlays.mut_by_selector(&sel) {
f(&mut state, &mut o.state);
}
}
}
input.pre_update();
input.update(&mut input_mngr, &mut system_mngr);
input.post_update();
input
.pointers
.iter_mut()
.for_each(|p| p.interact(&mut overlays, &mut state));
overlays
.iter_mut()
.for_each(|o| o.after_input(&mut overlay_mngr, &mut state));
log::debug!("Rendering frame");
overlays
.iter_mut()
.filter(|o| o.state.want_visible)
.for_each(|o| o.render(&mut state));
log::debug!("Rendering overlays");
overlays
.iter_mut()
.for_each(|o| o.after_render(&mut overlay_mngr, &state.graphics));
// chaperone
// close font handles?
// playspace moved end frame
let mut seconds_since_vsync = 0f32;
std::thread::sleep(Duration::from_secs_f32(
if system_mngr.get_time_since_last_vsync(&mut seconds_since_vsync, &mut 0u64) {
(frame_time - seconds_since_vsync).max(0.0)
} else {
0.011
},
));
}
input.pre_update();
input.update(&mut input_mngr, &mut system_mngr);
input.post_update();
input.pointers.iter_mut().for_each(|p| {
let dist = p.interact(&mut overlays, &mut state);
if dist > 0.001 {
lines.draw_from(p.data.line_id, p.pose, dist, Vec4::ONE);
} else {
lines.draw_from(p.data.line_id, p.pose, 20.0, Vec4::ONE);
// lines.hide(p.data.line_id);
}
});
lines.update(&mut overlay_mngr, &mut state);
overlays
.iter_mut()
.for_each(|o| o.after_input(&mut overlay_mngr, &mut state));
log::debug!("Rendering frame");
overlays
.iter_mut()
.filter(|o| o.state.want_visible)
.for_each(|o| o.render(&mut state));
log::debug!("Rendering overlays");
overlays
.iter_mut()
.for_each(|o| o.after_render(&mut overlay_mngr, &state.graphics));
// chaperone
// close font handles?
// playspace moved end frame
state.input.on_new_frame();
let mut seconds_since_vsync = 0f32;
std::thread::sleep(Duration::from_secs_f32(
if system_mngr.get_time_since_last_vsync(&mut seconds_since_vsync, &mut 0u64) {
frame_time - (seconds_since_vsync % frame_time)
} else {
frame_time
},
));
}
}

41
src/backend/openvr/overlay.rs
View File

@@ -6,7 +6,11 @@ use ovr_overlay::{
};
use vulkano::{image::ImageAccess, Handle, VulkanObject};
use crate::{backend::overlay::OverlayData, graphics::WlxGraphics, state::AppState};
use crate::{
backend::overlay::{OverlayData, RelativeTo},
graphics::WlxGraphics,
state::AppState,
};
#[derive(Default)]
pub(super) struct OpenVrOverlayData {
@@ -16,10 +20,11 @@ pub(super) struct OpenVrOverlayData {
pub(super) color: Vec4,
pub(super) curvature: f32,
pub(super) sort_order: u32,
pub(super) relative_to: RelativeTo,
}
impl OverlayData<OpenVrOverlayData> {
pub fn initialize(
pub(super) fn initialize(
&mut self,
overlay: &mut OverlayManager,
app: &mut AppState,
@@ -33,6 +38,11 @@ impl OverlayData<OpenVrOverlayData> {
};
log::debug!("{}: initialize", self.state.name);
//watch
if self.state.id == 0 {
self.data.sort_order = 68;
}
self.data.handle = Some(handle);
self.data.color = Vec4::ONE;
@@ -41,11 +51,12 @@ impl OverlayData<OpenVrOverlayData> {
self.upload_width(overlay);
self.upload_color(overlay);
self.upload_curvature(overlay);
self.upload_sort_order(overlay);
handle
}
pub fn after_input(&mut self, overlay: &mut OverlayManager, app: &mut AppState) {
pub(super) fn after_input(&mut self, overlay: &mut OverlayManager, app: &mut AppState) {
if self.state.want_visible && !self.data.visible {
self.show(overlay, app);
} else if !self.state.want_visible && self.data.visible {
@@ -53,9 +64,12 @@ impl OverlayData<OpenVrOverlayData> {
}
}
pub fn after_render(&mut self, overlay: &mut OverlayManager, graphics: &WlxGraphics) {
pub(super) fn after_render(&mut self, overlay: &mut OverlayManager, graphics: &WlxGraphics) {
if self.data.visible {
self.upload_transform(overlay);
if self.state.dirty {
self.upload_transform(overlay);
self.state.dirty = false;
}
self.upload_texture(overlay, graphics);
}
}
@@ -83,7 +97,7 @@ impl OverlayData<OpenVrOverlayData> {
self.data.visible = false;
}
fn upload_color(&self, overlay: &mut OverlayManager) {
pub(super) fn upload_color(&self, overlay: &mut OverlayManager) {
let Some(handle) = self.data.handle else {
log::debug!("{}: No overlay handle", self.state.name);
return;
@@ -104,7 +118,7 @@ impl OverlayData<OpenVrOverlayData> {
}
}
fn upload_width(&self, overlay: &mut OverlayManager) {
pub(super) fn upload_width(&self, overlay: &mut OverlayManager) {
let Some(handle) = self.data.handle else {
log::debug!("{}: No overlay handle", self.state.name);
return;
@@ -134,7 +148,7 @@ impl OverlayData<OpenVrOverlayData> {
}
}
fn upload_transform(&self, overlay: &mut OverlayManager) {
pub(super) fn upload_transform(&self, overlay: &mut OverlayManager) {
let Some(handle) = self.data.handle else {
log::debug!("{}: No overlay handle", self.state.name);
return;
@@ -170,7 +184,7 @@ impl OverlayData<OpenVrOverlayData> {
}
}
fn upload_texture(&mut self, overlay: &mut OverlayManager, graphics: &WlxGraphics) {
pub(super) fn upload_texture(&mut self, overlay: &mut OverlayManager, graphics: &WlxGraphics) {
let Some(handle) = self.data.handle else {
log::debug!("{}: No overlay handle", self.state.name);
return;
@@ -210,8 +224,13 @@ impl OverlayData<OpenVrOverlayData> {
m_nQueueFamilyIndex: graphics.queue.queue_family_index(),
};
log::info!("Usages: {:?}", image.usage());
log::info!("nImage: {}, nFormat: {:?}, nWidth: {}, nHeight: {}, nSampleCount: {}, nQueueFamilyIndex: {}", texture.m_nImage, format, texture.m_nWidth, texture.m_nHeight, texture.m_nSampleCount, texture.m_nQueueFamilyIndex);
log::debug!(
"UploadTex: {:?}, {}x{}, {:?}",
format,
texture.m_nWidth,
texture.m_nHeight,
image.usage()
);
if let Err(e) = overlay.set_image_vulkan(handle, &mut texture) {
panic!("Failed to set overlay texture: {}", e);
}

12
src/backend/overlay.rs
View File

@@ -3,7 +3,7 @@ use std::sync::{
Arc,
};
use glam::{Affine3A, Quat, Vec3A};
use glam::{Affine2, Affine3A, Quat, Vec3A};
use vulkano::image::ImageViewAbstract;
use crate::state::AppState;
@@ -22,12 +22,13 @@ pub struct OverlayState {
pub want_visible: bool,
pub show_hide: bool,
pub grabbable: bool,
pub dirty: bool,
pub transform: Affine3A,
pub spawn_point: Vec3A,
pub spawn_rotation: Quat,
pub relative_to: RelativeTo,
pub primary_pointer: Option<usize>,
pub interaction_transform: Affine3A,
pub interaction_transform: Affine2,
}
impl Default for OverlayState {
@@ -40,12 +41,13 @@ impl Default for OverlayState {
want_visible: false,
show_hide: false,
grabbable: false,
dirty: false,
relative_to: RelativeTo::None,
spawn_point: Vec3A::NEG_Z,
spawn_rotation: Quat::IDENTITY,
transform: Affine3A::IDENTITY,
primary_pointer: None,
interaction_transform: Affine3A::IDENTITY,
interaction_transform: Affine2::IDENTITY,
}
}
}
@@ -111,12 +113,14 @@ impl OverlayRenderer for FallbackRenderer {
fn resume(&mut self, _app: &mut AppState) {}
fn render(&mut self, _app: &mut AppState) {}
fn view(&mut self) -> Option<Arc<dyn ImageViewAbstract>> {
unimplemented!()
None
}
}
// Boilerplate and dummies
#[derive(Clone, Copy, Debug, Default)]
pub enum RelativeTo {
#[default]
None,
Head,
Hand(usize),