openxr rendering

This commit is contained in:
galister
2024-01-28 01:53:47 +01:00
parent 4aa52c1e67
commit 93125dd65e
7 changed files with 368 additions and 159 deletions

View File

@@ -39,7 +39,6 @@ where
{
pub fn new(app: &mut AppState) -> Self {
let mut overlays = IdMap::new();
let (screens, extent) = if std::env::var("WAYLAND_DISPLAY").is_ok() {
get_screens_wayland(&app.session)
} else {
@@ -51,7 +50,7 @@ where
overlays.insert(watch.state.id, watch);
let mut keyboard = create_keyboard(&app);
keyboard.state.want_visible = false;
keyboard.state.want_visible = true;
overlays.insert(keyboard.state.id, keyboard);
let mut first = true;
@@ -62,7 +61,6 @@ where
}
overlays.insert(screen.state.id, screen);
}
Self { overlays, extent }
}

View File

@@ -3,6 +3,8 @@ use openxr as xr;
use crate::{backend::input::Pointer, state::AppState};
use super::XrState;
type XrSession = xr::Session<xr::Vulkan>;
pub(super) struct OpenXrInputSource {
@@ -29,8 +31,9 @@ pub(super) struct OpenXrHandSource {
}
impl OpenXrInputSource {
pub fn new(session: XrSession) -> Self {
let mut action_set = session
pub fn new(xr: &XrState) -> Self {
let mut action_set = xr
.session
.instance()
.create_action_set("wlx-overlay-s", "WlxOverlay-S Actions", 0)
.expect("Failed to create action set");
@@ -38,41 +41,44 @@ impl OpenXrInputSource {
let left_source = OpenXrHandSource::new(&mut action_set, "left");
let right_source = OpenXrHandSource::new(&mut action_set, "right");
session.attach_action_sets(&[&action_set]).unwrap();
xr.session.attach_action_sets(&[&action_set]).unwrap();
let stage = session
let stage = xr
.session
.create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)
.unwrap();
Self {
action_set,
hands: [
OpenXrHand::new(session.clone(), left_source),
OpenXrHand::new(session, right_source),
OpenXrHand::new(&xr, left_source),
OpenXrHand::new(&xr, right_source),
],
stage,
}
}
pub fn update(&self, session: &XrSession, time: xr::Time, state: &mut AppState) {
session.sync_actions(&[(&self.action_set).into()]).unwrap();
pub fn update(&self, xr: &XrState, state: &mut AppState) {
xr.session
.sync_actions(&[(&self.action_set).into()])
.unwrap();
for i in 0..2 {
self.hands[i].update(
&mut state.input_state.pointers[i],
&self.stage,
session,
time,
&xr.session,
xr.predicted_display_time,
);
}
}
}
impl OpenXrHand {
pub(super) fn new(session: XrSession, source: OpenXrHandSource) -> Self {
pub(super) fn new(xr: &XrState, source: OpenXrHandSource) -> Self {
let space = source
.action_pose
.create_space(session.clone(), xr::Path::NULL, xr::Posef::IDENTITY)
.create_space(xr.session.clone(), xr::Path::NULL, xr::Posef::IDENTITY)
.unwrap();
Self { source, space }

View File

@@ -7,14 +7,10 @@ use std::{
};
use anyhow::{bail, ensure};
use ash::vk::{self};
use glam::{Affine3A, Quat, Vec3};
use openxr as xr;
use vulkano::{
image::{view::ImageView, ImageCreateInfo, ImageUsage},
render_pass::{Framebuffer, FramebufferCreateInfo},
Handle, VulkanObject,
};
use vulkano::{Handle, VulkanObject};
use xr::{CompositionLayerFlags, EyeVisibility};
use crate::{
backend::{common::OverlayContainer, input::interact, openxr::overlay::OpenXrOverlayData},
@@ -31,6 +27,13 @@ mod overlay;
const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
const VIEW_COUNT: u32 = 2;
struct XrState {
instance: xr::Instance,
system: xr::SystemId,
session: xr::Session<xr::Vulkan>,
predicted_display_time: xr::Time,
}
pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
let (xr_instance, system) = match init_xr() {
Ok((xr_instance, system)) => (xr_instance, system),
@@ -45,40 +48,51 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
.unwrap()[0];
log::info!("Using environment blend mode: {:?}", environment_blend_mode);
let mut state = {
let mut app_state = {
let graphics = WlxGraphics::new_xr(xr_instance.clone(), system);
AppState::from_graphics(graphics)
};
let mut overlays = OverlayContainer::<OpenXrOverlayData>::new(&mut state);
let mut overlays = OverlayContainer::<OpenXrOverlayData>::new(&mut app_state);
state.hid_provider.set_desktop_extent(overlays.extent);
app_state.hid_provider.set_desktop_extent(overlays.extent);
let (session, mut frame_wait, mut frame_stream) = unsafe {
xr_instance
.create_session::<xr::Vulkan>(
system,
&xr::vulkan::SessionCreateInfo {
instance: state.graphics.instance.handle().as_raw() as _,
physical_device: state.graphics.device.physical_device().handle().as_raw() as _,
device: state.graphics.device.handle().as_raw() as _,
queue_family_index: state.graphics.queue.queue_family_index(),
instance: app_state.graphics.instance.handle().as_raw() as _,
physical_device: app_state
.graphics
.device
.physical_device()
.handle()
.as_raw() as _,
device: app_state.graphics.device.handle().as_raw() as _,
queue_family_index: app_state.graphics.queue.queue_family_index(),
queue_index: 0,
},
)
.unwrap()
};
let input_source = input::OpenXrInputSource::new(session.clone());
let mut xr_state = XrState {
instance: xr_instance,
system,
session,
predicted_display_time: xr::Time::from_nanos(0),
};
let input_source = input::OpenXrInputSource::new(&xr_state);
let mut swapchain = None;
let mut session_running = false;
let mut event_storage = xr::EventDataBuffer::new();
'main_loop: loop {
if !running.load(Ordering::Relaxed) {
log::warn!("Received shutdown signal.");
match session.request_exit() {
match xr_state.session.request_exit() {
Ok(_) => log::info!("OpenXR session exit requested."),
Err(xr::sys::Result::ERROR_SESSION_NOT_RUNNING) => break 'main_loop,
Err(e) => {
@@ -88,7 +102,7 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
}
}
while let Some(event) = xr_instance.poll_event(&mut event_storage).unwrap() {
while let Some(event) = xr_state.instance.poll_event(&mut event_storage).unwrap() {
use xr::Event::*;
match event {
SessionStateChanged(e) => {
@@ -97,11 +111,11 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
println!("entered state {:?}", e.state());
match e.state() {
xr::SessionState::READY => {
session.begin(VIEW_TYPE).unwrap();
xr_state.session.begin(VIEW_TYPE).unwrap();
session_running = true;
}
xr::SessionState::STOPPING => {
session.end().unwrap();
xr_state.session.end().unwrap();
session_running = false;
}
xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => {
@@ -128,6 +142,8 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
let xr_frame_state = frame_wait.wait().unwrap();
frame_stream.begin().unwrap();
xr_state.predicted_display_time = xr_frame_state.predicted_display_time;
if !xr_frame_state.should_render {
frame_stream
.end(
@@ -139,11 +155,12 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
continue 'main_loop;
}
state.input_state.pre_update();
input_source.update(&session, xr_frame_state.predicted_display_time, &mut state);
state.input_state.post_update();
app_state.input_state.pre_update();
input_source.update(&xr_state, &mut app_state);
app_state.input_state.post_update();
let (_, views) = session
let (_, views) = xr_state
.session
.locate_views(
VIEW_TYPE,
xr_frame_state.predicted_display_time,
@@ -151,97 +168,55 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
)
.unwrap();
state.input_state.hmd = hmd_pose_from_views(&views);
app_state.input_state.hmd = hmd_pose_from_views(&views);
let _pointer_lengths = interact(&mut overlays, &mut state);
let _pointer_lengths = interact(&mut overlays, &mut app_state);
//TODO lines
overlays
.iter_mut()
.filter(|o| o.state.want_visible)
.for_each(|o| o.render(&mut state));
let mut layers = vec![];
state.hid_provider.on_new_frame();
let swapchain = swapchain.get_or_insert_with(|| {
let views = xr_instance
.enumerate_view_configuration_views(system, VIEW_TYPE)
.unwrap();
debug_assert_eq!(views.len(), VIEW_COUNT as usize);
debug_assert_eq!(views[0], views[1]);
let resolution = vk::Extent2D {
width: views[0].recommended_image_rect_width,
height: views[0].recommended_image_rect_height,
};
log::info!(
"Swapchain resolution: {}x{}",
resolution.width,
resolution.height
);
let swapchain = session
.create_swapchain(&xr::SwapchainCreateInfo {
create_flags: xr::SwapchainCreateFlags::EMPTY,
usage_flags: xr::SwapchainUsageFlags::COLOR_ATTACHMENT
| xr::SwapchainUsageFlags::SAMPLED,
format: state.graphics.native_format as _,
sample_count: 1,
width: resolution.width,
height: resolution.height,
face_count: 1,
array_size: VIEW_COUNT,
mip_count: 1,
})
.unwrap();
// thanks @yshui
let swapchain_images = swapchain
.enumerate_images()
.unwrap()
.into_iter()
.map(|handle| {
let vk_image = vk::Image::from_raw(handle);
let raw_image = unsafe {
vulkano::image::sys::RawImage::from_handle(
state.graphics.device.clone(),
vk_image,
ImageCreateInfo {
format: state.graphics.native_format,
extent: [resolution.width * 2, resolution.height, 1],
usage: ImageUsage::COLOR_ATTACHMENT | ImageUsage::TRANSFER_DST,
..Default::default()
},
)
.unwrap()
};
// SAFETY: OpenXR guarantees that the image is a swapchain image, thus has memory backing it.
let image = Arc::new(unsafe { raw_image.assume_bound() });
let view = ImageView::new_default(image).unwrap();
let fb = Framebuffer::new(
todo!(),
FramebufferCreateInfo {
attachments: vec![view.clone()],
..Default::default()
},
)
.unwrap();
XrFramebuffer {
framebuffer: fb,
color: view,
}
})
.collect();
XrSwapchain {
handle: swapchain,
buffers: swapchain_images,
resolution: [resolution.width, resolution.height, 1],
for o in overlays.iter_mut() {
if !o.state.want_visible {
continue;
}
});
let image_index = swapchain.handle.acquire_image().unwrap();
if !o.data.init {
o.init(&mut app_state);
o.data.init = true;
}
o.render(&mut app_state);
let transform = o.state.transform;
let Some((sub_image, extent)) = o.present_xr(&xr_state, &mut app_state) else {
continue;
};
let quad = xr::CompositionLayerQuad::new()
.pose(transform_to_posef(&transform))
.sub_image(sub_image)
.eye_visibility(EyeVisibility::BOTH)
.layer_flags(CompositionLayerFlags::CORRECT_CHROMATIC_ABERRATION)
.space(&input_source.stage)
.size(extent);
layers.push(quad);
}
let frame_ref = layers
.iter()
.map(|f| f as &xr::CompositionLayerBase<xr::Vulkan>)
.collect::<Vec<_>>();
frame_stream
.end(
xr_state.predicted_display_time,
environment_blend_mode,
&frame_ref,
)
.unwrap();
app_state.hid_provider.on_new_frame();
}
Ok(())
@@ -268,13 +243,13 @@ fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
enabled_extensions.khr_vulkan_enable2 = true;
enabled_extensions.extx_overlay = true;
#[cfg(not(debug_assertions))]
//#[cfg(not(debug_assertions))]
let layers = [];
#[cfg(debug_assertions)]
let layers = [
"XR_APILAYER_LUNARG_api_dump",
"XR_APILAYER_LUNARG_standard_validation",
];
//#[cfg(debug_assertions)]
//let layers = [
// "XR_APILAYER_LUNARG_api_dump",
// "XR_APILAYER_LUNARG_standard_validation",
//];
let Ok(xr_instance) = entry.create_instance(
&xr::ApplicationInfo {
@@ -321,17 +296,6 @@ fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
Ok((xr_instance, system))
}
struct XrSwapchain {
handle: xr::Swapchain<xr::Vulkan>,
buffers: Vec<XrFramebuffer>,
resolution: [u32; 3],
}
struct XrFramebuffer {
framebuffer: Arc<Framebuffer>,
color: Arc<ImageView>,
}
fn hmd_pose_from_views(views: &Vec<xr::View>) -> Affine3A {
let pos = {
let pos0: Vec3 = unsafe { std::mem::transmute(views[0].pose.position) };
@@ -361,3 +325,22 @@ fn quat_lerp(a: Quat, mut b: Quat, t: f32) -> Quat {
)
.normalize()
}
fn transform_to_posef(transform: &Affine3A) -> xr::Posef {
let translation = transform.translation;
let rotation = Quat::from_mat3a(&transform.matrix3);
xr::Posef {
orientation: xr::Quaternionf {
x: rotation.x,
y: rotation.y,
z: rotation.z,
w: rotation.w,
},
position: xr::Vector3f {
x: translation.x,
y: translation.y,
z: translation.z,
},
}
}

View File

@@ -1,7 +1,176 @@
pub struct OpenXrOverlayData {}
use std::sync::Arc;
impl Default for OpenXrOverlayData {
fn default() -> Self {
Self {}
use super::XrState;
use crate::{
backend::overlay::OverlayData,
graphics::{WlxPass, WlxPipeline},
shaders::{frag_srgb, vert_common},
state::AppState,
};
use ash::vk::{self};
use openxr as xr;
use vulkano::{
command_buffer::CommandBufferUsage,
image::{sampler::Filter, view::ImageView, ImageCreateInfo, ImageUsage},
render_pass::{Framebuffer, FramebufferCreateInfo},
Handle,
};
#[derive(Default)]
pub struct OpenXrOverlayData {
inner: Option<XrOverlayData>,
pub(super) init: bool,
}
impl OverlayData<OpenXrOverlayData> {
pub(super) fn initialize(&mut self, xr: &XrState, state: &mut AppState) {
let Some(my_view) = self.view() else {
log::error!("Failed to get view for overlay");
return;
};
self.data.inner = {
let extent = my_view.image().extent();
let swapchain = xr
.session
.create_swapchain(&xr::SwapchainCreateInfo {
create_flags: xr::SwapchainCreateFlags::EMPTY,
usage_flags: xr::SwapchainUsageFlags::COLOR_ATTACHMENT
| xr::SwapchainUsageFlags::SAMPLED,
format: state.graphics.native_format as _,
sample_count: 1,
width: extent[0],
height: extent[1],
face_count: 1,
array_size: 1,
mip_count: 1,
})
.unwrap();
let framebuffers = swapchain
.enumerate_images()
.unwrap()
.into_iter()
.map(|handle| {
let vk_image = vk::Image::from_raw(handle);
// thanks @yshui
let raw_image = unsafe {
vulkano::image::sys::RawImage::from_handle(
state.graphics.device.clone(),
vk_image,
ImageCreateInfo {
format: state.graphics.native_format,
extent,
usage: ImageUsage::COLOR_ATTACHMENT | ImageUsage::TRANSFER_DST,
..Default::default()
},
)
.unwrap()
};
// SAFETY: OpenXR guarantees that the image is a swapchain image, thus has memory backing it.
let image = Arc::new(unsafe { raw_image.assume_bound() });
let view = ImageView::new_default(image).unwrap();
// HACK: maybe not create one pipeline per image?
let pipeline = state.graphics.create_pipeline(
view.clone(),
vert_common::load(state.graphics.device.clone()).unwrap(),
frag_srgb::load(state.graphics.device.clone()).unwrap(),
state.graphics.native_format,
);
let set = pipeline.uniform_sampler(0, my_view.clone(), Filter::Linear);
let pass = pipeline.create_pass(
[view.image().extent()[0] as _, view.image().extent()[1] as _],
state.graphics.quad_verts.clone(),
state.graphics.quad_indices.clone(),
vec![set],
);
let inner = Framebuffer::new(
pipeline.render_pass.clone(),
FramebufferCreateInfo {
attachments: vec![view.clone()],
extent: [view.image().extent()[0] as _, view.image().extent()[1] as _],
layers: 1,
..Default::default()
},
)
.unwrap();
XrFramebuffer {
inner,
view,
pipeline,
pass,
}
})
.collect();
Some(XrOverlayData {
swapchain,
framebuffers,
extent,
})
};
}
pub(super) fn present_xr(
&mut self,
xr: &XrState,
state: &mut AppState,
) -> Option<(xr::SwapchainSubImage<xr::Vulkan>, xr::Extent2Df)> {
if self.data.inner.is_none() {
self.initialize(xr, state);
return None;
}
let data = self.data.inner.as_mut().unwrap();
let idx = data.swapchain.acquire_image().unwrap();
data.swapchain.wait_image(xr::Duration::INFINITE).unwrap();
let frame = &data.framebuffers[idx as usize];
let mut command_buffer = state
.graphics
.create_command_buffer(CommandBufferUsage::OneTimeSubmit)
.begin_render_pass(&frame.pipeline);
command_buffer.run_ref(&frame.pass);
command_buffer.end_render_pass().build_and_execute_now();
data.swapchain.release_image().unwrap();
let extent = xr::Extent2Df {
width: self.state.width,
height: (data.extent[1] as f32 / data.extent[0] as f32) * self.state.width,
};
Some((
xr::SwapchainSubImage::new()
.swapchain(&data.swapchain)
.image_rect(xr::Rect2Di {
offset: xr::Offset2Di { x: 0, y: 0 },
extent: xr::Extent2Di {
width: data.extent[0] as _,
height: data.extent[1] as _,
},
})
.image_array_index(0),
extent,
))
}
}
struct XrOverlayData {
swapchain: xr::Swapchain<xr::Vulkan>,
extent: [u32; 3],
framebuffers: Vec<XrFramebuffer>,
}
struct XrFramebuffer {
inner: Arc<Framebuffer>,
view: Arc<ImageView>,
pipeline: Arc<WlxPipeline>,
pass: WlxPass,
}

View File

@@ -129,6 +129,11 @@ where
T: Default,
{
pub fn init(&mut self, app: &mut AppState) {
self.state.transform.translation = app
.input_state
.hmd
.transform_point3a(self.state.spawn_point);
self.state.realign(&app.input_state.hmd);
self.backend.init(app);
}
pub fn render(&mut self, app: &mut AppState) {

View File

@@ -104,10 +104,12 @@ pub struct WlxGraphics {
impl WlxGraphics {
#[cfg(feature = "openxr")]
pub fn new_xr(xr_instance: openxr::Instance, system: openxr::SystemId) -> Arc<Self> {
use std::ffi::{self, c_char, CString};
use vulkano::Handle;
let vk_target_version = vk::make_api_version(0, 1, 1, 0); // Vulkan 1.1 guarantees multiview support
let target_version = vulkano::Version::V1_1;
let vk_target_version = vk::make_api_version(0, 1, 3, 0); // Vulkan 1.1 guarantees multiview support
let target_version = vulkano::Version::V1_3;
let library = VulkanLibrary::new().unwrap();
let vk_entry = unsafe { ash::Entry::load().unwrap() };
@@ -131,7 +133,9 @@ impl WlxGraphics {
Instance::from_handle(
library,
ash::vk::Instance::from_raw(vk_instance as _),
InstanceCreateInfo::default(), // FIXME
InstanceCreateInfo {
..Default::default()
},
)
};
@@ -167,6 +171,27 @@ impl WlxGraphics {
.position(|(_, q)| q.queue_flags.intersects(QueueFlags::GRAPHICS))
.expect("Vulkan device has no graphics queue") as u32;
let device_extensions = DeviceExtensions {
khr_external_memory: true,
khr_external_memory_fd: true,
ext_external_memory_dma_buf: true,
ext_image_drm_format_modifier: true,
amd_memory_overallocation_behavior: true,
..DeviceExtensions::empty()
};
let device_extensions_raw = device_extensions
.clone()
.into_iter()
.filter_map(|(name, enabled)| {
if enabled {
Some(ffi::CString::new(name).unwrap().into_raw() as *const c_char)
} else {
None
}
})
.collect::<Vec<_>>();
let (device, mut queues) = unsafe {
let vk_device = xr_instance
.create_vulkan_device(
@@ -178,10 +203,8 @@ impl WlxGraphics {
.queue_family_index(queue_family_index)
.queue_priorities(&[1.0])
.build()])
.push_next(&mut vk::PhysicalDeviceMultiviewFeatures {
multiview: vk::TRUE,
..Default::default()
}) as *const _ as *const _,
.enabled_extension_names(&device_extensions_raw)
as *const _ as *const _,
)
.expect("XR error creating Vulkan device")
.map_err(vk::Result::from_raw)
@@ -190,10 +213,24 @@ impl WlxGraphics {
vulkano::device::Device::from_handle(
physical_device,
vk::Device::from_raw(vk_device as _),
DeviceCreateInfo::default(), // FIXME
DeviceCreateInfo {
queue_create_infos: vec![QueueCreateInfo {
queue_family_index,
..Default::default()
}],
enabled_extensions: device_extensions,
..Default::default()
},
)
};
// Drop the CStrings
device_extensions_raw
.into_iter()
.for_each(|c_string| unsafe {
let _ = CString::from_raw(c_string as _);
});
let queue = queues.next().unwrap();
let memory_allocator = Arc::new(StandardMemoryAllocator::new_default(device.clone()));
@@ -255,6 +292,7 @@ impl WlxGraphics {
khr_external_memory_fd: true,
ext_external_memory_dma_buf: true,
ext_image_drm_format_modifier: true,
amd_memory_overallocation_behavior: true,
..DeviceExtensions::empty()
};
@@ -565,6 +603,12 @@ impl WlxGraphics {
}
pub fn render_texture(&self, width: u32, height: u32, format: Format) -> Arc<Image> {
log::debug!(
"Render texture: {}x{} {}MB",
width,
height,
(width * height * 4) / (1024 * 1024)
);
Image::new(
self.memory_allocator.clone(),
ImageCreateInfo {
@@ -736,6 +780,12 @@ impl WlxCommandBuffer {
format: Format,
data: &[u8],
) -> Arc<Image> {
log::debug!(
"Texture2D: {}x{} {}MB",
width,
height,
data.len() / (1024 * 1024)
);
let image = Image::new(
self.graphics.memory_allocator.clone(),
ImageCreateInfo {
@@ -958,6 +1008,7 @@ impl WlxPipeline {
pub fn swap_framebuffer(&mut self, new_framebuffer: Arc<Framebuffer>) -> Arc<Framebuffer> {
let old = self.framebuffer.clone();
self.view = new_framebuffer.attachments()[0].clone();
self.framebuffer = new_framebuffer;
old
}

View File

@@ -23,7 +23,7 @@ use wlx_capture::{
WlxCapture,
};
use glam::{vec2, Affine2, Quat, Vec2, Vec3};
use glam::{vec2, vec3a, Affine2, Quat, Vec2, Vec3};
use crate::{
backend::{
@@ -241,14 +241,9 @@ impl ScreenRenderer {
}
impl OverlayRenderer for ScreenRenderer {
fn init(&mut self, _app: &mut AppState) {
self.receiver = Some(self.capture.init());
}
fn init(&mut self, _app: &mut AppState) {}
fn render(&mut self, app: &mut AppState) {
let Some(receiver) = self.receiver.as_mut() else {
log::error!("No receiver");
return;
};
let receiver = self.receiver.get_or_insert_with(|| self.capture.init());
for frame in receiver.try_iter() {
match frame {
@@ -407,6 +402,8 @@ where
show_hide: true,
grabbable: true,
spawn_rotation: Quat::from_axis_angle(axis, angle),
spawn_point: vec3a(0., 0.5, -1.),
width: 1.5,
interaction_transform,
..Default::default()
},