separate transfer and capture queues

src/backend/openvr/lines.rs

@@ -31,7 +31,7 @@ pub(super) struct LinePool {
 impl LinePool {
     pub fn new(graphics: Arc<WlxGraphics>) -> anyhow::Result<Self> {
         let mut command_buffer =
-            graphics.create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+            graphics.create_uploads_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
 
         let buf = vec![255; 16];
 

src/backend/openvr/mod.rs

@@ -357,7 +357,7 @@ pub fn openvr_run(running: Arc<AtomicBool>, show_by_default: bool) -> Result<(),
 
         log::trace!("Rendering overlays");
 
-        if let Some(mut future) = buffers.execute_now(state.graphics.queue.clone())? {
+        if let Some(mut future) = buffers.execute_now(state.graphics.graphics_queue.clone())? {
             if let Err(e) = future.flush() {
                 return Err(BackendError::Fatal(e.into()));
             }

src/backend/openvr/overlay.rs

@@ -265,8 +265,8 @@ impl OverlayData<OpenVrOverlayData> {
             m_pDevice: graphics.device.handle().as_raw() as *mut _,
             m_pPhysicalDevice: graphics.device.physical_device().handle().as_raw() as *mut _,
             m_pInstance: graphics.instance.handle().as_raw() as *mut _,
-            m_pQueue: graphics.queue.handle().as_raw() as *mut _,
+            m_pQueue: graphics.graphics_queue.handle().as_raw() as *mut _,
-            m_nQueueFamilyIndex: graphics.queue.queue_family_index(),
+            m_nQueueFamilyIndex: graphics.graphics_queue.queue_family_index(),
         };
         log::trace!(
             "{}: UploadTex {:?}, {}x{}, {:?}",

src/backend/openxr/mod.rs

@@ -116,7 +116,7 @@ pub fn openxr_run(running: Arc<AtomicBool>, show_by_default: bool) -> Result<(),
                 instance: app.graphics.instance.handle().as_raw() as _,
                 physical_device: app.graphics.device.physical_device().handle().as_raw() as _,
                 device: app.graphics.device.handle().as_raw() as _,
-                queue_family_index: app.graphics.queue.queue_family_index(),
+                queue_family_index: app.graphics.graphics_queue.queue_family_index(),
                 queue_index: 0,
             },
         )?;
@@ -405,7 +405,7 @@ pub fn openxr_run(running: Arc<AtomicBool>, show_by_default: bool) -> Result<(),
 
         lines.render(app.graphics.clone(), &mut buffers)?;
 
-        let future = buffers.execute_now(app.graphics.queue.clone())?;
+        let future = buffers.execute_now(app.graphics.graphics_queue.clone())?;
         if let Some(mut future) = future {
             if let Err(e) = future.flush() {
                 return Err(BackendError::Fatal(e.into()));

src/backend/openxr/skybox.rs

@@ -33,7 +33,7 @@ impl Skybox {
     pub fn new(app: &AppState) -> anyhow::Result<Self> {
         let mut command_buffer = app
             .graphics
-            .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+            .create_uploads_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
 
         let mut maybe_image = None;
 

src/backend/uidev/mod.rs

@@ -81,6 +81,7 @@ impl PreviewState {
     }
 }
 
+#[allow(clippy::too_many_lines)]
 pub fn uidev_run(panel_name: &str) -> anyhow::Result<()> {
     let (graphics, event_loop, window, surface) = WlxGraphics::new_window()?;
     window.set_resizable(false);
@@ -169,10 +170,13 @@ pub fn uidev_run(panel_name: &str) -> anyhow::Result<()> {
                     last_draw = std::time::Instant::now();
 
                     canvas_cmd_buf
-                        .execute_after(state.graphics.queue.clone(), Box::new(acquire_future))
+                        .execute_after(
+                            state.graphics.graphics_queue.clone(),
+                            Box::new(acquire_future),
+                        )
                         .unwrap()
                         .then_swapchain_present(
-                            graphics.queue.clone(),
+                            graphics.graphics_queue.clone(),
                             SwapchainPresentInfo::swapchain_image_index(
                                 preview.swapchain.clone(),
                                 image_index,

src/graphics/dds.rs

@@ -9,7 +9,7 @@ use vulkano::{
     DeviceSize,
 };
 
-use super::WlxCommandBuffer;
+use super::WlxUploadsBuffer;
 
 pub trait WlxCommandBufferDds {
     fn texture2d_dds<R>(&mut self, r: R) -> anyhow::Result<Arc<Image>>
@@ -17,7 +17,7 @@ pub trait WlxCommandBufferDds {
         R: Read;
 }
 
-impl WlxCommandBufferDds for WlxCommandBuffer {
+impl WlxCommandBufferDds for WlxUploadsBuffer {
     fn texture2d_dds<R>(&mut self, r: R) -> anyhow::Result<Arc<Image>>
     where
         R: Read,

src/graphics/mod.rs

@@ -114,7 +114,9 @@ pub const BLEND_ALPHA: AttachmentBlend = AttachmentBlend {
 pub struct WlxGraphics {
     pub instance: Arc<Instance>,
     pub device: Arc<Device>,
-    pub queue: Arc<Queue>,
+    pub graphics_queue: Arc<Queue>,
+    pub transfer_queue: Arc<Queue>,
+    pub capture_queue: Arc<Queue>,
 
     pub native_format: Format,
     pub texture_filtering: Filter,
@@ -244,12 +246,21 @@ impl WlxGraphics {
             physical_device.properties().device_name,
         );
 
-        let queue_family_index = physical_device
+        let queue_family_index_gfx = physical_device
             .queue_family_properties()
             .iter()
             .enumerate()
             .position(|(_, q)| q.queue_flags.intersects(QueueFlags::GRAPHICS))
-            .expect("Vulkan device has no graphics queue") as u32;
+            .expect("Vulkan device has no graphics queue")
+            as u32;
+
+        let queue_family_index_xfer = physical_device
+            .queue_family_properties()
+            .iter()
+            .enumerate()
+            .position(|(_, q)| q.queue_flags.intersects(QueueFlags::TRANSFER))
+            .expect("Vulkan device has no transfer queue")
+            as u32;
 
         let mut device_extensions = DeviceExtensions::empty();
         let dmabuf_extensions = get_dmabuf_extensions();
@@ -285,11 +296,19 @@ impl WlxGraphics {
             ..Default::default()
         };
 
-        let queue_priorities = [1.0];
+        let queue_priorities = vec![1.0];
 
-        let queue_create_infos = [vk::DeviceQueueCreateInfo::default()
+        let queue_create_infos = [
-            .queue_family_index(queue_family_index)
+            vk::DeviceQueueCreateInfo::default()
-            .queue_priorities(&queue_priorities)];
+                .queue_family_index(queue_family_index_gfx)
+                .queue_priorities(&queue_priorities),
+            vk::DeviceQueueCreateInfo::default()
+                .queue_family_index(queue_family_index_xfer)
+                .queue_priorities(&queue_priorities),
+            vk::DeviceQueueCreateInfo::default()
+                .queue_family_index(queue_family_index_xfer)
+                .queue_priorities(&queue_priorities),
+        ];
 
         let mut device_create_info = vk::DeviceCreateInfo::default()
             .queue_create_infos(&queue_create_infos)
@@ -323,11 +342,23 @@ impl WlxGraphics {
                 physical_device,
                 vk::Device::from_raw(vk_device as _),
                 DeviceCreateInfo {
-                    queue_create_infos: vec![QueueCreateInfo {
+                    queue_create_infos: vec![
-                        queue_family_index,
+                        QueueCreateInfo {
-                        queues: vec![1.0],
+                            queue_family_index: queue_family_index_gfx,
+                            queues: queue_priorities.clone(),
                             ..Default::default()
-                    }],
+                        },
+                        QueueCreateInfo {
+                            queue_family_index: queue_family_index_xfer,
+                            queues: queue_priorities.clone(),
+                            ..Default::default()
+                        },
+                        QueueCreateInfo {
+                            queue_family_index: queue_family_index_xfer,
+                            queues: queue_priorities,
+                            ..Default::default()
+                        },
+                    ],
                     enabled_extensions: device_extensions,
                     enabled_features: features,
                     ..Default::default()
@@ -351,9 +382,15 @@ impl WlxGraphics {
                 let _ = CString::from_raw(c_string.cast_mut());
             });
 
-        let queue = queues
+        let graphics_queue = queues
             .next()
-            .ok_or_else(|| anyhow::anyhow!("no GPU queues available"))?;
+            .ok_or_else(|| anyhow::anyhow!("no graphics queues available"))?;
+        let transfer_queue = queues
+            .next()
+            .ok_or_else(|| anyhow::anyhow!("no transfer queues available"))?;
+        let capture_queue = queues
+            .next()
+            .ok_or_else(|| anyhow::anyhow!("not enough transfer queues available"))?;
 
         let memory_allocator = memory_allocator(device.clone());
         let command_buffer_allocator = Arc::new(StandardCommandBufferAllocator::new(
@@ -374,7 +411,9 @@ impl WlxGraphics {
         let me = Self {
             instance,
             device,
-            queue,
+            graphics_queue,
+            transfer_queue,
+            capture_queue,
             native_format: Format::R8G8B8A8_UNORM,
             texture_filtering,
             memory_allocator,
@@ -419,7 +458,8 @@ impl WlxGraphics {
 
         let dmabuf_extensions = get_dmabuf_extensions();
 
-        let (physical_device, my_extensions, queue_family_index) = instance
+        let (physical_device, my_extensions, queue_family_index_gfx, queue_family_index_xfer) =
+            instance
                 .enumerate_physical_devices()?
                 .filter_map(|p| {
                     let mut my_extensions = vk_device_extensions_fn(&p);
@@ -461,7 +501,14 @@ impl WlxGraphics {
                         .position(|(_, q)| q.queue_flags.intersects(QueueFlags::GRAPHICS))
                         .map(|i| (p, my_extensions, i as u32))
                 })
-            .min_by_key(|(p, _, _)| match p.properties().device_type {
+                .filter_map(|(p, my_extensions, queue_family_index)| {
+                    p.queue_family_properties()
+                        .iter()
+                        .enumerate()
+                        .position(|(_, q)| q.queue_flags.intersects(QueueFlags::TRANSFER))
+                        .map(|i| (p, my_extensions, queue_family_index, i as u32))
+                })
+                .min_by_key(|(p, _, _, _)| match p.properties().device_type {
                     PhysicalDeviceType::DiscreteGpu => 0,
                     PhysicalDeviceType::IntegratedGpu => 1,
                     PhysicalDeviceType::VirtualGpu => 2,
@@ -490,10 +537,23 @@ impl WlxGraphics {
                     dynamic_rendering: true,
                     ..DeviceFeatures::empty()
                 },
-                queue_create_infos: vec![QueueCreateInfo {
+                queue_create_infos: vec![
-                    queue_family_index,
+                    QueueCreateInfo {
+                        queue_family_index: queue_family_index_gfx,
+                        queues: vec![1.0],
                         ..Default::default()
-                }],
+                    },
+                    QueueCreateInfo {
+                        queue_family_index: queue_family_index_xfer,
+                        queues: vec![1.0],
+                        ..Default::default()
+                    },
+                    QueueCreateInfo {
+                        queue_family_index: queue_family_index_xfer,
+                        queues: vec![1.0],
+                        ..Default::default()
+                    },
+                ],
                 ..Default::default()
             },
         )?;
@@ -507,9 +567,15 @@ impl WlxGraphics {
             device.enabled_extensions().ext_image_drm_format_modifier
         );
 
-        let queue = queues
+        let graphics_queue = queues
             .next()
-            .ok_or_else(|| anyhow::anyhow!("no GPU queues available"))?;
+            .ok_or_else(|| anyhow::anyhow!("no graphics queues available"))?;
+        let transfer_queue = queues
+            .next()
+            .ok_or_else(|| anyhow::anyhow!("no transfer queues available"))?;
+        let capture_queue = queues
+            .next()
+            .ok_or_else(|| anyhow::anyhow!("no transfer queues available"))?;
 
         let memory_allocator = memory_allocator(device.clone());
         let command_buffer_allocator = Arc::new(StandardCommandBufferAllocator::new(
@@ -530,7 +596,9 @@ impl WlxGraphics {
         let me = Self {
             instance,
             device,
-            queue,
+            graphics_queue,
+            transfer_queue,
+            capture_queue,
             memory_allocator,
             native_format: Format::R8G8B8A8_UNORM,
             texture_filtering,
@@ -675,7 +743,9 @@ impl WlxGraphics {
         let me = Self {
             instance,
             device,
-            queue,
+            graphics_queue: queue.clone(),
+            transfer_queue: queue.clone(), // what could go wrong!
+            capture_queue: queue,
             memory_allocator,
             native_format,
             texture_filtering: Filter::Linear,
@@ -982,18 +1052,57 @@ impl WlxGraphics {
         )?))
     }
 
+    /// Creates a CommandBuffer to be used for graphics workloads on the main thread.
     pub fn create_command_buffer(
         self: &Arc<Self>,
         usage: CommandBufferUsage,
     ) -> anyhow::Result<WlxCommandBuffer> {
         let command_buffer = AutoCommandBufferBuilder::primary(
             self.command_buffer_allocator.clone(),
-            self.queue.queue_family_index(),
+            self.graphics_queue.queue_family_index(),
             usage,
         )?;
         Ok(WlxCommandBuffer {
             graphics: self.clone(),
+            queue: self.graphics_queue.clone(),
             command_buffer,
+            dummy: None,
+        })
+    }
+
+    /// Creates a CommandBuffer to be used for texture uploads on the main thread.
+    pub fn create_uploads_command_buffer(
+        self: &Arc<Self>,
+        usage: CommandBufferUsage,
+    ) -> anyhow::Result<WlxUploadsBuffer> {
+        let command_buffer = AutoCommandBufferBuilder::primary(
+            self.command_buffer_allocator.clone(),
+            self.transfer_queue.queue_family_index(),
+            usage,
+        )?;
+        Ok(WlxUploadsBuffer {
+            graphics: self.clone(),
+            queue: self.transfer_queue.clone(),
+            command_buffer,
+            dummy: None,
+        })
+    }
+
+    /// Creates a CommandBuffer to be used for texture uploads on the capture thread.
+    pub fn create_capture_command_buffer(
+        self: &Arc<Self>,
+        usage: CommandBufferUsage,
+    ) -> anyhow::Result<WlxUploadsBuffer> {
+        let command_buffer = AutoCommandBufferBuilder::primary(
+            self.command_buffer_allocator.clone(),
+            self.capture_queue.queue_family_index(),
+            usage,
+        )?;
+        Ok(WlxUploadsBuffer {
+            graphics: self.clone(),
+            queue: self.capture_queue.clone(),
+            command_buffer,
+            dummy: None,
         })
     }
 
@@ -1017,7 +1126,7 @@ impl WlxGraphics {
         let command_buffer = unsafe {
             let mut builder = RecordingCommandBuffer::new(
                 self.command_buffer_allocator.clone(),
-                self.queue.queue_family_index(),
+                self.graphics_queue.queue_family_index(),
                 CommandBufferLevel::Primary,
                 CommandBufferBeginInfo {
                     usage: CommandBufferUsage::OneTimeSubmit,
@@ -1041,7 +1150,7 @@ impl WlxGraphics {
         let fns = self.device.fns();
         unsafe {
             (fns.v1_0.queue_submit)(
-                self.queue.handle(),
+                self.graphics_queue.handle(),
                 1,
                 [SubmitInfo::default().command_buffers(&[command_buffer.handle()])].as_ptr(),
                 fence.handle(),
@@ -1053,12 +1162,34 @@ impl WlxGraphics {
     }
 }
 
-pub struct WlxCommandBuffer {
+pub type WlxCommandBuffer = AnyCommandBuffer<GraphicsBuffer>;
+pub type WlxUploadsBuffer = AnyCommandBuffer<UploadBuffer>;
+
+pub struct GraphicsBuffer;
+pub struct UploadBuffer;
+
+pub struct AnyCommandBuffer<T> {
     pub graphics: Arc<WlxGraphics>,
     pub command_buffer: AutoCommandBufferBuilder<PrimaryAutoCommandBuffer>,
+    pub queue: Arc<Queue>,
+    dummy: Option<T>,
 }
 
-impl WlxCommandBuffer {
+impl<T> AnyCommandBuffer<T> {
+    pub fn build_and_execute(self) -> anyhow::Result<CommandBufferExecFuture<NowFuture>> {
+        let queue = self.queue.clone();
+        Ok(self.command_buffer.build()?.execute(queue)?)
+    }
+
+    pub fn build_and_execute_now(self) -> anyhow::Result<()> {
+        let mut exec = self.build_and_execute()?;
+        exec.flush()?;
+        exec.cleanup_finished();
+        Ok(())
+    }
+}
+
+impl AnyCommandBuffer<GraphicsBuffer> {
     pub fn begin_rendering(&mut self, render_target: Arc<ImageView>) -> anyhow::Result<()> {
         self.command_buffer.begin_rendering(RenderingInfo {
             contents: SubpassContents::SecondaryCommandBuffers,
@@ -1073,12 +1204,23 @@ impl WlxCommandBuffer {
         Ok(())
     }
 
+    pub fn build(self) -> anyhow::Result<Arc<PrimaryAutoCommandBuffer>> {
+        Ok(self.command_buffer.build()?)
+    }
+
     pub fn run_ref(&mut self, pass: &WlxPass) -> anyhow::Result<()> {
         self.command_buffer
             .execute_commands(pass.command_buffer.clone())?;
         Ok(())
     }
 
+    pub fn end_rendering(&mut self) -> anyhow::Result<()> {
+        self.command_buffer.end_rendering()?;
+        Ok(())
+    }
+}
+
+impl AnyCommandBuffer<UploadBuffer> {
     pub fn texture2d_raw(
         &mut self,
         width: u32,
@@ -1125,27 +1267,6 @@ impl WlxCommandBuffer {
 
         Ok(image)
     }
-
-    pub fn end_rendering(&mut self) -> anyhow::Result<()> {
-        self.command_buffer.end_rendering()?;
-        Ok(())
-    }
-
-    pub fn build(self) -> anyhow::Result<Arc<PrimaryAutoCommandBuffer>> {
-        Ok(self.command_buffer.build()?)
-    }
-
-    pub fn build_and_execute(self) -> anyhow::Result<CommandBufferExecFuture<NowFuture>> {
-        let queue = self.graphics.queue.clone();
-        Ok(self.build()?.execute(queue)?)
-    }
-
-    pub fn build_and_execute_now(self) -> anyhow::Result<()> {
-        let mut exec = self.build_and_execute()?;
-        exec.flush()?;
-        exec.cleanup_finished();
-        Ok(())
-    }
 }
 
 pub struct WlxPipeline {
@@ -1325,7 +1446,7 @@ impl WlxPass {
         let pipeline_inner = pipeline.inner();
         let mut command_buffer = AutoCommandBufferBuilder::secondary(
             pipeline.graphics.command_buffer_allocator.clone(),
-            pipeline.graphics.queue.queue_family_index(),
+            pipeline.graphics.graphics_queue.queue_family_index(),
             CommandBufferUsage::MultipleSubmit,
             CommandBufferInheritanceInfo {
                 render_pass: Some(CommandBufferInheritanceRenderPassType::BeginRendering(

src/gui/font.rs

@@ -5,7 +5,7 @@ use freetype::{bitmap::PixelMode, face::LoadFlag, Face, Library};
 use idmap::IdMap;
 use vulkano::{command_buffer::CommandBufferUsage, format::Format, image::Image};
 
-use crate::graphics::{WlxCommandBuffer, WlxGraphics};
+use crate::graphics::{WlxGraphics, WlxUploadsBuffer};
 
 pub struct FontCache {
     primary_font: Arc<str>,
@@ -202,7 +202,7 @@ impl FontCache {
         cp: usize,
         size: isize,
         graphics: Arc<WlxGraphics>,
-        cmd_buffer: &mut Option<WlxCommandBuffer>,
+        cmd_buffer: &mut Option<WlxUploadsBuffer>,
     ) -> anyhow::Result<Rc<Glyph>> {
         let key = self.get_font_for_cp(cp, size);
 
@@ -240,7 +240,8 @@ impl FontCache {
         };
 
         if cmd_buffer.is_none() {
-            *cmd_buffer = Some(graphics.create_command_buffer(CommandBufferUsage::OneTimeSubmit)?);
+            *cmd_buffer =
+                Some(graphics.create_uploads_command_buffer(CommandBufferUsage::OneTimeSubmit)?);
         }
 
         let texture = cmd_buffer.as_mut().unwrap().texture2d_raw(

src/gui/modular/mod.rs

@@ -554,7 +554,7 @@ fn sprite_from_path(path: Arc<str>, app: &mut AppState) -> anyhow::Result<Arc<Im
 
     let mut command_buffer = app
         .graphics
-        .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+        .create_uploads_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
 
     match command_buffer.texture2d_dds(f) {
         Ok(image) => {

src/overlays/screen.rs

@@ -53,7 +53,7 @@ use crate::{
     },
     config::{def_pw_tokens, GeneralConfig, PwTokenMap},
     graphics::{
-        fourcc_to_vk, CommandBuffers, WlxCommandBuffer, WlxGraphics, WlxPipeline, SWAPCHAIN_FORMAT,
+        fourcc_to_vk, CommandBuffers, WlxGraphics, WlxPipeline, WlxUploadsBuffer, SWAPCHAIN_FORMAT,
     },
     hid::{MOUSE_LEFT, MOUSE_MIDDLE, MOUSE_RIGHT},
     state::{AppSession, AppState, KeyboardFocus, ScreenMeta},
@@ -180,7 +180,7 @@ impl ScreenPipeline {
         })
     }
 
-    fn ensure_mouse_initialized(&mut self, uploads: &mut WlxCommandBuffer) -> anyhow::Result<()> {
+    fn ensure_mouse_initialized(&mut self, uploads: &mut WlxUploadsBuffer) -> anyhow::Result<()> {
         if self.mouse.is_some() {
             return Ok(());
         }
@@ -386,7 +386,7 @@ fn upload_image(
 ) -> Option<Arc<Image>> {
     let mut upload = match me
         .graphics
-        .create_command_buffer(CommandBufferUsage::OneTimeSubmit)
+        .create_capture_command_buffer(CommandBufferUsage::OneTimeSubmit)
     {
         Ok(x) => x,
         Err(e) => {
@@ -556,7 +556,7 @@ impl OverlayRenderer for ScreenRenderer {
                 let mut pipeline = ScreenPipeline::new(&capture.image.extent(), app)?;
                 let mut upload = app
                     .graphics
-                    .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+                    .create_uploads_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
                 pipeline.ensure_mouse_initialized(&mut upload)?;
                 upload.build_and_execute_now()?;
                 pipeline

src/overlays/wayvr.rs

@@ -564,7 +564,7 @@ impl WayVRRenderer {
     ) -> anyhow::Result<()> {
         let mut upload = self
             .graphics
-            .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+            .create_uploads_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
 
         let tex = upload.texture2d_raw(
             u32::from(data.width),