Bound Vulkan host buffer pool memory (#201)

Co-authored-by: Dafenx <196083014+Dafenxz0@users.noreply.github.com>

src/SharpEmu.Libs/VideoOut/VulkanHostBufferPool.cs

@@ -0,0 +1,109 @@
+// Copyright (C) 2026 SharpEmu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+using Silk.NET.Vulkan;
+using VkBuffer = Silk.NET.Vulkan.Buffer;
+
+namespace SharpEmu.Libs.VideoOut;
+
+internal readonly record struct VulkanHostBufferPoolKey(
+    BufferUsageFlags Usage,
+    ulong Capacity);
+
+internal readonly record struct VulkanHostBufferAllocation(
+    VkBuffer Buffer,
+    DeviceMemory Memory,
+    VulkanHostBufferPoolKey Key,
+    nint Mapped);
+
+internal sealed class VulkanHostBufferPool : IDisposable
+{
+    private readonly Dictionary<VulkanHostBufferPoolKey, Stack<VulkanHostBufferAllocation>>
+        _available = [];
+    private readonly Dictionary<ulong, VulkanHostBufferAllocation> _allocations = [];
+    private readonly HashSet<ulong> _cachedHandles = [];
+    private readonly Action<VulkanHostBufferAllocation> _destroy;
+
+    public VulkanHostBufferPool(
+        ulong maximumCachedBytes,
+        Action<VulkanHostBufferAllocation> destroy)
+    {
+        MaximumCachedBytes = maximumCachedBytes;
+        _destroy = destroy;
+    }
+
+    public ulong MaximumCachedBytes { get; }
+
+    public ulong CachedBytes { get; private set; }
+
+    public bool TryRent(
+        VulkanHostBufferPoolKey key,
+        out VulkanHostBufferAllocation allocation)
+    {
+        if (!_available.TryGetValue(key, out var available) ||
+            !available.TryPop(out allocation))
+        {
+            allocation = default;
+            return false;
+        }
+
+        _cachedHandles.Remove(allocation.Buffer.Handle);
+        CachedBytes -= allocation.Key.Capacity;
+        return true;
+    }
+
+    public void Register(VulkanHostBufferAllocation allocation)
+    {
+        if (allocation.Buffer.Handle == 0)
+        {
+            throw new ArgumentException("A pooled buffer must have a valid handle.", nameof(allocation));
+        }
+
+        _allocations.Add(allocation.Buffer.Handle, allocation);
+    }
+
+    public bool Return(VkBuffer buffer, DeviceMemory memory)
+    {
+        if (!_allocations.TryGetValue(buffer.Handle, out var allocation) ||
+            allocation.Memory.Handle != memory.Handle)
+        {
+            return false;
+        }
+
+        if (!_cachedHandles.Add(buffer.Handle))
+        {
+            return true;
+        }
+
+        if (allocation.Key.Capacity > MaximumCachedBytes - CachedBytes)
+        {
+            _cachedHandles.Remove(buffer.Handle);
+            _allocations.Remove(buffer.Handle);
+            _destroy(allocation);
+            return true;
+        }
+
+        if (!_available.TryGetValue(allocation.Key, out var available))
+        {
+            available = [];
+            _available.Add(allocation.Key, available);
+        }
+
+        available.Push(allocation);
+        CachedBytes += allocation.Key.Capacity;
+        return true;
+    }
+
+    public void Dispose()
+    {
+        foreach (var allocation in _allocations.Values)
+        {
+            _destroy(allocation);
+        }
+
+        _allocations.Clear();
+        _available.Clear();
+        _cachedHandles.Clear();
+        CachedBytes = 0;
+    }
+}

src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.cs

@@ -124,6 +124,7 @@ internal static unsafe class VulkanVideoPresenter
     // stays tighter than the drain budget because queued draws pin their
     // pooled guest-data arrays until the render thread uploads them.
     private const int MaxPendingGuestWork = 64;
+    private const ulong MaximumCachedHostBufferBytes = 128UL * 1024 * 1024;
     // A captured 4K flip can consume tens of MiB of device-local memory.
     // Retain only a short presentation queue while always preserving the
     // newest generation; older immutable versions are retired immediately.
@@ -2392,9 +2393,7 @@ internal static unsafe class VulkanVideoPresenter
             new(ReferenceEqualityComparer.Instance);
         private readonly Dictionary<DescriptorLayoutKey, DescriptorLayoutBundle>
             _descriptorLayouts = new();
-        private readonly Dictionary<HostBufferPoolKey, Stack<HostBufferAllocation>>
+        private readonly VulkanHostBufferPool _hostBufferPool;
-            _hostBufferPool = new();
-        private readonly Dictionary<ulong, HostBufferAllocation> _hostBufferAllocations = new();
         private readonly List<GuestBufferAllocation> _guestBufferAllocations = [];
         private readonly Queue<PendingGuestSubmission> _pendingGuestSubmissions = new();
         private readonly Dictionary<string, ulong> _lastSubmittedTimelineByGuestQueue =
@@ -2416,10 +2415,6 @@ internal static unsafe class VulkanVideoPresenter
             GuestRasterState Raster,
             GuestDepthState Depth);
 
-        private readonly record struct HostBufferPoolKey(
-            BufferUsageFlags Usage,
-            ulong Capacity);
-
         private readonly record struct DescriptorLayoutKey(
             ShaderStageFlags Stages,
             string Resources);
@@ -2432,12 +2427,6 @@ internal static unsafe class VulkanVideoPresenter
             DescriptorSetLayout DescriptorSetLayout,
             PipelineLayout PipelineLayout);
 
-        private sealed record HostBufferAllocation(
-            VkBuffer Buffer,
-            DeviceMemory Memory,
-            HostBufferPoolKey Key,
-            nint Mapped);
-
         private readonly record struct DirtyGuestBufferRange(ulong Offset, ulong Length);
 
         private sealed class GuestBufferAllocation
@@ -2627,6 +2616,9 @@ internal static unsafe class VulkanVideoPresenter
 
         public Presenter(uint width, uint height)
         {
+            _hostBufferPool = new VulkanHostBufferPool(
+                MaximumCachedHostBufferBytes,
+                DestroyHostBufferAllocation);
             var options = WindowOptions.DefaultVulkan;
             options.Size = new Vector2D<int>((int)DefaultWindowWidth, (int)DefaultWindowHeight);
             options.Title = VideoOutExports.GetWindowTitle();
@@ -5351,7 +5343,8 @@ internal static unsafe class VulkanVideoPresenter
                     resources.IndexBuffer = CreateHostBuffer(
                         indexBuffer.Data.AsSpan(0, indexBuffer.Length),
                         BufferUsageFlags.IndexBufferBit,
-                        out resources.IndexMemory);
+                        out resources.IndexMemory,
+                        out _);
                     resources.Index32Bit = indexBuffer.Is32Bit;
                     if (indexBuffer.Pooled)
                     {
@@ -7647,8 +7640,8 @@ internal static unsafe class VulkanVideoPresenter
             var buffer = CreateHostBuffer(
                 guestBuffer.Data.AsSpan(0, guestBuffer.Length),
                 BufferUsageFlags.StorageBufferBit,
-                out var memory);
+                out var memory,
-            var allocation = _hostBufferAllocations[buffer.Handle];
+                out var mapped);
             if (guestBuffer.Pooled)
             {
                 GuestDataPool.Return(guestBuffer.Data);
@@ -7661,7 +7654,7 @@ internal static unsafe class VulkanVideoPresenter
                 WriteBackToGuest = false,
                 Buffer = buffer,
                 Memory = memory,
-                Mapped = allocation.Mapped,
+                Mapped = mapped,
                 Offset = 0,
                 Size = (ulong)Math.Max(guestBuffer.Length, sizeof(uint)),
                 GuestOffset = 0,
@@ -7881,7 +7874,8 @@ internal static unsafe class VulkanVideoPresenter
             var buffer = CreateHostBuffer(
                 source,
                 BufferUsageFlags.VertexBufferBit,
-                out var memory);
+                out var memory,
+                out _);
             var size = (ulong)Math.Max(guestBuffer.Length, sizeof(uint));
             if (_setDebugUtilsObjectName is not null)
             {
@@ -7950,19 +7944,15 @@ internal static unsafe class VulkanVideoPresenter
         private VkBuffer CreateHostBuffer(
             ReadOnlySpan<byte> data,
             BufferUsageFlags usage,
-            out DeviceMemory memory)
+            out DeviceMemory memory,
+            out nint mapped)
         {
             var size = (ulong)Math.Max(data.Length, sizeof(uint));
             var capacity = BitOperations.RoundUpToPowerOf2(size);
-            var key = new HostBufferPoolKey(usage, capacity);
+            var key = new VulkanHostBufferPoolKey(usage, capacity);
-            if (!_hostBufferPool.TryGetValue(key, out var available))
-            {
-                available = new Stack<HostBufferAllocation>();
-                _hostBufferPool.Add(key, available);
-            }
 
-            HostBufferAllocation allocation;
+            VulkanHostBufferAllocation allocation;
-            if (available.TryPop(out var pooled))
+            if (_hostBufferPool.TryRent(key, out var pooled))
             {
                 allocation = pooled;
             }
@@ -7980,15 +7970,16 @@ internal static unsafe class VulkanVideoPresenter
                 Check(
                     _vk.MapMemory(_device, allocatedMemory, 0, capacity, 0, &persistentMapping),
                     "vkMapMemory(host persistent)");
-                allocation = new HostBufferAllocation(
+                allocation = new VulkanHostBufferAllocation(
                     buffer,
                     allocatedMemory,
                     key,
                     (nint)persistentMapping);
-                _hostBufferAllocations.Add(buffer.Handle, allocation);
+                _hostBufferPool.Register(allocation);
             }
 
             memory = allocation.Memory;
+            mapped = allocation.Mapped;
             fixed (byte* source = data)
             {
                 System.Buffer.MemoryCopy(
@@ -8008,10 +7999,8 @@ internal static unsafe class VulkanVideoPresenter
                 return;
             }
 
-            if (_hostBufferAllocations.TryGetValue(buffer.Handle, out var allocation) &&
+            if (_hostBufferPool.Return(buffer, memory))
-                allocation.Memory.Handle == memory.Handle)
             {
-                _hostBufferPool[allocation.Key].Push(allocation);
                 return;
             }
 
@@ -8022,6 +8011,13 @@ internal static unsafe class VulkanVideoPresenter
             }
         }
 
+        private void DestroyHostBufferAllocation(VulkanHostBufferAllocation allocation)
+        {
+            _vk.UnmapMemory(_device, allocation.Memory);
+            _vk.DestroyBuffer(_device, allocation.Buffer, null);
+            _vk.FreeMemory(_device, allocation.Memory, null);
+        }
+
         private static PrimitiveTopology GetPrimitiveTopology(uint primitiveType) =>
             primitiveType switch
             {
@@ -13735,13 +13731,7 @@ internal static unsafe class VulkanVideoPresenter
                 DestroyGuestBufferAllocation(allocation);
             }
             _guestBufferAllocations.Clear();
-            foreach (var allocation in _hostBufferAllocations.Values)
+            _hostBufferPool.Dispose();
-            {
-                _vk.DestroyBuffer(_device, allocation.Buffer, null);
-                _vk.FreeMemory(_device, allocation.Memory, null);
-            }
-            _hostBufferAllocations.Clear();
-            _hostBufferPool.Clear();
             foreach (var guestImage in _guestImages.Values)
             {
                 DestroyGuestImage(guestImage);

tests/SharpEmu.Libs.Tests/VideoOut/VulkanHostBufferPoolTests.cs

@@ -0,0 +1,60 @@
+// Copyright (C) 2026 SharpEmu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+using SharpEmu.Libs.VideoOut;
+using Silk.NET.Vulkan;
+using VkBuffer = Silk.NET.Vulkan.Buffer;
+using Xunit;
+
+namespace SharpEmu.Libs.Tests.VideoOut;
+
+public sealed class VulkanHostBufferPoolTests
+{
+    [Fact]
+    public void ReturnedAllocationCanBeRentedAgain()
+    {
+        var destroyed = new List<VulkanHostBufferAllocation>();
+        using var pool = new VulkanHostBufferPool(1024, destroyed.Add);
+        var key = new VulkanHostBufferPoolKey(BufferUsageFlags.StorageBufferBit, 256);
+        var allocation = Allocation(1, 2, key);
+
+        pool.Register(allocation);
+        Assert.True(pool.Return(allocation.Buffer, allocation.Memory));
+        Assert.Equal(256UL, pool.CachedBytes);
+
+        Assert.True(pool.TryRent(key, out var rented));
+        Assert.Equal(allocation, rented);
+        Assert.Equal(0UL, pool.CachedBytes);
+        Assert.Empty(destroyed);
+    }
+
+    [Fact]
+    public void ReturnDestroysAllocationThatWouldExceedBudget()
+    {
+        var destroyed = new List<VulkanHostBufferAllocation>();
+        using var pool = new VulkanHostBufferPool(256, destroyed.Add);
+        var key = new VulkanHostBufferPoolKey(BufferUsageFlags.VertexBufferBit, 512);
+        var allocation = Allocation(3, 4, key);
+
+        pool.Register(allocation);
+
+        Assert.True(pool.Return(allocation.Buffer, allocation.Memory));
+        Assert.Equal(0UL, pool.CachedBytes);
+        Assert.Equal([allocation], destroyed);
+        Assert.False(pool.TryRent(key, out _));
+    }
+
+    [Fact]
+    public void UnknownAllocationIsNotClaimedByPool()
+    {
+        using var pool = new VulkanHostBufferPool(1024, _ => { });
+
+        Assert.False(pool.Return(new VkBuffer(9), new DeviceMemory(10)));
+    }
+
+    private static VulkanHostBufferAllocation Allocation(
+        ulong buffer,
+        ulong memory,
+        VulkanHostBufferPoolKey key) =>
+        new(new VkBuffer(buffer), new DeviceMemory(memory), key, 0);
+}