[Memory/Kernel] Fix Windows allocation-granularity and mutex-resolution bugs (#748)

- Fixed guest mappings now go through a granule-aware allocator so adjacent PS5 16 KiB pages sharing a 64 KiB Windows allocation granule no longer collide and fail. - TryBackFixedRange routes free/reserved gaps through the same granule-safe path, fixing strays that stranded the rest of a granule. - NORMAL pthread mutex self-relock reverted to real EDEADLK instead of silent compatibility recursion, which was starving other threads. - TryResolveMutexState now checks the handle-keyed lookup before falling through to "not found" on a fresh, never-cached address.
2026-08-08 10:45:52 +08:00 · 2026-08-02 23:15:42 +02:00
parent 4b5ea6a793
commit f36ce4084a
3 changed files with 493 additions and 37 deletions
@@ -26,6 +26,7 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA

    private long _mappingGeneration;
    private const ulong PageSize = 0x1000;
+    private const ulong HostAllocationGranularity = 0x10000;
    private const ulong GuestAllocationArenaAddress = 0x00006000_0000_0000;
    private const ulong GuestAllocationArenaSize = 0x0100_0000;
    private const ulong GuestAllocationArenaStartOffset = PageSize;
@@ -117,6 +118,9 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
    private const uint PAGE_READONLY = 0x02;

    private readonly IHostMemory _hostMemory;
+
+    private readonly object _fixedAllocationGate = new();
+    private readonly HashSet<ulong> _fixedGranuleReservationBases = new();
    private ulong _guestAllocationArenaBase;
    private readonly SortedDictionary<ulong, ulong> _guestAllocationFreeRanges = new();
    private readonly Dictionary<ulong, (ulong Offset, ulong Size)> _guestAllocations = new();
@@ -247,7 +251,12 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
        // reserve-only + lazy commit only when a huge non-exec commit fails —
        // that is the Poppy / large-reservation path #608 was aiming for.
        var reservedOnly = false;
-        var result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        var result = TryAllocateFixedThroughGranules(desiredAddress, alignedSize, hostProtection, traceReject: false);
+        if (result == 0)
+        {
+            result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        }
+
        if (result == 0 && allowLazyReserve)
        {
            result = _hostMemory.Reserve(desiredAddress, alignedSize, HostPageProtection.ReadWrite);
@@ -329,7 +338,16 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA

        // Prefer a full commit. Only fall back to reserve-only when a large
        // non-executable commit cannot be satisfied (see TryAllocateAtExact).
-        ulong result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        ulong result = 0;
+        if (desiredAddress != 0)
+        {
+            result = TryAllocateFixedThroughGranules(desiredAddress, alignedSize, hostProtection, traceReject: false);
+        }
+
+        if (result == 0)
+        {
+            result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        }

        if (result == 0)
        {
@@ -436,6 +454,183 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
        return $"fail:{primeBytes:X}";
    }

+    private ulong TryAllocateFixedThroughGranules(
+        ulong desiredAddress,
+        ulong alignedSize,
+        HostPageProtection hostProtection,
+        bool traceReject = true)
+    {
+        if (!OperatingSystem.IsWindows() || desiredAddress == 0 || alignedSize == 0)
+        {
+            return 0;
+        }
+
+        var requestStart = AlignDown(desiredAddress, PageSize);
+        ulong requestEnd;
+        ulong granuleEnd;
+        try
+        {
+            requestEnd = AlignUp(desiredAddress + alignedSize, PageSize);
+            granuleEnd = AlignUp(requestEnd, HostAllocationGranularity);
+        }
+        catch (OverflowException)
+        {
+            return 0;
+        }
+
+        var granuleStart = AlignDown(requestStart, HostAllocationGranularity);
+
+        lock (_fixedAllocationGate)
+        {
+            var newReservations = new List<ulong>();
+
+            void Reject(ulong segmentAddress, string reason)
+            {
+                if (traceReject)
+                {
+                    Log.Warn(
+                        $"fixed-alloc reject: want=0x{desiredAddress:X16}+0x{alignedSize:X} segment=0x{segmentAddress:X16} {reason}");
+                }
+                foreach (var reservationBase in newReservations)
+                {
+                    _hostMemory.Free(reservationBase);
+                    _fixedGranuleReservationBases.Remove(reservationBase);
+                }
+            }
+
+            var cursor = granuleStart;
+            while (cursor < granuleEnd)
+            {
+                if (!_hostMemory.Query(cursor, out var info))
+                {
+                    Reject(cursor, "query-failed");
+                    return 0;
+                }
+
+                var segmentEnd = info.RegionSize > ulong.MaxValue - info.BaseAddress
+                    ? ulong.MaxValue
+                    : info.BaseAddress + info.RegionSize;
+                segmentEnd = Math.Min(segmentEnd, granuleEnd);
+                if (segmentEnd <= cursor)
+                {
+                    Reject(cursor, "query-no-progress");
+                    return 0;
+                }
+
+                if (info.State == HostRegionState.Free)
+                {
+                    var alignedReserveBase = AlignUp(cursor, HostAllocationGranularity);
+                    var unreservableEnd = Math.Min(segmentEnd, alignedReserveBase);
+                    if (unreservableEnd > cursor && cursor < requestEnd && unreservableEnd > requestStart)
+                    {
+                        Reject(cursor, $"free-but-unreservable head (granule base 0x{AlignDown(cursor, HostAllocationGranularity):X16} owned elsewhere)");
+                        return 0;
+                    }
+
+                    if (alignedReserveBase < segmentEnd)
+                    {
+                        var reserved = _hostMemory.Reserve(alignedReserveBase, segmentEnd - alignedReserveBase, HostPageProtection.ReadWrite);
+                        if (reserved != alignedReserveBase)
+                        {
+                            if (reserved != 0)
+                            {
+                                _hostMemory.Free(reserved);
+                            }
+
+                            Reject(alignedReserveBase, "reserve-failed");
+                            return 0;
+                        }
+
+                        _fixedGranuleReservationBases.Add(alignedReserveBase);
+                        newReservations.Add(alignedReserveBase);
+                    }
+                }
+                else
+                {
+                    var trusted = _fixedGranuleReservationBases.Contains(info.AllocationBase) ||
+                        IsTrackedRegionBase(info.AllocationBase);
+                    if (!trusted && cursor < requestEnd && segmentEnd > requestStart)
+                    {
+                        Reject(cursor, $"foreign {info.State} allocBase=0x{info.AllocationBase:X16} prot=0x{info.RawProtection:X}");
+                        return 0;
+                    }
+                }
+
+                cursor = segmentEnd;
+            }
+
+            var commitCursor = requestStart;
+            while (commitCursor < requestEnd)
+            {
+                if (!_hostMemory.Query(commitCursor, out var info))
+                {
+                    Reject(commitCursor, "commit-query-failed");
+                    return 0;
+                }
+
+                var segmentEnd = info.RegionSize > ulong.MaxValue - info.BaseAddress
+                    ? ulong.MaxValue
+                    : info.BaseAddress + info.RegionSize;
+                segmentEnd = Math.Min(segmentEnd, requestEnd);
+                if (segmentEnd <= commitCursor)
+                {
+                    Reject(commitCursor, "commit-no-progress");
+                    return 0;
+                }
+
+                if (info.State != HostRegionState.Committed &&
+                    !_hostMemory.Commit(commitCursor, segmentEnd - commitCursor, hostProtection))
+                {
+                    Reject(commitCursor, "commit-failed");
+                    return 0;
+                }
+
+                commitCursor = segmentEnd;
+            }
+
+            if (newReservations.Count == 0)
+            {
+                TraceVmem($"Fixed alloc committed into existing granule reservations: 0x{desiredAddress:X16}+0x{alignedSize:X}");
+            }
+
+            return desiredAddress;
+        }
+    }
+
+    private bool IsTrackedRegionBase(ulong allocationBase)
+    {
+        _gate.EnterReadLock();
+        try
+        {
+            var low = 0;
+            var high = _regions.Count - 1;
+            while (low <= high)
+            {
+                var middle = low + ((high - low) >> 1);
+                var address = _regions[middle].VirtualAddress;
+                if (address == allocationBase)
+                {
+                    return true;
+                }
+
+                if (address < allocationBase)
+                {
+                    low = middle + 1;
+                }
+                else
+                {
+                    high = middle - 1;
+                }
+            }
+
+            return false;
+        }
+        finally
+        {
+            _gate.ExitReadLock();
+        }
+    }
+
    public bool TryBackFixedRange(ulong address, ulong size, bool executable)
    {
        if (size == 0)
@@ -463,7 +658,7 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
        // MemoryRegions are inserted only once every gap in the range has been
        // backed. If any gap fails to back, every earlier host allocation is freed
        // and no region is inserted, so the address space is left untouched.
-        var stagedAllocations = new List<(ulong Address, ulong Size)>();
+        var stagedAllocations = new List<(ulong Address, ulong Size, bool GranuleTracked)>();

        var cursor = start;
        while (cursor < end)
@@ -482,7 +677,21 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
                goto Rollback;
            }

-            if (info.State == HostRegionState.Free)
+            var needsGranuleAwareBacking = OperatingSystem.IsWindows() &&
+                (info.State == HostRegionState.Free || info.State == HostRegionState.Reserved);
+
+            if (needsGranuleAwareBacking)
+            {
+                var runSize = runEnd - cursor;
+                if (TryAllocateFixedThroughGranules(cursor, runSize, hostProtection, traceReject: false) != cursor)
+                {
+                    goto Rollback;
+                }
+
+                stagedAllocations.Add((cursor, runSize, true));
+                TraceVmem($"Backed fixed range gap: 0x{cursor:X16} - 0x{runEnd:X16} ({runSize} bytes)");
+            }
+            else if (info.State == HostRegionState.Free)
            {
                var runSize = runEnd - cursor;
                var allocated = _hostMemory.Allocate(cursor, runSize, hostProtection);
@@ -496,10 +705,11 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
                    goto Rollback;
                }

-                stagedAllocations.Add((cursor, runSize));
+                stagedAllocations.Add((cursor, runSize, false));
                TraceVmem($"Backed fixed range gap: 0x{cursor:X16} - 0x{runEnd:X16} ({runSize} bytes)");
            }

+
            cursor = runEnd;
        }

@@ -513,7 +723,7 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
        _gate.EnterWriteLock();
        try
        {
-            foreach (var (gapAddress, gapSize) in stagedAllocations)
+            foreach (var (gapAddress, gapSize, _) in stagedAllocations)
            {
                InsertRegionSorted(new MemoryRegion
                {
@@ -533,9 +743,12 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
        return true;

    Rollback:
-        foreach (var (gapAddress, _) in stagedAllocations)
+        foreach (var (gapAddress, _, granuleTracked) in stagedAllocations)
        {
-            _hostMemory.Free(gapAddress);
+            if (!granuleTracked)
+            {
+                _hostMemory.Free(gapAddress);
+            }
        }

        return false;
@@ -791,24 +1004,41 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
    {
        lock (_guestAllocationGate)
        {
-            _gate.EnterWriteLock();
-            try
+            lock (_fixedAllocationGate)
            {
-                foreach (var region in _regions)
+                _gate.EnterWriteLock();
+                try
                {
-                    _hostMemory.Free(region.VirtualAddress);
+                    var freedBases = new HashSet<ulong>();
+                    foreach (var region in _regions)
+                    {
+                        if (freedBases.Add(region.VirtualAddress))
+                        {
+                            _hostMemory.Free(region.VirtualAddress);
+                        }
+                    }
+
+                    foreach (var reservationBase in _fixedGranuleReservationBases)
+                    {
+                        if (freedBases.Add(reservationBase))
+                        {
+                            _hostMemory.Free(reservationBase);
+                        }
+                    }
+
+                    _fixedGranuleReservationBases.Clear();
+                    _regions.Clear();
+                    _pageProtections.Clear();
+                    lock (_allocationSearchHintGate)
+                    {
+                        _allocationSearchHints.Clear();
+                    }
+                    Interlocked.Increment(ref _mappingGeneration);
                }
-                _regions.Clear();
-                _pageProtections.Clear();
-                lock (_allocationSearchHintGate)
+                finally
                {
-                    _allocationSearchHints.Clear();
+                    _gate.ExitWriteLock();
                }
-                Interlocked.Increment(ref _mappingGeneration);
-            }
-            finally
-            {
-                _gate.ExitWriteLock();
            }

            _guestAllocationArenaBase = 0;
@@ -1402,6 +1632,42 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
            }
        }

+        if (OperatingSystem.IsWindows() && !region.IsReservedOnly)
+        {
+            var previous = low > 0 ? _regions[low - 1] : null;
+            var next = low < _regions.Count ? _regions[low] : null;
+            var mergePrevious = previous is not null &&
+                !previous.IsReservedOnly &&
+                previous.IsExecutable == region.IsExecutable &&
+                previous.Protection == region.Protection &&
+                previous.VirtualAddress + previous.Size == region.VirtualAddress;
+            var mergeNext = next is not null &&
+                !next.IsReservedOnly &&
+                next.IsExecutable == region.IsExecutable &&
+                next.Protection == region.Protection &&
+                region.VirtualAddress + region.Size == next.VirtualAddress;
+
+            if (mergePrevious && mergeNext)
+            {
+                previous!.Size += region.Size + next!.Size;
+                _regions.RemoveAt(low);
+                return;
+            }
+
+            if (mergePrevious)
+            {
+                previous!.Size += region.Size;
+                return;
+            }
+
+            if (mergeNext)
+            {
+                next!.VirtualAddress = region.VirtualAddress;
+                next.Size += region.Size;
+                return;
+            }
+        }
+
        _regions.Insert(low, region);
    }