[Memory/Kernel] Fix Windows allocation-granularity and mutex-resolution bugs (#748)

- Fixed guest mappings now go through a granule-aware allocator so adjacent PS5 16 KiB pages sharing a 64 KiB Windows allocation granule no longer collide and fail. - TryBackFixedRange routes free/reserved gaps through the same granule-safe path, fixing strays that stranded the rest of a granule. - NORMAL pthread mutex self-relock reverted to real EDEADLK instead of silent compatibility recursion, which was starving other threads. - TryResolveMutexState now checks the handle-keyed lookup before falling through to "not found" on a fresh, never-cached address.
2026-08-03 16:39:51 +08:00 · 2026-08-02 23:15:42 +02:00
parent 4b5ea6a793
commit f36ce4084a
3 changed files with 493 additions and 37 deletions
@@ -26,6 +26,7 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA

    private long _mappingGeneration;
    private const ulong PageSize = 0x1000;
+    private const ulong HostAllocationGranularity = 0x10000;
    private const ulong GuestAllocationArenaAddress = 0x00006000_0000_0000;
    private const ulong GuestAllocationArenaSize = 0x0100_0000;
    private const ulong GuestAllocationArenaStartOffset = PageSize;
@@ -117,6 +118,9 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
    private const uint PAGE_READONLY = 0x02;

    private readonly IHostMemory _hostMemory;
+
+    private readonly object _fixedAllocationGate = new();
+    private readonly HashSet<ulong> _fixedGranuleReservationBases = new();
    private ulong _guestAllocationArenaBase;
    private readonly SortedDictionary<ulong, ulong> _guestAllocationFreeRanges = new();
    private readonly Dictionary<ulong, (ulong Offset, ulong Size)> _guestAllocations = new();
@@ -247,7 +251,12 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
        // reserve-only + lazy commit only when a huge non-exec commit fails —
        // that is the Poppy / large-reservation path #608 was aiming for.
        var reservedOnly = false;
-        var result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        var result = TryAllocateFixedThroughGranules(desiredAddress, alignedSize, hostProtection, traceReject: false);
+        if (result == 0)
+        {
+            result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        }
+
        if (result == 0 && allowLazyReserve)
        {
            result = _hostMemory.Reserve(desiredAddress, alignedSize, HostPageProtection.ReadWrite);
@@ -329,7 +338,16 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA

        // Prefer a full commit. Only fall back to reserve-only when a large
        // non-executable commit cannot be satisfied (see TryAllocateAtExact).
-        ulong result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        ulong result = 0;
+        if (desiredAddress != 0)
+        {
+            result = TryAllocateFixedThroughGranules(desiredAddress, alignedSize, hostProtection, traceReject: false);
+        }
+
+        if (result == 0)
+        {
+            result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        }

        if (result == 0)
        {
@@ -436,6 +454,183 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
        return $"fail:{primeBytes:X}";
    }

+    private ulong TryAllocateFixedThroughGranules(
+        ulong desiredAddress,
+        ulong alignedSize,
+        HostPageProtection hostProtection,
+        bool traceReject = true)
+    {
+        if (!OperatingSystem.IsWindows() || desiredAddress == 0 || alignedSize == 0)
+        {
+            return 0;
+        }
+
+        var requestStart = AlignDown(desiredAddress, PageSize);
+        ulong requestEnd;
+        ulong granuleEnd;
+        try
+        {
+            requestEnd = AlignUp(desiredAddress + alignedSize, PageSize);
+            granuleEnd = AlignUp(requestEnd, HostAllocationGranularity);
+        }
+        catch (OverflowException)
+        {
+            return 0;
+        }
+
+        var granuleStart = AlignDown(requestStart, HostAllocationGranularity);
+
+        lock (_fixedAllocationGate)
+        {
+            var newReservations = new List<ulong>();
+
+            void Reject(ulong segmentAddress, string reason)
+            {
+                if (traceReject)
+                {
+                    Log.Warn(
+                        $"fixed-alloc reject: want=0x{desiredAddress:X16}+0x{alignedSize:X} segment=0x{segmentAddress:X16} {reason}");
+                }
+                foreach (var reservationBase in newReservations)
+                {
+                    _hostMemory.Free(reservationBase);
+                    _fixedGranuleReservationBases.Remove(reservationBase);
+                }
+            }
+
+            var cursor = granuleStart;
+            while (cursor < granuleEnd)
+            {
+                if (!_hostMemory.Query(cursor, out var info))
+                {
+                    Reject(cursor, "query-failed");
+                    return 0;
+                }
+
+                var segmentEnd = info.RegionSize > ulong.MaxValue - info.BaseAddress
+                    ? ulong.MaxValue
+                    : info.BaseAddress + info.RegionSize;
+                segmentEnd = Math.Min(segmentEnd, granuleEnd);
+                if (segmentEnd <= cursor)
+                {
+                    Reject(cursor, "query-no-progress");
+                    return 0;
+                }
+
+                if (info.State == HostRegionState.Free)
+                {
+                    var alignedReserveBase = AlignUp(cursor, HostAllocationGranularity);
+                    var unreservableEnd = Math.Min(segmentEnd, alignedReserveBase);
+                    if (unreservableEnd > cursor && cursor < requestEnd && unreservableEnd > requestStart)
+                    {
+                        Reject(cursor, $"free-but-unreservable head (granule base 0x{AlignDown(cursor, HostAllocationGranularity):X16} owned elsewhere)");
+                        return 0;
+                    }
+
+                    if (alignedReserveBase < segmentEnd)
+                    {
+                        var reserved = _hostMemory.Reserve(alignedReserveBase, segmentEnd - alignedReserveBase, HostPageProtection.ReadWrite);
+                        if (reserved != alignedReserveBase)
+                        {
+                            if (reserved != 0)
+                            {
+                                _hostMemory.Free(reserved);
+                            }
+
+                            Reject(alignedReserveBase, "reserve-failed");
+                            return 0;
+                        }
+
+                        _fixedGranuleReservationBases.Add(alignedReserveBase);
+                        newReservations.Add(alignedReserveBase);
+                    }
+                }
+                else
+                {
+                    var trusted = _fixedGranuleReservationBases.Contains(info.AllocationBase) ||
+                        IsTrackedRegionBase(info.AllocationBase);
+                    if (!trusted && cursor < requestEnd && segmentEnd > requestStart)
+                    {
+                        Reject(cursor, $"foreign {info.State} allocBase=0x{info.AllocationBase:X16} prot=0x{info.RawProtection:X}");
+                        return 0;
+                    }
+                }
+
+                cursor = segmentEnd;
+            }
+
+            var commitCursor = requestStart;
+            while (commitCursor < requestEnd)
+            {
+                if (!_hostMemory.Query(commitCursor, out var info))
+                {
+                    Reject(commitCursor, "commit-query-failed");
+                    return 0;
+                }
+
+                var segmentEnd = info.RegionSize > ulong.MaxValue - info.BaseAddress
+                    ? ulong.MaxValue
+                    : info.BaseAddress + info.RegionSize;
+                segmentEnd = Math.Min(segmentEnd, requestEnd);
+                if (segmentEnd <= commitCursor)
+                {
+                    Reject(commitCursor, "commit-no-progress");
+                    return 0;
+                }
+
+                if (info.State != HostRegionState.Committed &&
+                    !_hostMemory.Commit(commitCursor, segmentEnd - commitCursor, hostProtection))
+                {
+                    Reject(commitCursor, "commit-failed");
+                    return 0;
+                }
+
+                commitCursor = segmentEnd;
+            }
+
+            if (newReservations.Count == 0)
+            {
+                TraceVmem($"Fixed alloc committed into existing granule reservations: 0x{desiredAddress:X16}+0x{alignedSize:X}");
+            }
+
+            return desiredAddress;
+        }
+    }
+
+    private bool IsTrackedRegionBase(ulong allocationBase)
+    {
+        _gate.EnterReadLock();
+        try
+        {
+            var low = 0;
+            var high = _regions.Count - 1;
+            while (low <= high)
+            {
+                var middle = low + ((high - low) >> 1);
+                var address = _regions[middle].VirtualAddress;
+                if (address == allocationBase)
+                {
+                    return true;
+                }
+
+                if (address < allocationBase)
+                {
+                    low = middle + 1;
+                }
+                else
+                {
+                    high = middle - 1;
+                }
+            }
+
+            return false;
+        }
+        finally
+        {
+            _gate.ExitReadLock();
+        }
+    }
+
    public bool TryBackFixedRange(ulong address, ulong size, bool executable)
    {
        if (size == 0)
@@ -463,7 +658,7 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
        // MemoryRegions are inserted only once every gap in the range has been
        // backed. If any gap fails to back, every earlier host allocation is freed
        // and no region is inserted, so the address space is left untouched.
-        var stagedAllocations = new List<(ulong Address, ulong Size)>();
+        var stagedAllocations = new List<(ulong Address, ulong Size, bool GranuleTracked)>();

        var cursor = start;
        while (cursor < end)
@@ -482,7 +677,21 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
                goto Rollback;
            }

-            if (info.State == HostRegionState.Free)
+            var needsGranuleAwareBacking = OperatingSystem.IsWindows() &&
+                (info.State == HostRegionState.Free || info.State == HostRegionState.Reserved);
+
+            if (needsGranuleAwareBacking)
+            {
+                var runSize = runEnd - cursor;
+                if (TryAllocateFixedThroughGranules(cursor, runSize, hostProtection, traceReject: false) != cursor)
+                {
+                    goto Rollback;
+                }
+
+                stagedAllocations.Add((cursor, runSize, true));
+                TraceVmem($"Backed fixed range gap: 0x{cursor:X16} - 0x{runEnd:X16} ({runSize} bytes)");
+            }
+            else if (info.State == HostRegionState.Free)
            {
                var runSize = runEnd - cursor;
                var allocated = _hostMemory.Allocate(cursor, runSize, hostProtection);
@@ -496,10 +705,11 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
                    goto Rollback;
                }

-                stagedAllocations.Add((cursor, runSize));
+                stagedAllocations.Add((cursor, runSize, false));
                TraceVmem($"Backed fixed range gap: 0x{cursor:X16} - 0x{runEnd:X16} ({runSize} bytes)");
            }

+
            cursor = runEnd;
        }

@@ -513,7 +723,7 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
        _gate.EnterWriteLock();
        try
        {
-            foreach (var (gapAddress, gapSize) in stagedAllocations)
+            foreach (var (gapAddress, gapSize, _) in stagedAllocations)
            {
                InsertRegionSorted(new MemoryRegion
                {
@@ -533,9 +743,12 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
        return true;

    Rollback:
-        foreach (var (gapAddress, _) in stagedAllocations)
+        foreach (var (gapAddress, _, granuleTracked) in stagedAllocations)
        {
-            _hostMemory.Free(gapAddress);
+            if (!granuleTracked)
+            {
+                _hostMemory.Free(gapAddress);
+            }
        }

        return false;
@@ -791,24 +1004,41 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
    {
        lock (_guestAllocationGate)
        {
-            _gate.EnterWriteLock();
-            try
+            lock (_fixedAllocationGate)
            {
-                foreach (var region in _regions)
+                _gate.EnterWriteLock();
+                try
                {
-                    _hostMemory.Free(region.VirtualAddress);
+                    var freedBases = new HashSet<ulong>();
+                    foreach (var region in _regions)
+                    {
+                        if (freedBases.Add(region.VirtualAddress))
+                        {
+                            _hostMemory.Free(region.VirtualAddress);
+                        }
+                    }
+
+                    foreach (var reservationBase in _fixedGranuleReservationBases)
+                    {
+                        if (freedBases.Add(reservationBase))
+                        {
+                            _hostMemory.Free(reservationBase);
+                        }
+                    }
+
+                    _fixedGranuleReservationBases.Clear();
+                    _regions.Clear();
+                    _pageProtections.Clear();
+                    lock (_allocationSearchHintGate)
+                    {
+                        _allocationSearchHints.Clear();
+                    }
+                    Interlocked.Increment(ref _mappingGeneration);
                }
-                _regions.Clear();
-                _pageProtections.Clear();
-                lock (_allocationSearchHintGate)
+                finally
                {
-                    _allocationSearchHints.Clear();
+                    _gate.ExitWriteLock();
                }
-                Interlocked.Increment(ref _mappingGeneration);
-            }
-            finally
-            {
-                _gate.ExitWriteLock();
            }

            _guestAllocationArenaBase = 0;
@@ -1402,6 +1632,42 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
            }
        }

+        if (OperatingSystem.IsWindows() && !region.IsReservedOnly)
+        {
+            var previous = low > 0 ? _regions[low - 1] : null;
+            var next = low < _regions.Count ? _regions[low] : null;
+            var mergePrevious = previous is not null &&
+                !previous.IsReservedOnly &&
+                previous.IsExecutable == region.IsExecutable &&
+                previous.Protection == region.Protection &&
+                previous.VirtualAddress + previous.Size == region.VirtualAddress;
+            var mergeNext = next is not null &&
+                !next.IsReservedOnly &&
+                next.IsExecutable == region.IsExecutable &&
+                next.Protection == region.Protection &&
+                region.VirtualAddress + region.Size == next.VirtualAddress;
+
+            if (mergePrevious && mergeNext)
+            {
+                previous!.Size += region.Size + next!.Size;
+                _regions.RemoveAt(low);
+                return;
+            }
+
+            if (mergePrevious)
+            {
+                previous!.Size += region.Size;
+                return;
+            }
+
+            if (mergeNext)
+            {
+                next!.VirtualAddress = region.VirtualAddress;
+                next.Size += region.Size;
+                return;
+            }
+        }
+
        _regions.Insert(low, region);
    }

@@ -918,15 +918,8 @@ public static class KernelPthreadCompatExports
                        return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY;
                    }

-                    // Several Gen5 runtimes layer their own owner/count bookkeeping
-                    // over a NORMAL kernel mutex. Returning EDEADLK here
-                    // leaves that guest bookkeeping out of sync with the HLE owner and
-                    // turns the wrapper into a permanent lock/unlock retry loop. Keep
-                    // the compatibility recursion used by the original implementation;
-                    // ERRORCHECK mutexes still take the strict EDEADLK path below.
-                    state.RecursionCount++;
-                    TracePthreadMutex(ctx, "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK);
-                    return (int)OrbisGen2Result.ORBIS_GEN2_OK;
+                    TracePthreadMutex(ctx, "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DEADLOCK);
+                    return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DEADLOCK;
                }
                else
                {
@@ -1264,15 +1257,15 @@ public static class KernelPthreadCompatExports
            return CreateImplicitMutexState(ctx, mutexAddress, MutexTypeAdaptiveNp, out resolvedAddress, out state);
        }

+        if (pointedHandle != 0 && pointedHandle != mutexAddress && _mutexStates.TryGetValue(pointedHandle, out state))
+        {
+            _mutexStates[mutexAddress] = state;
+            resolvedAddress = pointedHandle;
+            return true;
+        }
+
        if (pointedHandle != 0)
        {
-            if (_mutexStates.TryGetValue(pointedHandle, out state))
-            {
-                _mutexStates.TryAdd(mutexAddress, state);
-                resolvedAddress = pointedHandle;
-                return true;
-            }
-
            resolvedAddress = pointedHandle;
            return false;
        }
@@ -104,6 +104,45 @@ public sealed class GuestMemoryAllocatorTests
        Assert.Equal(0UL, (ulong)memory.GetPointer(address));
    }

+    [Fact]
+    public void AdjacentFixedGuestPageMappingsShareAHostGranule()
+    {
+        if (!OperatingSystem.IsWindows())
+        {
+            return;
+        }
+
+        using var memory = new PhysicalVirtualMemory(new GranularityAwareHostMemory());
+        const ulong baseAddress = 0x0000008001600000;
+
+        Assert.Equal(baseAddress, memory.AllocateAt(baseAddress, 0x4000, executable: false, allowAlternative: false));
+        Assert.Equal(
+            baseAddress + 0x4000,
+            memory.AllocateAt(baseAddress + 0x4000, 0x4000, executable: false, allowAlternative: false));
+        Assert.Equal(
+            baseAddress + 0x8000,
+            memory.AllocateAt(baseAddress + 0x8000, 0x8000, executable: false, allowAlternative: false));
+
+        Assert.True(memory.IsAccessible(baseAddress, 0x10000));
+    }
+
+    [Fact]
+    public void TryBackFixedRangeSharesAHostGranuleAcrossCallsOnWindows()
+    {
+        if (!OperatingSystem.IsWindows())
+        {
+            return;
+        }
+
+        using var memory = new PhysicalVirtualMemory(new GranularityAwareHostMemory());
+        const ulong baseAddress = 0x0000008001600000;
+
+        Assert.True(memory.TryBackFixedRange(baseAddress, 0x4000, executable: false));
+        Assert.True(memory.TryBackFixedRange(baseAddress + 0x4000, 0x4000, executable: false));
+
+        Assert.True(memory.IsAccessible(baseAddress, 0x8000));
+    }
+
    [Fact]
    public void AlignedAllocationDoesNotRetainOverallocatedMappingsOutsideMacOS()
    {
@@ -133,6 +172,11 @@ public sealed class GuestMemoryAllocatorTests
    [Fact]
    public void TryBackFixedRangeRollsBackEarlierGapsWhenLaterGapCannotBeBacked()
    {
+        if (OperatingSystem.IsWindows())
+        {
+            return;
+        }
+
        // Layout: committed | free | committed | free
        // First free gap allocates successfully, second fails.
        // The first allocation must be freed — nothing should leak.
@@ -154,6 +198,11 @@ public sealed class GuestMemoryAllocatorTests
    [Fact]
    public void TryBackFixedRangeFillsOnlyTheFreePagesOfAPartiallyOccupiedRange()
    {
+        if (OperatingSystem.IsWindows())
+        {
+            return;
+        }
+
        const ulong rangeBase = 0x0000_0020_2F00_0000;
        const ulong rangeSize = 0x40_0000;
        const ulong occupiedSize = 0x4_0000;
@@ -519,6 +568,154 @@ public sealed class GuestMemoryAllocatorTests
        }
    }

+    private sealed class GranularityAwareHostMemory : IHostMemory
+    {
+        private const ulong Granularity = 0x10000;
+        private const ulong Page = 0x1000;
+
+        private readonly SortedDictionary<ulong, (ulong Size, SortedSet<ulong> CommittedPages)> _allocations = new();
+
+        public ulong Allocate(ulong desiredAddress, ulong size, HostPageProtection protection)
+        {
+            var reservedBase = Reserve(desiredAddress, size, protection);
+            if (reservedBase != 0)
+            {
+                var start = desiredAddress == 0 ? reservedBase : AlignDown(desiredAddress, Page);
+                Commit(start, size, protection);
+            }
+
+            return reservedBase;
+        }
+
+        public ulong Reserve(ulong desiredAddress, ulong size, HostPageProtection protection)
+        {
+            if (desiredAddress == 0)
+            {
+                return 0;
+            }
+
+            var allocationBase = AlignDown(desiredAddress, Granularity);
+            var end = AlignUp(desiredAddress + size, Page);
+            foreach (var (existingBase, existing) in _allocations)
+            {
+                if (allocationBase < existingBase + existing.Size && existingBase < end)
+                {
+                    return 0;
+                }
+            }
+
+            _allocations[allocationBase] = (end - allocationBase, new SortedSet<ulong>());
+            return allocationBase;
+        }
+
+        public bool Commit(ulong address, ulong size, HostPageProtection protection)
+        {
+            var start = AlignDown(address, Page);
+            var end = AlignUp(address + size, Page);
+            if (!TryFindAllocation(start, out var allocationBase, out var allocation) ||
+                end > allocationBase + allocation.Size)
+            {
+                return false;
+            }
+
+            for (var page = start; page < end; page += Page)
+            {
+                allocation.CommittedPages.Add(page);
+            }
+
+            return true;
+        }
+
+        public bool Free(ulong address) => _allocations.Remove(address);
+
+        public bool Protect(ulong address, ulong size, HostPageProtection protection, out uint rawOldProtection)
+        {
+            rawOldProtection = 0;
+            return true;
+        }
+
+        public bool ProtectRaw(ulong address, ulong size, uint rawProtection, out uint rawOldProtection)
+        {
+            rawOldProtection = 0;
+            return true;
+        }
+
+        public bool Query(ulong address, out HostRegionInfo info)
+        {
+            var page = AlignDown(address, Page);
+            if (TryFindAllocation(page, out var allocationBase, out var allocation))
+            {
+                var committed = allocation.CommittedPages.Contains(page);
+                var runEnd = page + Page;
+                while (runEnd < allocationBase + allocation.Size &&
+                       allocation.CommittedPages.Contains(runEnd) == committed)
+                {
+                    runEnd += Page;
+                }
+
+                info = new HostRegionInfo(
+                    page,
+                    allocationBase,
+                    runEnd - page,
+                    committed ? HostRegionState.Committed : HostRegionState.Reserved,
+                    0,
+                    committed ? HostPageProtection.ReadWrite : HostPageProtection.NoAccess,
+                    0,
+                    0);
+                return true;
+            }
+
+            var freeEnd = ulong.MaxValue;
+            foreach (var existingBase in _allocations.Keys)
+            {
+                if (existingBase > page)
+                {
+                    freeEnd = existingBase;
+                    break;
+                }
+            }
+
+            info = new HostRegionInfo(
+                page,
+                0,
+                freeEnd - page,
+                HostRegionState.Free,
+                0,
+                HostPageProtection.NoAccess,
+                0,
+                0);
+            return true;
+        }
+
+        public void FlushInstructionCache(ulong address, ulong size)
+        {
+        }
+
+        private bool TryFindAllocation(
+            ulong address,
+            out ulong allocationBase,
+            out (ulong Size, SortedSet<ulong> CommittedPages) allocation)
+        {
+            foreach (var (existingBase, existing) in _allocations)
+            {
+                if (address >= existingBase && address < existingBase + existing.Size)
+                {
+                    allocationBase = existingBase;
+                    allocation = existing;
+                    return true;
+                }
+            }
+
+            allocationBase = 0;
+            allocation = default;
+            return false;
+        }
+
+        private static ulong AlignDown(ulong value, ulong alignment) => value & ~(alignment - 1);
+
+        private static ulong AlignUp(ulong value, ulong alignment) => (value + alignment - 1) & ~(alignment - 1);
+    }
+
    private sealed class LazyHostMemory(ulong address) : IHostMemory, IDisposable
    {
        public bool CommitSucceeds { get; set; } = true;