[Memory/Kernel] Fix Windows allocation-granularity and mutex-resolution bugs (#748)

- Fixed guest mappings now go through a granule-aware allocator so
  adjacent PS5 16 KiB pages sharing a 64 KiB Windows allocation
  granule no longer collide and fail.
- TryBackFixedRange routes free/reserved gaps through the same
  granule-safe path, fixing strays that stranded the rest of a granule.
- NORMAL pthread mutex self-relock reverted to real EDEADLK instead of
  silent compatibility recursion, which was starving other threads.
- TryResolveMutexState now checks the handle-keyed lookup before
  falling through to "not found" on a fresh, never-cached address.

src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs

@@ -26,6 +26,7 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
 
     private long _mappingGeneration;
     private const ulong PageSize = 0x1000;
+    private const ulong HostAllocationGranularity = 0x10000;
     private const ulong GuestAllocationArenaAddress = 0x00006000_0000_0000;
     private const ulong GuestAllocationArenaSize = 0x0100_0000;
     private const ulong GuestAllocationArenaStartOffset = PageSize;
@@ -117,6 +118,9 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
     private const uint PAGE_READONLY = 0x02;
 
     private readonly IHostMemory _hostMemory;
+
+    private readonly object _fixedAllocationGate = new();
+    private readonly HashSet<ulong> _fixedGranuleReservationBases = new();
     private ulong _guestAllocationArenaBase;
     private readonly SortedDictionary<ulong, ulong> _guestAllocationFreeRanges = new();
     private readonly Dictionary<ulong, (ulong Offset, ulong Size)> _guestAllocations = new();
@@ -247,7 +251,12 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
         // reserve-only + lazy commit only when a huge non-exec commit fails —
         // that is the Poppy / large-reservation path #608 was aiming for.
         var reservedOnly = false;
-        var result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        var result = TryAllocateFixedThroughGranules(desiredAddress, alignedSize, hostProtection, traceReject: false);
+        if (result == 0)
+        {
+            result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        }
+
         if (result == 0 && allowLazyReserve)
         {
             result = _hostMemory.Reserve(desiredAddress, alignedSize, HostPageProtection.ReadWrite);
@@ -329,7 +338,16 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
 
         // Prefer a full commit. Only fall back to reserve-only when a large
         // non-executable commit cannot be satisfied (see TryAllocateAtExact).
-        ulong result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        ulong result = 0;
+        if (desiredAddress != 0)
+        {
+            result = TryAllocateFixedThroughGranules(desiredAddress, alignedSize, hostProtection, traceReject: false);
+        }
+
+        if (result == 0)
+        {
+            result = _hostMemory.Allocate(desiredAddress, alignedSize, hostProtection);
+        }
 
         if (result == 0)
         {
@@ -436,6 +454,183 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
         return $"fail:{primeBytes:X}";
     }
 
+    private ulong TryAllocateFixedThroughGranules(
+        ulong desiredAddress,
+        ulong alignedSize,
+        HostPageProtection hostProtection,
+        bool traceReject = true)
+    {
+        if (!OperatingSystem.IsWindows() || desiredAddress == 0 || alignedSize == 0)
+        {
+            return 0;
+        }
+
+        var requestStart = AlignDown(desiredAddress, PageSize);
+        ulong requestEnd;
+        ulong granuleEnd;
+        try
+        {
+            requestEnd = AlignUp(desiredAddress + alignedSize, PageSize);
+            granuleEnd = AlignUp(requestEnd, HostAllocationGranularity);
+        }
+        catch (OverflowException)
+        {
+            return 0;
+        }
+
+        var granuleStart = AlignDown(requestStart, HostAllocationGranularity);
+
+        lock (_fixedAllocationGate)
+        {
+            var newReservations = new List<ulong>();
+
+            void Reject(ulong segmentAddress, string reason)
+            {
+                if (traceReject)
+                {
+                    Log.Warn(
+                        $"fixed-alloc reject: want=0x{desiredAddress:X16}+0x{alignedSize:X} segment=0x{segmentAddress:X16} {reason}");
+                }
+                foreach (var reservationBase in newReservations)
+                {
+                    _hostMemory.Free(reservationBase);
+                    _fixedGranuleReservationBases.Remove(reservationBase);
+                }
+            }
+
+            var cursor = granuleStart;
+            while (cursor < granuleEnd)
+            {
+                if (!_hostMemory.Query(cursor, out var info))
+                {
+                    Reject(cursor, "query-failed");
+                    return 0;
+                }
+
+                var segmentEnd = info.RegionSize > ulong.MaxValue - info.BaseAddress
+                    ? ulong.MaxValue
+                    : info.BaseAddress + info.RegionSize;
+                segmentEnd = Math.Min(segmentEnd, granuleEnd);
+                if (segmentEnd <= cursor)
+                {
+                    Reject(cursor, "query-no-progress");
+                    return 0;
+                }
+
+                if (info.State == HostRegionState.Free)
+                {
+                    var alignedReserveBase = AlignUp(cursor, HostAllocationGranularity);
+                    var unreservableEnd = Math.Min(segmentEnd, alignedReserveBase);
+                    if (unreservableEnd > cursor && cursor < requestEnd && unreservableEnd > requestStart)
+                    {
+                        Reject(cursor, $"free-but-unreservable head (granule base 0x{AlignDown(cursor, HostAllocationGranularity):X16} owned elsewhere)");
+                        return 0;
+                    }
+
+                    if (alignedReserveBase < segmentEnd)
+                    {
+                        var reserved = _hostMemory.Reserve(alignedReserveBase, segmentEnd - alignedReserveBase, HostPageProtection.ReadWrite);
+                        if (reserved != alignedReserveBase)
+                        {
+                            if (reserved != 0)
+                            {
+                                _hostMemory.Free(reserved);
+                            }
+
+                            Reject(alignedReserveBase, "reserve-failed");
+                            return 0;
+                        }
+
+                        _fixedGranuleReservationBases.Add(alignedReserveBase);
+                        newReservations.Add(alignedReserveBase);
+                    }
+                }
+                else
+                {
+                    var trusted = _fixedGranuleReservationBases.Contains(info.AllocationBase) ||
+                        IsTrackedRegionBase(info.AllocationBase);
+                    if (!trusted && cursor < requestEnd && segmentEnd > requestStart)
+                    {
+                        Reject(cursor, $"foreign {info.State} allocBase=0x{info.AllocationBase:X16} prot=0x{info.RawProtection:X}");
+                        return 0;
+                    }
+                }
+
+                cursor = segmentEnd;
+            }
+
+            var commitCursor = requestStart;
+            while (commitCursor < requestEnd)
+            {
+                if (!_hostMemory.Query(commitCursor, out var info))
+                {
+                    Reject(commitCursor, "commit-query-failed");
+                    return 0;
+                }
+
+                var segmentEnd = info.RegionSize > ulong.MaxValue - info.BaseAddress
+                    ? ulong.MaxValue
+                    : info.BaseAddress + info.RegionSize;
+                segmentEnd = Math.Min(segmentEnd, requestEnd);
+                if (segmentEnd <= commitCursor)
+                {
+                    Reject(commitCursor, "commit-no-progress");
+                    return 0;
+                }
+
+                if (info.State != HostRegionState.Committed &&
+                    !_hostMemory.Commit(commitCursor, segmentEnd - commitCursor, hostProtection))
+                {
+                    Reject(commitCursor, "commit-failed");
+                    return 0;
+                }
+
+                commitCursor = segmentEnd;
+            }
+
+            if (newReservations.Count == 0)
+            {
+                TraceVmem($"Fixed alloc committed into existing granule reservations: 0x{desiredAddress:X16}+0x{alignedSize:X}");
+            }
+
+            return desiredAddress;
+        }
+    }
+
+    private bool IsTrackedRegionBase(ulong allocationBase)
+    {
+        _gate.EnterReadLock();
+        try
+        {
+            var low = 0;
+            var high = _regions.Count - 1;
+            while (low <= high)
+            {
+                var middle = low + ((high - low) >> 1);
+                var address = _regions[middle].VirtualAddress;
+                if (address == allocationBase)
+                {
+                    return true;
+                }
+
+                if (address < allocationBase)
+                {
+                    low = middle + 1;
+                }
+                else
+                {
+                    high = middle - 1;
+                }
+            }
+
+            return false;
+        }
+        finally
+        {
+            _gate.ExitReadLock();
+        }
+    }
+
     public bool TryBackFixedRange(ulong address, ulong size, bool executable)
     {
         if (size == 0)
@@ -463,7 +658,7 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
         // MemoryRegions are inserted only once every gap in the range has been
         // backed. If any gap fails to back, every earlier host allocation is freed
         // and no region is inserted, so the address space is left untouched.
-        var stagedAllocations = new List<(ulong Address, ulong Size)>();
+        var stagedAllocations = new List<(ulong Address, ulong Size, bool GranuleTracked)>();
 
         var cursor = start;
         while (cursor < end)
@@ -482,7 +677,21 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
                 goto Rollback;
             }
 
-            if (info.State == HostRegionState.Free)
+            var needsGranuleAwareBacking = OperatingSystem.IsWindows() &&
+                (info.State == HostRegionState.Free || info.State == HostRegionState.Reserved);
+
+            if (needsGranuleAwareBacking)
+            {
+                var runSize = runEnd - cursor;
+                if (TryAllocateFixedThroughGranules(cursor, runSize, hostProtection, traceReject: false) != cursor)
+                {
+                    goto Rollback;
+                }
+
+                stagedAllocations.Add((cursor, runSize, true));
+                TraceVmem($"Backed fixed range gap: 0x{cursor:X16} - 0x{runEnd:X16} ({runSize} bytes)");
+            }
+            else if (info.State == HostRegionState.Free)
             {
                 var runSize = runEnd - cursor;
                 var allocated = _hostMemory.Allocate(cursor, runSize, hostProtection);
@@ -496,10 +705,11 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
                     goto Rollback;
                 }
 
-                stagedAllocations.Add((cursor, runSize));
+                stagedAllocations.Add((cursor, runSize, false));
                 TraceVmem($"Backed fixed range gap: 0x{cursor:X16} - 0x{runEnd:X16} ({runSize} bytes)");
             }
 
+
             cursor = runEnd;
         }
 
@@ -513,7 +723,7 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
         _gate.EnterWriteLock();
         try
         {
-            foreach (var (gapAddress, gapSize) in stagedAllocations)
+            foreach (var (gapAddress, gapSize, _) in stagedAllocations)
             {
                 InsertRegionSorted(new MemoryRegion
                 {
@@ -533,9 +743,12 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
         return true;
 
     Rollback:
-        foreach (var (gapAddress, _) in stagedAllocations)
+        foreach (var (gapAddress, _, granuleTracked) in stagedAllocations)
         {
-            _hostMemory.Free(gapAddress);
+            if (!granuleTracked)
+            {
+                _hostMemory.Free(gapAddress);
+            }
         }
 
         return false;
@@ -791,24 +1004,41 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
     {
         lock (_guestAllocationGate)
         {
-            _gate.EnterWriteLock();
+            lock (_fixedAllocationGate)
-            try
             {
-                foreach (var region in _regions)
+                _gate.EnterWriteLock();
+                try
                 {
-                    _hostMemory.Free(region.VirtualAddress);
+                    var freedBases = new HashSet<ulong>();
+                    foreach (var region in _regions)
+                    {
+                        if (freedBases.Add(region.VirtualAddress))
+                        {
+                            _hostMemory.Free(region.VirtualAddress);
+                        }
+                    }
+
+                    foreach (var reservationBase in _fixedGranuleReservationBases)
+                    {
+                        if (freedBases.Add(reservationBase))
+                        {
+                            _hostMemory.Free(reservationBase);
+                        }
+                    }
+
+                    _fixedGranuleReservationBases.Clear();
+                    _regions.Clear();
+                    _pageProtections.Clear();
+                    lock (_allocationSearchHintGate)
+                    {
+                        _allocationSearchHints.Clear();
+                    }
+                    Interlocked.Increment(ref _mappingGeneration);
                 }
-                _regions.Clear();
+                finally
-                _pageProtections.Clear();
-                lock (_allocationSearchHintGate)
                 {
-                    _allocationSearchHints.Clear();
+                    _gate.ExitWriteLock();
                 }
-                Interlocked.Increment(ref _mappingGeneration);
-            }
-            finally
-            {
-                _gate.ExitWriteLock();
             }
 
             _guestAllocationArenaBase = 0;
@@ -1402,6 +1632,42 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA
             }
         }
 
+        if (OperatingSystem.IsWindows() && !region.IsReservedOnly)
+        {
+            var previous = low > 0 ? _regions[low - 1] : null;
+            var next = low < _regions.Count ? _regions[low] : null;
+            var mergePrevious = previous is not null &&
+                !previous.IsReservedOnly &&
+                previous.IsExecutable == region.IsExecutable &&
+                previous.Protection == region.Protection &&
+                previous.VirtualAddress + previous.Size == region.VirtualAddress;
+            var mergeNext = next is not null &&
+                !next.IsReservedOnly &&
+                next.IsExecutable == region.IsExecutable &&
+                next.Protection == region.Protection &&
+                region.VirtualAddress + region.Size == next.VirtualAddress;
+
+            if (mergePrevious && mergeNext)
+            {
+                previous!.Size += region.Size + next!.Size;
+                _regions.RemoveAt(low);
+                return;
+            }
+
+            if (mergePrevious)
+            {
+                previous!.Size += region.Size;
+                return;
+            }
+
+            if (mergeNext)
+            {
+                next!.VirtualAddress = region.VirtualAddress;
+                next.Size += region.Size;
+                return;
+            }
+        }
+
         _regions.Insert(low, region);
     }
 

src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs

@@ -918,15 +918,8 @@ public static class KernelPthreadCompatExports
                         return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY;
                     }
 
-                    // Several Gen5 runtimes layer their own owner/count bookkeeping
+                    TracePthreadMutex(ctx, "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DEADLOCK);
-                    // over a NORMAL kernel mutex. Returning EDEADLK here
+                    return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DEADLOCK;
-                    // leaves that guest bookkeeping out of sync with the HLE owner and
-                    // turns the wrapper into a permanent lock/unlock retry loop. Keep
-                    // the compatibility recursion used by the original implementation;
-                    // ERRORCHECK mutexes still take the strict EDEADLK path below.
-                    state.RecursionCount++;
-                    TracePthreadMutex(ctx, "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK);
-                    return (int)OrbisGen2Result.ORBIS_GEN2_OK;
                 }
                 else
                 {
@@ -1264,15 +1257,15 @@ public static class KernelPthreadCompatExports
             return CreateImplicitMutexState(ctx, mutexAddress, MutexTypeAdaptiveNp, out resolvedAddress, out state);
         }
 
+        if (pointedHandle != 0 && pointedHandle != mutexAddress && _mutexStates.TryGetValue(pointedHandle, out state))
+        {
+            _mutexStates[mutexAddress] = state;
+            resolvedAddress = pointedHandle;
+            return true;
+        }
+
         if (pointedHandle != 0)
         {
-            if (_mutexStates.TryGetValue(pointedHandle, out state))
-            {
-                _mutexStates.TryAdd(mutexAddress, state);
-                resolvedAddress = pointedHandle;
-                return true;
-            }
-
             resolvedAddress = pointedHandle;
             return false;
         }

tests/SharpEmu.Libs.Tests/Memory/GuestMemoryAllocatorTests.cs

@@ -104,6 +104,45 @@ public sealed class GuestMemoryAllocatorTests
         Assert.Equal(0UL, (ulong)memory.GetPointer(address));
     }
 
+    [Fact]
+    public void AdjacentFixedGuestPageMappingsShareAHostGranule()
+    {
+        if (!OperatingSystem.IsWindows())
+        {
+            return;
+        }
+
+        using var memory = new PhysicalVirtualMemory(new GranularityAwareHostMemory());
+        const ulong baseAddress = 0x0000008001600000;
+
+        Assert.Equal(baseAddress, memory.AllocateAt(baseAddress, 0x4000, executable: false, allowAlternative: false));
+        Assert.Equal(
+            baseAddress + 0x4000,
+            memory.AllocateAt(baseAddress + 0x4000, 0x4000, executable: false, allowAlternative: false));
+        Assert.Equal(
+            baseAddress + 0x8000,
+            memory.AllocateAt(baseAddress + 0x8000, 0x8000, executable: false, allowAlternative: false));
+
+        Assert.True(memory.IsAccessible(baseAddress, 0x10000));
+    }
+
+    [Fact]
+    public void TryBackFixedRangeSharesAHostGranuleAcrossCallsOnWindows()
+    {
+        if (!OperatingSystem.IsWindows())
+        {
+            return;
+        }
+
+        using var memory = new PhysicalVirtualMemory(new GranularityAwareHostMemory());
+        const ulong baseAddress = 0x0000008001600000;
+
+        Assert.True(memory.TryBackFixedRange(baseAddress, 0x4000, executable: false));
+        Assert.True(memory.TryBackFixedRange(baseAddress + 0x4000, 0x4000, executable: false));
+
+        Assert.True(memory.IsAccessible(baseAddress, 0x8000));
+    }
+
     [Fact]
     public void AlignedAllocationDoesNotRetainOverallocatedMappingsOutsideMacOS()
     {
@@ -133,6 +172,11 @@ public sealed class GuestMemoryAllocatorTests
     [Fact]
     public void TryBackFixedRangeRollsBackEarlierGapsWhenLaterGapCannotBeBacked()
     {
+        if (OperatingSystem.IsWindows())
+        {
+            return;
+        }
+
         // Layout: committed | free | committed | free
         // First free gap allocates successfully, second fails.
         // The first allocation must be freed — nothing should leak.
@@ -154,6 +198,11 @@ public sealed class GuestMemoryAllocatorTests
     [Fact]
     public void TryBackFixedRangeFillsOnlyTheFreePagesOfAPartiallyOccupiedRange()
     {
+        if (OperatingSystem.IsWindows())
+        {
+            return;
+        }
+
         const ulong rangeBase = 0x0000_0020_2F00_0000;
         const ulong rangeSize = 0x40_0000;
         const ulong occupiedSize = 0x4_0000;
@@ -519,6 +568,154 @@ public sealed class GuestMemoryAllocatorTests
         }
     }
 
+    private sealed class GranularityAwareHostMemory : IHostMemory
+    {
+        private const ulong Granularity = 0x10000;
+        private const ulong Page = 0x1000;
+
+        private readonly SortedDictionary<ulong, (ulong Size, SortedSet<ulong> CommittedPages)> _allocations = new();
+
+        public ulong Allocate(ulong desiredAddress, ulong size, HostPageProtection protection)
+        {
+            var reservedBase = Reserve(desiredAddress, size, protection);
+            if (reservedBase != 0)
+            {
+                var start = desiredAddress == 0 ? reservedBase : AlignDown(desiredAddress, Page);
+                Commit(start, size, protection);
+            }
+
+            return reservedBase;
+        }
+
+        public ulong Reserve(ulong desiredAddress, ulong size, HostPageProtection protection)
+        {
+            if (desiredAddress == 0)
+            {
+                return 0;
+            }
+
+            var allocationBase = AlignDown(desiredAddress, Granularity);
+            var end = AlignUp(desiredAddress + size, Page);
+            foreach (var (existingBase, existing) in _allocations)
+            {
+                if (allocationBase < existingBase + existing.Size && existingBase < end)
+                {
+                    return 0;
+                }
+            }
+
+            _allocations[allocationBase] = (end - allocationBase, new SortedSet<ulong>());
+            return allocationBase;
+        }
+
+        public bool Commit(ulong address, ulong size, HostPageProtection protection)
+        {
+            var start = AlignDown(address, Page);
+            var end = AlignUp(address + size, Page);
+            if (!TryFindAllocation(start, out var allocationBase, out var allocation) ||
+                end > allocationBase + allocation.Size)
+            {
+                return false;
+            }
+
+            for (var page = start; page < end; page += Page)
+            {
+                allocation.CommittedPages.Add(page);
+            }
+
+            return true;
+        }
+
+        public bool Free(ulong address) => _allocations.Remove(address);
+
+        public bool Protect(ulong address, ulong size, HostPageProtection protection, out uint rawOldProtection)
+        {
+            rawOldProtection = 0;
+            return true;
+        }
+
+        public bool ProtectRaw(ulong address, ulong size, uint rawProtection, out uint rawOldProtection)
+        {
+            rawOldProtection = 0;
+            return true;
+        }
+
+        public bool Query(ulong address, out HostRegionInfo info)
+        {
+            var page = AlignDown(address, Page);
+            if (TryFindAllocation(page, out var allocationBase, out var allocation))
+            {
+                var committed = allocation.CommittedPages.Contains(page);
+                var runEnd = page + Page;
+                while (runEnd < allocationBase + allocation.Size &&
+                       allocation.CommittedPages.Contains(runEnd) == committed)
+                {
+                    runEnd += Page;
+                }
+
+                info = new HostRegionInfo(
+                    page,
+                    allocationBase,
+                    runEnd - page,
+                    committed ? HostRegionState.Committed : HostRegionState.Reserved,
+                    0,
+                    committed ? HostPageProtection.ReadWrite : HostPageProtection.NoAccess,
+                    0,
+                    0);
+                return true;
+            }
+
+            var freeEnd = ulong.MaxValue;
+            foreach (var existingBase in _allocations.Keys)
+            {
+                if (existingBase > page)
+                {
+                    freeEnd = existingBase;
+                    break;
+                }
+            }
+
+            info = new HostRegionInfo(
+                page,
+                0,
+                freeEnd - page,
+                HostRegionState.Free,
+                0,
+                HostPageProtection.NoAccess,
+                0,
+                0);
+            return true;
+        }
+
+        public void FlushInstructionCache(ulong address, ulong size)
+        {
+        }
+
+        private bool TryFindAllocation(
+            ulong address,
+            out ulong allocationBase,
+            out (ulong Size, SortedSet<ulong> CommittedPages) allocation)
+        {
+            foreach (var (existingBase, existing) in _allocations)
+            {
+                if (address >= existingBase && address < existingBase + existing.Size)
+                {
+                    allocationBase = existingBase;
+                    allocation = existing;
+                    return true;
+                }
+            }
+
+            allocationBase = 0;
+            allocation = default;
+            return false;
+        }
+
+        private static ulong AlignDown(ulong value, ulong alignment) => value & ~(alignment - 1);
+
+        private static ulong AlignUp(ulong value, ulong alignment) => (value + alignment - 1) & ~(alignment - 1);
+    }
+
     private sealed class LazyHostMemory(ulong address) : IHostMemory, IDisposable
     {
         public bool CommitSucceeds { get; set; } = true;