Fix macOS fixed-address allocation collisions (#246)

src/SharpEmu.HLE/Host/Posix/PosixHostMemory.cs

@@ -163,13 +163,18 @@ internal sealed unsafe class PosixHostMemory : IHostMemory
         private const int PROT_EXEC = 0x4;
 
         private const int MAP_PRIVATE = 0x02;
-        private const int MAP_FIXED = 0x10;
         private static readonly int MAP_ANON = OperatingSystem.IsMacOS() ? 0x1000 : 0x20;
         private static readonly int MAP_NORESERVE = OperatingSystem.IsMacOS() ? 0 : 0x4000;
 
         // Linux-only: fail instead of clobbering an existing mapping.
         private const int MAP_FIXED_NOREPLACE = 0x100000;
 
+        private const int KERN_SUCCESS = 0;
+
+        // On Darwin fixed placement is represented by the absence of
+        // VM_FLAGS_ANYWHERE. Do not add VM_FLAGS_OVERWRITE: overlap must fail.
+        private const int VM_FLAGS_FIXED = 0;
+
         private static readonly nint MAP_FAILED = -1;
 
         private static readonly object Gate = new();
@@ -181,6 +186,7 @@ internal sealed unsafe class PosixHostMemory : IHostMemory
             public ulong Size;
             public uint DefaultProtect;
             public Dictionary<ulong, uint>? PageProtects;
+            public bool UsesMachAllocation;
 
             public ulong End => Base + Size;
 
@@ -251,6 +257,7 @@ internal sealed unsafe class PosixHostMemory : IHostMemory
                 }
 
                 nint result;
+                var usesMachAllocation = false;
                 if (address != null)
                 {
                     // Win32 maps at exactly the requested address or fails
@@ -280,15 +287,17 @@ internal sealed unsafe class PosixHostMemory : IHostMemory
                     if (result == MAP_FAILED && OperatingSystem.IsMacOS())
                     {
                         // The hint-only attempt above didn't land at the requested
-                        // address. This is routinely the case for the PS5's fixed
-                        // 32 GiB image base under Rosetta 2 on Apple Silicon, where
-                        // the kernel never honors that hint. Retry with true
-                        // MAP_FIXED: this can clobber an untracked host mapping
-                        // (dyld, the runtime's JIT heap, Rosetta) if one already
-                        // sits exactly there, but without it guest images that
-                        // require this base never load at all on macOS.
-                        Trace($"exact mmap hint failed, retrying with MAP_FIXED: addr=0x{(ulong)address:X16}");
-                        result = mmap((nint)address, (nuint)alignedSize, posixProtect, flags | MAP_FIXED, -1, 0);
+                        // address. mach_vm_allocate with fixed placement and no
+                        // overwrite flag atomically maps the requested range or
+                        // fails if any host mapping already owns it. Unlike
+                        // MAP_FIXED, it cannot clobber CLR, dyld, JIT, or Rosetta
+                        // memory that is absent from our shadow table.
+                        Trace($"exact mmap hint failed, retrying with fixed Mach allocation: addr=0x{(ulong)address:X16}");
+                        result = AllocateDarwinFixed(
+                            (nint)address,
+                            alignedSize,
+                            posixProtect);
+                        usesMachAllocation = result != MAP_FAILED;
                     }
 
                     if (result == MAP_FAILED || (ulong)result != (ulong)address)
@@ -316,7 +325,8 @@ internal sealed unsafe class PosixHostMemory : IHostMemory
                 {
                     Base = (ulong)result,
                     Size = alignedSize,
-                    DefaultProtect = protect
+                    DefaultProtect = protect,
+                    UsesMachAllocation = usesMachAllocation,
                 };
 
                 return (void*)result;
@@ -335,8 +345,15 @@ internal sealed unsafe class PosixHostMemory : IHostMemory
                     return false;
                 }
 
-                Regions.Remove((ulong)address);
-                return munmap((nint)address, (nuint)region.Size) == 0;
+                var released = region.UsesMachAllocation
+                    ? mach_vm_deallocate(mach_task_self(), (ulong)address, region.Size) == KERN_SUCCESS
+                    : munmap((nint)address, (nuint)region.Size) == 0;
+                if (released)
+                {
+                    Regions.Remove((ulong)address);
+                }
+
+                return released;
             }
         }
 
@@ -504,6 +521,40 @@ internal sealed unsafe class PosixHostMemory : IHostMemory
             };
         }
 
+        private static nint AllocateDarwinFixed(nint requestedAddress, ulong size, int posixProtect)
+        {
+            var address = (ulong)requestedAddress;
+            var result = mach_vm_allocate(
+                mach_task_self(),
+                ref address,
+                size,
+                VM_FLAGS_FIXED);
+            if (result != KERN_SUCCESS || address != (ulong)requestedAddress)
+            {
+                if (result == KERN_SUCCESS)
+                {
+                    _ = mach_vm_deallocate(mach_task_self(), address, size);
+                }
+
+                Trace(
+                    $"fixed Mach allocation failed: addr=0x{(ulong)requestedAddress:X16} " +
+                    $"size=0x{size:X} kern_return={result}");
+                return MAP_FAILED;
+            }
+
+            if (mprotect((nint)address, (nuint)size, posixProtect) == 0)
+            {
+                return (nint)address;
+            }
+
+            var error = Marshal.GetLastPInvokeError();
+            _ = mach_vm_deallocate(mach_task_self(), address, size);
+            Trace(
+                $"fixed Mach allocation protection failed: addr=0x{address:X16} " +
+                $"size=0x{size:X} errno={error}");
+            return MAP_FAILED;
+        }
+
         private static void Trace(string message)
         {
             if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_VMEM"), "1", StringComparison.Ordinal))
@@ -524,5 +575,14 @@ internal sealed unsafe class PosixHostMemory : IHostMemory
 
         [DllImport("libc", SetLastError = true)]
         private static extern int mprotect(nint addr, nuint length, int prot);
+
+        [DllImport("libSystem.B.dylib")]
+        private static extern uint mach_task_self();
+
+        [DllImport("libSystem.B.dylib")]
+        private static extern int mach_vm_allocate(uint target, ref ulong address, ulong size, int flags);
+
+        [DllImport("libSystem.B.dylib")]
+        private static extern int mach_vm_deallocate(uint target, ulong address, ulong size);
     }
 }

src/SharpEmu.HLE/SharpEmu.HLE.csproj

@@ -14,6 +14,7 @@ SPDX-License-Identifier: GPL-2.0-or-later
 
   <ItemGroup>
     <InternalsVisibleTo Include="SharpEmu.Core" />
+    <InternalsVisibleTo Include="SharpEmu.Libs.Tests" />
   </ItemGroup>
 
   <ItemGroup>

tests/SharpEmu.Libs.Tests/Memory/PosixHostMemoryTests.cs

@@ -0,0 +1,62 @@
+// Copyright (C) 2026 SharpEmu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+using System.Runtime.InteropServices;
+using SharpEmu.HLE.Host;
+using SharpEmu.HLE.Host.Posix;
+using Xunit;
+
+namespace SharpEmu.Libs.Tests.Memory;
+
+public sealed unsafe class PosixHostMemoryTests
+{
+    private const int ProtRead = 0x1;
+    private const int ProtWrite = 0x2;
+    private const int MapPrivate = 0x02;
+    private const int MapAnonymousDarwin = 0x1000;
+    private static readonly nint MapFailed = -1;
+
+    [Fact]
+    public void ExactAllocationDoesNotReplaceUntrackedDarwinMapping()
+    {
+        if (!OperatingSystem.IsMacOS())
+        {
+            return;
+        }
+
+        var size = checked((nuint)Environment.SystemPageSize);
+        var externalMapping = mmap(
+            0,
+            size,
+            ProtRead | ProtWrite,
+            MapPrivate | MapAnonymousDarwin,
+            -1,
+            0);
+        Assert.NotEqual(MapFailed, externalMapping);
+
+        try
+        {
+            var sentinel = new Span<byte>((void*)externalMapping, checked((int)size));
+            sentinel.Fill(0xA5);
+
+            var hostMemory = new PosixHostMemory();
+            var result = hostMemory.Allocate(
+                (ulong)externalMapping,
+                (ulong)size,
+                HostPageProtection.ReadWrite);
+
+            Assert.Equal(0UL, result);
+            Assert.All(sentinel.ToArray(), value => Assert.Equal(0xA5, value));
+        }
+        finally
+        {
+            Assert.Equal(0, munmap(externalMapping, size));
+        }
+    }
+
+    [DllImport("libc", SetLastError = true)]
+    private static extern nint mmap(nint addr, nuint length, int prot, int flags, int fd, long offset);
+
+    [DllImport("libc", SetLastError = true)]
+    private static extern int munmap(nint addr, nuint length);
+}