kernel: return -1/errno from POSIX file syscalls on failure (#461)

* kernel: return -1/errno from POSIX open and fstat on failure The POSIX-named open (wuCroIGjt2g) and fstat (mqQMh1zPPT8) exports routed straight to the raw sceKernel* implementations, which report failure via the 0x8002xxxx OrbisGen2Result sentinel in the return value. libc callers follow the POSIX ABI and expect -1 with errno set, so they stored the sentinel as a valid fd. Unity's IL2CPP file layer did exactly this while probing the absent /app0/Media/il2cpp.usym: open returned NOT_FOUND (0x80020002), the guest kept the sentinel as an fd, passed it back into fstat, and eventually dereferenced a null pointer (vmovups xmm0,[rdi], rdi=0) deep in a native .prx, crashing with 0xC0000005. Wrap both entry points to translate a failed raw result into -1/errno, mirroring the existing PosixStat/PosixLseek convention. Add a shared PosixFailure helper (fstat maps a bad handle to EBADF; path calls default to ENOENT) and route it through PosixStat too. Covered by two regression tests reproducing the missing-file and misused-sentinel-fd cases. * kernel: return -1/errno from POSIX close, read and write on failure Same defect class as open/fstat: the POSIX-named close (bY-PO6JhzhQ), read (AqBioC2vF3I) and write (FN4gaPmuFV8) exports forwarded the raw sceKernel* core result, leaking the 0x8002xxxx sentinel to libc callers that expect -1/errno on a bad fd. close in particular is on the crashing Unity path, invoked on the sentinel the guest mistook for an fd. Wrap all three through PosixFailure with EBADF as the fd-not-found errno. Add regression tests for each, and correct the socket test that had locked in the old raw-sentinel contract for a double close. --------- Co-authored-by: slick-daddy <slick-daddy@users.noreply.github.com>
2026-07-30 22:49:53 +08:00 · 2026-07-20 14:17:08 +03:00
parent d151e151c2
commit bb3318a503
4 changed files with 163 additions and 24 deletions
@@ -41,6 +41,90 @@ public sealed class KernelMemoryCompatExportsTests
        Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
    }

+    [Fact]
+    public void PosixOpen_MissingFileReturnsMinusOne()
+    {
+        const ulong memoryBase = 0x1_0000_0000;
+        const ulong pathAddress = memoryBase + 0x100;
+        var memory = new FakeCpuMemory(memoryBase, 0x1000);
+        var context = new CpuContext(memory, Generation.Gen5);
+        memory.WriteCString(pathAddress, "/__sharpemu_test_missing__/il2cpp.usym");
+        context[CpuRegister.Rdi] = pathAddress;
+        context[CpuRegister.Rsi] = 0; // O_RDONLY
+
+        var result = KernelMemoryCompatExports.PosixOpen(context);
+
+        // A libc open() failure must be -1, not the raw 0x8002xxxx sentinel the
+        // guest would otherwise store as a valid fd and later dereference.
+        Assert.Equal(-1, result);
+        Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
+    }
+
+    [Fact]
+    public void PosixFstat_BadDescriptorReturnsMinusOne()
+    {
+        const ulong memoryBase = 0x1_0000_0000;
+        const ulong statAddress = memoryBase + 0x400;
+        var memory = new FakeCpuMemory(memoryBase, 0x1000);
+        var context = new CpuContext(memory, Generation.Gen5);
+        context[CpuRegister.Rdi] = 0x80020002; // the not-found sentinel misused as an fd
+        context[CpuRegister.Rsi] = statAddress;
+
+        var result = KernelMemoryCompatExports.PosixFstat(context);
+
+        Assert.Equal(-1, result);
+        Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
+    }
+
+    [Fact]
+    public void PosixClose_BadDescriptorReturnsMinusOne()
+    {
+        const ulong memoryBase = 0x1_0000_0000;
+        var memory = new FakeCpuMemory(memoryBase, 0x1000);
+        var context = new CpuContext(memory, Generation.Gen5);
+        context[CpuRegister.Rdi] = 0x80020002; // never-opened / sentinel fd
+
+        var result = KernelMemoryCompatExports.PosixClose(context);
+
+        Assert.Equal(-1, result);
+        Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
+    }
+
+    [Fact]
+    public void PosixRead_BadDescriptorReturnsMinusOne()
+    {
+        const ulong memoryBase = 0x1_0000_0000;
+        const ulong bufferAddress = memoryBase + 0x200;
+        var memory = new FakeCpuMemory(memoryBase, 0x1000);
+        var context = new CpuContext(memory, Generation.Gen5);
+        context[CpuRegister.Rdi] = 0x80020002; // never-opened / sentinel fd
+        context[CpuRegister.Rsi] = bufferAddress;
+        context[CpuRegister.Rdx] = 0x40;
+
+        var result = KernelMemoryCompatExports.PosixRead(context);
+
+        Assert.Equal(-1, result);
+        Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
+    }
+
+    [Fact]
+    public void PosixWrite_BadDescriptorReturnsMinusOne()
+    {
+        const ulong memoryBase = 0x1_0000_0000;
+        const ulong bufferAddress = memoryBase + 0x200;
+        var memory = new FakeCpuMemory(memoryBase, 0x1000);
+        var context = new CpuContext(memory, Generation.Gen5);
+        memory.WriteCString(bufferAddress, "payload");
+        context[CpuRegister.Rdi] = 0x80020002; // never-opened / sentinel fd
+        context[CpuRegister.Rsi] = bufferAddress;
+        context[CpuRegister.Rdx] = 0x7;
+
+        var result = KernelMemoryCompatExports.PosixWrite(context);
+
+        Assert.Equal(-1, result);
+        Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
+    }
+
    [Fact]
    public void Sprintf_ReadsVariadicDoubleFromXmmRegister()
    {
@@ -37,10 +37,11 @@ public sealed class KernelSocketCompatExportsTests
                KernelMemoryCompatExports.PosixClose(context));
            Assert.Equal(0UL, context[CpuRegister.Rax]);

+            // A second close of an already-closed fd fails per the POSIX ABI:
+            // -1 with errno set, not the raw Orbis NOT_FOUND sentinel.
            context[CpuRegister.Rdi] = unchecked((ulong)guestFd);
-            Assert.Equal(
-                (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND,
-                KernelMemoryCompatExports.PosixClose(context));
+            Assert.Equal(-1, KernelMemoryCompatExports.PosixClose(context));
+            Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
        }
        finally
        {