From bb3318a50372dcdbee40cf46958741f5f2f4a586 Mon Sep 17 00:00:00 2001 From: Slick Daddy <129640104+slick-daddy@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:17:08 +0300 Subject: [PATCH] kernel: return -1/errno from POSIX file syscalls on failure (#461) * kernel: return -1/errno from POSIX open and fstat on failure The POSIX-named open (wuCroIGjt2g) and fstat (mqQMh1zPPT8) exports routed straight to the raw sceKernel* implementations, which report failure via the 0x8002xxxx OrbisGen2Result sentinel in the return value. libc callers follow the POSIX ABI and expect -1 with errno set, so they stored the sentinel as a valid fd. Unity's IL2CPP file layer did exactly this while probing the absent /app0/Media/il2cpp.usym: open returned NOT_FOUND (0x80020002), the guest kept the sentinel as an fd, passed it back into fstat, and eventually dereferenced a null pointer (vmovups xmm0,[rdi], rdi=0) deep in a native .prx, crashing with 0xC0000005. Wrap both entry points to translate a failed raw result into -1/errno, mirroring the existing PosixStat/PosixLseek convention. Add a shared PosixFailure helper (fstat maps a bad handle to EBADF; path calls default to ENOENT) and route it through PosixStat too. Covered by two regression tests reproducing the missing-file and misused-sentinel-fd cases. * kernel: return -1/errno from POSIX close, read and write on failure Same defect class as open/fstat: the POSIX-named close (bY-PO6JhzhQ), read (AqBioC2vF3I) and write (FN4gaPmuFV8) exports forwarded the raw sceKernel* core result, leaking the 0x8002xxxx sentinel to libc callers that expect -1/errno on a bad fd. close in particular is on the crashing Unity path, invoked on the sentinel the guest mistook for an fd. Wrap all three through PosixFailure with EBADF as the fd-not-found errno. Add regression tests for each, and correct the socket test that had locked in the old raw-sentinel contract for a double close. --------- Co-authored-by: slick-daddy --- src/SharpEmu.Libs/Kernel/KernelExports.cs | 4 +- .../Kernel/KernelMemoryCompatExports.cs | 92 +++++++++++++++---- .../Kernel/KernelMemoryCompatExportsTests.cs | 84 +++++++++++++++++ .../Kernel/KernelSocketCompatExportsTests.cs | 7 +- 4 files changed, 163 insertions(+), 24 deletions(-) diff --git a/src/SharpEmu.Libs/Kernel/KernelExports.cs b/src/SharpEmu.Libs/Kernel/KernelExports.cs index 09fb5b2c..3e4297f5 100644 --- a/src/SharpEmu.Libs/Kernel/KernelExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelExports.cs @@ -362,7 +362,7 @@ public static class KernelExports ExportName = "open", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int Open(CpuContext ctx) => KernelMemoryCompatExports.KernelOpenUnderscore(ctx); + public static int Open(CpuContext ctx) => KernelMemoryCompatExports.PosixOpen(ctx); [SysAbiExport( Nid = "1G3lF1Gg1k8", @@ -376,7 +376,7 @@ public static class KernelExports ExportName = "fstat", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libc")] - public static int Fstat(CpuContext ctx) => KernelMemoryCompatExports.KernelFstat(ctx); + public static int Fstat(CpuContext ctx) => KernelMemoryCompatExports.PosixFstat(ctx); [SysAbiExport( Nid = "hcuQgD53UxM", diff --git a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs index 17f4288d..80e7e42d 100644 --- a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs @@ -65,7 +65,10 @@ public static partial class KernelMemoryCompatExports private const uint HostPageExecuteReadWrite = 0x40; private const uint HostPageExecuteWriteCopy = 0x80; private const uint HostPageGuard = 0x100; + private const int Enoent = 2; + private const int Ebadf = 9; private const int Enomem = 12; + private const int Eacces = 13; private const int Efault = 14; private const int Einval = 22; private const int Erange = 34; @@ -1545,7 +1548,13 @@ public static partial class KernelMemoryCompatExports ExportName = "close", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int PosixClose(CpuContext ctx) => KernelCloseCore(ctx, unchecked((int)ctx[CpuRegister.Rdi])); + public static int PosixClose(CpuContext ctx) + { + var result = KernelCloseCore(ctx, unchecked((int)ctx[CpuRegister.Rdi])); + return result == (int)OrbisGen2Result.ORBIS_GEN2_OK + ? 0 + : PosixFailure(ctx, result, notFoundErrno: Ebadf); + } [SysAbiExport( Nid = "UK2Tl2DWUns", @@ -1602,6 +1611,29 @@ public static partial class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + // Translates a failed raw Orbis kernel result into the libc/POSIX ABI: + // return -1 with errno set. The raw sceKernel* implementations report the + // 0x8002xxxx sentinel through the return value, but the POSIX-named exports + // (open/fstat/close/read/write/stat) are called by libc code that expects a + // negative result on failure. Leaking the raw sentinel makes callers store + // it as a "valid" fd or handle and later dereference it - the null-pointer + // access violation seen when Unity's IL2CPP file layer probes an absent + // il2cpp.usym. fd-based calls pass notFoundErrno=Ebadf; path-based calls + // leave the Enoent default. + private static int PosixFailure(CpuContext ctx, int orbisResult, int notFoundErrno = Enoent) + { + var errno = orbisResult switch + { + (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT => Einval, + (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT => Efault, + (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED => Eacces, + _ => notFoundErrno, + }; + KernelRuntimeCompatExports.TrySetErrno(ctx, errno); + ctx[CpuRegister.Rax] = ulong.MaxValue; + return -1; + } + [SysAbiExport( Nid = "E6ao34wPw+U", ExportName = "stat", @@ -1610,23 +1642,29 @@ public static partial class KernelMemoryCompatExports public static int PosixStat(CpuContext ctx) { var result = KernelStat(ctx); - if (result == (int)OrbisGen2Result.ORBIS_GEN2_OK) - { - return 0; - } + return result == (int)OrbisGen2Result.ORBIS_GEN2_OK + ? 0 + : PosixFailure(ctx, result); + } - // stat(2) follows the libc/POSIX ABI: failures return -1 and expose - // the reason through errno. Returning the raw Orbis kernel code here - // makes callers treat a missing file as a non-negative success value. - var errno = result switch - { - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT => Einval, - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT => Efault, - _ => 2, // ENOENT - }; - KernelRuntimeCompatExports.TrySetErrno(ctx, errno); - ctx[CpuRegister.Rax] = ulong.MaxValue; - return -1; + // POSIX open(2): translates a failed raw open into -1/errno. On success + // KernelOpenUnderscore already writes the fd into RAX (the import bridge + // prefers a written RAX over the return value), so returning 0 is correct. + public static int PosixOpen(CpuContext ctx) + { + var result = KernelOpenUnderscore(ctx); + return result == (int)OrbisGen2Result.ORBIS_GEN2_OK + ? 0 + : PosixFailure(ctx, result); + } + + // POSIX fstat(2): a bad fd maps to EBADF rather than the path-oriented ENOENT. + public static int PosixFstat(CpuContext ctx) + { + var result = KernelFstat(ctx); + return result == (int)OrbisGen2Result.ORBIS_GEN2_OK + ? 0 + : PosixFailure(ctx, result, notFoundErrno: Ebadf); } [SysAbiExport( @@ -2096,7 +2134,15 @@ public static partial class KernelMemoryCompatExports ExportName = "read", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int PosixRead(CpuContext ctx) => KernelReadUnderscore(ctx); + public static int PosixRead(CpuContext ctx) + { + // On success KernelReadUnderscore writes the byte count into RAX, which + // the import bridge prefers over this return value. + var result = KernelReadUnderscore(ctx); + return result == (int)OrbisGen2Result.ORBIS_GEN2_OK + ? 0 + : PosixFailure(ctx, result, notFoundErrno: Ebadf); + } [SysAbiExport( Nid = "Cg4srZ6TKbU", @@ -2295,7 +2341,15 @@ public static partial class KernelMemoryCompatExports ExportName = "write", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int PosixWrite(CpuContext ctx) => KernelWriteUnderscore(ctx); + public static int PosixWrite(CpuContext ctx) + { + // On success KernelWriteUnderscore writes the byte count into RAX, which + // the import bridge prefers over this return value. + var result = KernelWriteUnderscore(ctx); + return result == (int)OrbisGen2Result.ORBIS_GEN2_OK + ? 0 + : PosixFailure(ctx, result, notFoundErrno: Ebadf); + } [SysAbiExport( Nid = "4wSze92BhLI", diff --git a/tests/SharpEmu.Libs.Tests/Kernel/KernelMemoryCompatExportsTests.cs b/tests/SharpEmu.Libs.Tests/Kernel/KernelMemoryCompatExportsTests.cs index c7a50581..203b9702 100644 --- a/tests/SharpEmu.Libs.Tests/Kernel/KernelMemoryCompatExportsTests.cs +++ b/tests/SharpEmu.Libs.Tests/Kernel/KernelMemoryCompatExportsTests.cs @@ -41,6 +41,90 @@ public sealed class KernelMemoryCompatExportsTests Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]); } + [Fact] + public void PosixOpen_MissingFileReturnsMinusOne() + { + const ulong memoryBase = 0x1_0000_0000; + const ulong pathAddress = memoryBase + 0x100; + var memory = new FakeCpuMemory(memoryBase, 0x1000); + var context = new CpuContext(memory, Generation.Gen5); + memory.WriteCString(pathAddress, "/__sharpemu_test_missing__/il2cpp.usym"); + context[CpuRegister.Rdi] = pathAddress; + context[CpuRegister.Rsi] = 0; // O_RDONLY + + var result = KernelMemoryCompatExports.PosixOpen(context); + + // A libc open() failure must be -1, not the raw 0x8002xxxx sentinel the + // guest would otherwise store as a valid fd and later dereference. + Assert.Equal(-1, result); + Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]); + } + + [Fact] + public void PosixFstat_BadDescriptorReturnsMinusOne() + { + const ulong memoryBase = 0x1_0000_0000; + const ulong statAddress = memoryBase + 0x400; + var memory = new FakeCpuMemory(memoryBase, 0x1000); + var context = new CpuContext(memory, Generation.Gen5); + context[CpuRegister.Rdi] = 0x80020002; // the not-found sentinel misused as an fd + context[CpuRegister.Rsi] = statAddress; + + var result = KernelMemoryCompatExports.PosixFstat(context); + + Assert.Equal(-1, result); + Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]); + } + + [Fact] + public void PosixClose_BadDescriptorReturnsMinusOne() + { + const ulong memoryBase = 0x1_0000_0000; + var memory = new FakeCpuMemory(memoryBase, 0x1000); + var context = new CpuContext(memory, Generation.Gen5); + context[CpuRegister.Rdi] = 0x80020002; // never-opened / sentinel fd + + var result = KernelMemoryCompatExports.PosixClose(context); + + Assert.Equal(-1, result); + Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]); + } + + [Fact] + public void PosixRead_BadDescriptorReturnsMinusOne() + { + const ulong memoryBase = 0x1_0000_0000; + const ulong bufferAddress = memoryBase + 0x200; + var memory = new FakeCpuMemory(memoryBase, 0x1000); + var context = new CpuContext(memory, Generation.Gen5); + context[CpuRegister.Rdi] = 0x80020002; // never-opened / sentinel fd + context[CpuRegister.Rsi] = bufferAddress; + context[CpuRegister.Rdx] = 0x40; + + var result = KernelMemoryCompatExports.PosixRead(context); + + Assert.Equal(-1, result); + Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]); + } + + [Fact] + public void PosixWrite_BadDescriptorReturnsMinusOne() + { + const ulong memoryBase = 0x1_0000_0000; + const ulong bufferAddress = memoryBase + 0x200; + var memory = new FakeCpuMemory(memoryBase, 0x1000); + var context = new CpuContext(memory, Generation.Gen5); + memory.WriteCString(bufferAddress, "payload"); + context[CpuRegister.Rdi] = 0x80020002; // never-opened / sentinel fd + context[CpuRegister.Rsi] = bufferAddress; + context[CpuRegister.Rdx] = 0x7; + + var result = KernelMemoryCompatExports.PosixWrite(context); + + Assert.Equal(-1, result); + Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]); + } + [Fact] public void Sprintf_ReadsVariadicDoubleFromXmmRegister() { diff --git a/tests/SharpEmu.Libs.Tests/Kernel/KernelSocketCompatExportsTests.cs b/tests/SharpEmu.Libs.Tests/Kernel/KernelSocketCompatExportsTests.cs index fb7a5111..5865b84d 100644 --- a/tests/SharpEmu.Libs.Tests/Kernel/KernelSocketCompatExportsTests.cs +++ b/tests/SharpEmu.Libs.Tests/Kernel/KernelSocketCompatExportsTests.cs @@ -37,10 +37,11 @@ public sealed class KernelSocketCompatExportsTests KernelMemoryCompatExports.PosixClose(context)); Assert.Equal(0UL, context[CpuRegister.Rax]); + // A second close of an already-closed fd fails per the POSIX ABI: + // -1 with errno set, not the raw Orbis NOT_FOUND sentinel. context[CpuRegister.Rdi] = unchecked((ulong)guestFd); - Assert.Equal( - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND, - KernelMemoryCompatExports.PosixClose(context)); + Assert.Equal(-1, KernelMemoryCompatExports.PosixClose(context)); + Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]); } finally {