kernel: return -1/errno from POSIX file syscalls on failure (#461)

* kernel: return -1/errno from POSIX open and fstat on failure

The POSIX-named open (wuCroIGjt2g) and fstat (mqQMh1zPPT8) exports routed
straight to the raw sceKernel* implementations, which report failure via
the 0x8002xxxx OrbisGen2Result sentinel in the return value. libc callers
follow the POSIX ABI and expect -1 with errno set, so they stored the
sentinel as a valid fd. Unity's IL2CPP file layer did exactly this while
probing the absent /app0/Media/il2cpp.usym: open returned NOT_FOUND
(0x80020002), the guest kept the sentinel as an fd, passed it back into
fstat, and eventually dereferenced a null pointer (vmovups xmm0,[rdi],
rdi=0) deep in a native .prx, crashing with 0xC0000005.

Wrap both entry points to translate a failed raw result into -1/errno,
mirroring the existing PosixStat/PosixLseek convention. Add a shared
PosixFailure helper (fstat maps a bad handle to EBADF; path calls default
to ENOENT) and route it through PosixStat too. Covered by two regression
tests reproducing the missing-file and misused-sentinel-fd cases.

* kernel: return -1/errno from POSIX close, read and write on failure

Same defect class as open/fstat: the POSIX-named close (bY-PO6JhzhQ),
read (AqBioC2vF3I) and write (FN4gaPmuFV8) exports forwarded the raw
sceKernel* core result, leaking the 0x8002xxxx sentinel to libc callers
that expect -1/errno on a bad fd. close in particular is on the crashing
Unity path, invoked on the sentinel the guest mistook for an fd.

Wrap all three through PosixFailure with EBADF as the fd-not-found errno.
Add regression tests for each, and correct the socket test that had
locked in the old raw-sentinel contract for a double close.

---------

Co-authored-by: slick-daddy <slick-daddy@users.noreply.github.com>
This commit is contained in:
Slick Daddy
2026-07-20 14:17:08 +03:00
committed by GitHub
parent d151e151c2
commit bb3318a503
4 changed files with 163 additions and 24 deletions
+2 -2
View File
@@ -362,7 +362,7 @@ public static class KernelExports
ExportName = "open",
Target = Generation.Gen4 | Generation.Gen5,
LibraryName = "libKernel")]
public static int Open(CpuContext ctx) => KernelMemoryCompatExports.KernelOpenUnderscore(ctx);
public static int Open(CpuContext ctx) => KernelMemoryCompatExports.PosixOpen(ctx);
[SysAbiExport(
Nid = "1G3lF1Gg1k8",
@@ -376,7 +376,7 @@ public static class KernelExports
ExportName = "fstat",
Target = Generation.Gen4 | Generation.Gen5,
LibraryName = "libc")]
public static int Fstat(CpuContext ctx) => KernelMemoryCompatExports.KernelFstat(ctx);
public static int Fstat(CpuContext ctx) => KernelMemoryCompatExports.PosixFstat(ctx);
[SysAbiExport(
Nid = "hcuQgD53UxM",
@@ -65,7 +65,10 @@ public static partial class KernelMemoryCompatExports
private const uint HostPageExecuteReadWrite = 0x40;
private const uint HostPageExecuteWriteCopy = 0x80;
private const uint HostPageGuard = 0x100;
private const int Enoent = 2;
private const int Ebadf = 9;
private const int Enomem = 12;
private const int Eacces = 13;
private const int Efault = 14;
private const int Einval = 22;
private const int Erange = 34;
@@ -1545,7 +1548,13 @@ public static partial class KernelMemoryCompatExports
ExportName = "close",
Target = Generation.Gen4 | Generation.Gen5,
LibraryName = "libKernel")]
public static int PosixClose(CpuContext ctx) => KernelCloseCore(ctx, unchecked((int)ctx[CpuRegister.Rdi]));
public static int PosixClose(CpuContext ctx)
{
var result = KernelCloseCore(ctx, unchecked((int)ctx[CpuRegister.Rdi]));
return result == (int)OrbisGen2Result.ORBIS_GEN2_OK
? 0
: PosixFailure(ctx, result, notFoundErrno: Ebadf);
}
[SysAbiExport(
Nid = "UK2Tl2DWUns",
@@ -1602,6 +1611,29 @@ public static partial class KernelMemoryCompatExports
return (int)OrbisGen2Result.ORBIS_GEN2_OK;
}
// Translates a failed raw Orbis kernel result into the libc/POSIX ABI:
// return -1 with errno set. The raw sceKernel* implementations report the
// 0x8002xxxx sentinel through the return value, but the POSIX-named exports
// (open/fstat/close/read/write/stat) are called by libc code that expects a
// negative result on failure. Leaking the raw sentinel makes callers store
// it as a "valid" fd or handle and later dereference it - the null-pointer
// access violation seen when Unity's IL2CPP file layer probes an absent
// il2cpp.usym. fd-based calls pass notFoundErrno=Ebadf; path-based calls
// leave the Enoent default.
private static int PosixFailure(CpuContext ctx, int orbisResult, int notFoundErrno = Enoent)
{
var errno = orbisResult switch
{
(int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT => Einval,
(int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT => Efault,
(int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED => Eacces,
_ => notFoundErrno,
};
KernelRuntimeCompatExports.TrySetErrno(ctx, errno);
ctx[CpuRegister.Rax] = ulong.MaxValue;
return -1;
}
[SysAbiExport(
Nid = "E6ao34wPw+U",
ExportName = "stat",
@@ -1610,23 +1642,29 @@ public static partial class KernelMemoryCompatExports
public static int PosixStat(CpuContext ctx)
{
var result = KernelStat(ctx);
if (result == (int)OrbisGen2Result.ORBIS_GEN2_OK)
{
return 0;
}
return result == (int)OrbisGen2Result.ORBIS_GEN2_OK
? 0
: PosixFailure(ctx, result);
}
// stat(2) follows the libc/POSIX ABI: failures return -1 and expose
// the reason through errno. Returning the raw Orbis kernel code here
// makes callers treat a missing file as a non-negative success value.
var errno = result switch
{
(int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT => Einval,
(int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT => Efault,
_ => 2, // ENOENT
};
KernelRuntimeCompatExports.TrySetErrno(ctx, errno);
ctx[CpuRegister.Rax] = ulong.MaxValue;
return -1;
// POSIX open(2): translates a failed raw open into -1/errno. On success
// KernelOpenUnderscore already writes the fd into RAX (the import bridge
// prefers a written RAX over the return value), so returning 0 is correct.
public static int PosixOpen(CpuContext ctx)
{
var result = KernelOpenUnderscore(ctx);
return result == (int)OrbisGen2Result.ORBIS_GEN2_OK
? 0
: PosixFailure(ctx, result);
}
// POSIX fstat(2): a bad fd maps to EBADF rather than the path-oriented ENOENT.
public static int PosixFstat(CpuContext ctx)
{
var result = KernelFstat(ctx);
return result == (int)OrbisGen2Result.ORBIS_GEN2_OK
? 0
: PosixFailure(ctx, result, notFoundErrno: Ebadf);
}
[SysAbiExport(
@@ -2096,7 +2134,15 @@ public static partial class KernelMemoryCompatExports
ExportName = "read",
Target = Generation.Gen4 | Generation.Gen5,
LibraryName = "libKernel")]
public static int PosixRead(CpuContext ctx) => KernelReadUnderscore(ctx);
public static int PosixRead(CpuContext ctx)
{
// On success KernelReadUnderscore writes the byte count into RAX, which
// the import bridge prefers over this return value.
var result = KernelReadUnderscore(ctx);
return result == (int)OrbisGen2Result.ORBIS_GEN2_OK
? 0
: PosixFailure(ctx, result, notFoundErrno: Ebadf);
}
[SysAbiExport(
Nid = "Cg4srZ6TKbU",
@@ -2295,7 +2341,15 @@ public static partial class KernelMemoryCompatExports
ExportName = "write",
Target = Generation.Gen4 | Generation.Gen5,
LibraryName = "libKernel")]
public static int PosixWrite(CpuContext ctx) => KernelWriteUnderscore(ctx);
public static int PosixWrite(CpuContext ctx)
{
// On success KernelWriteUnderscore writes the byte count into RAX, which
// the import bridge prefers over this return value.
var result = KernelWriteUnderscore(ctx);
return result == (int)OrbisGen2Result.ORBIS_GEN2_OK
? 0
: PosixFailure(ctx, result, notFoundErrno: Ebadf);
}
[SysAbiExport(
Nid = "4wSze92BhLI",
@@ -41,6 +41,90 @@ public sealed class KernelMemoryCompatExportsTests
Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
}
[Fact]
public void PosixOpen_MissingFileReturnsMinusOne()
{
const ulong memoryBase = 0x1_0000_0000;
const ulong pathAddress = memoryBase + 0x100;
var memory = new FakeCpuMemory(memoryBase, 0x1000);
var context = new CpuContext(memory, Generation.Gen5);
memory.WriteCString(pathAddress, "/__sharpemu_test_missing__/il2cpp.usym");
context[CpuRegister.Rdi] = pathAddress;
context[CpuRegister.Rsi] = 0; // O_RDONLY
var result = KernelMemoryCompatExports.PosixOpen(context);
// A libc open() failure must be -1, not the raw 0x8002xxxx sentinel the
// guest would otherwise store as a valid fd and later dereference.
Assert.Equal(-1, result);
Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
}
[Fact]
public void PosixFstat_BadDescriptorReturnsMinusOne()
{
const ulong memoryBase = 0x1_0000_0000;
const ulong statAddress = memoryBase + 0x400;
var memory = new FakeCpuMemory(memoryBase, 0x1000);
var context = new CpuContext(memory, Generation.Gen5);
context[CpuRegister.Rdi] = 0x80020002; // the not-found sentinel misused as an fd
context[CpuRegister.Rsi] = statAddress;
var result = KernelMemoryCompatExports.PosixFstat(context);
Assert.Equal(-1, result);
Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
}
[Fact]
public void PosixClose_BadDescriptorReturnsMinusOne()
{
const ulong memoryBase = 0x1_0000_0000;
var memory = new FakeCpuMemory(memoryBase, 0x1000);
var context = new CpuContext(memory, Generation.Gen5);
context[CpuRegister.Rdi] = 0x80020002; // never-opened / sentinel fd
var result = KernelMemoryCompatExports.PosixClose(context);
Assert.Equal(-1, result);
Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
}
[Fact]
public void PosixRead_BadDescriptorReturnsMinusOne()
{
const ulong memoryBase = 0x1_0000_0000;
const ulong bufferAddress = memoryBase + 0x200;
var memory = new FakeCpuMemory(memoryBase, 0x1000);
var context = new CpuContext(memory, Generation.Gen5);
context[CpuRegister.Rdi] = 0x80020002; // never-opened / sentinel fd
context[CpuRegister.Rsi] = bufferAddress;
context[CpuRegister.Rdx] = 0x40;
var result = KernelMemoryCompatExports.PosixRead(context);
Assert.Equal(-1, result);
Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
}
[Fact]
public void PosixWrite_BadDescriptorReturnsMinusOne()
{
const ulong memoryBase = 0x1_0000_0000;
const ulong bufferAddress = memoryBase + 0x200;
var memory = new FakeCpuMemory(memoryBase, 0x1000);
var context = new CpuContext(memory, Generation.Gen5);
memory.WriteCString(bufferAddress, "payload");
context[CpuRegister.Rdi] = 0x80020002; // never-opened / sentinel fd
context[CpuRegister.Rsi] = bufferAddress;
context[CpuRegister.Rdx] = 0x7;
var result = KernelMemoryCompatExports.PosixWrite(context);
Assert.Equal(-1, result);
Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
}
[Fact]
public void Sprintf_ReadsVariadicDoubleFromXmmRegister()
{
@@ -37,10 +37,11 @@ public sealed class KernelSocketCompatExportsTests
KernelMemoryCompatExports.PosixClose(context));
Assert.Equal(0UL, context[CpuRegister.Rax]);
// A second close of an already-closed fd fails per the POSIX ABI:
// -1 with errno set, not the raw Orbis NOT_FOUND sentinel.
context[CpuRegister.Rdi] = unchecked((ulong)guestFd);
Assert.Equal(
(int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND,
KernelMemoryCompatExports.PosixClose(context));
Assert.Equal(-1, KernelMemoryCompatExports.PosixClose(context));
Assert.Equal(ulong.MaxValue, context[CpuRegister.Rax]);
}
finally
{