[HLE] Fix AJM/ACM no-op stubs and accept Gen5 AudioOut2 mastering/batch calls (#807)

AjmModuleUnregister/AjmFinalize were pure no-ops (always returned success
without touching state); they now validate the context and actually remove
the codec registration / context entry, returning a real error when the
context is unknown. AjmModuleUnregister traces whether the codec was
actually registered, to help spot a title unregistering something it never
registered.

MaxCodecType was hardcoded to 25 based on known Sony codec ids, incorrectly
rejecting valid Gen5 codec types (e.g. 24). Registration is pure bookkeeping
(HashSet.Add); the only real constraint is that codecType must not overflow
the 32-bit instanceId it gets packed into, i.e. codecType < 2^18. Named the
instanceId bit-packing constants (InstanceIdSlotBits/InstanceIdSlotMask) so
the four call sites that used to hardcode 14/0x3FFF independently can't
drift out of sync, and MaxCodecType's formula is self-evident.

Accept sceAcmBatchInitialize/InitializeLite/Start/StartMultiple/Process as
successful no-ops -- the emulator runs no ACM DSP jobs, but Scream's workers
trap on int 0x41/0x42 asserts whenever a submission call reports failure.

Accept sceAudioOut2MasteringInit/Set3DLatency as no-ops -- the host mixer
has no mastering/object pipeline to tune, but returning failure makes
titles tear down their whole ACM context and abort audio arena bring-up.

Tested on Ghost of Yotei (PPSA26344): sceAudioOut2MasteringInit/Set3DLatency
unresolved-import warnings go from 2 to 0, unblocking 5 audio-related guest
threads that never spawned before (MovieDecoder, snd_stream_parsing_thread,
snd_stream_reader_thread, Psn, NCA::PumpThread), watchdog stall errors from
1 to 0. This alone gets the title past its Scream audio-init stall into
further boot, but it still hits an unrelated infinite retry loop in
sceVideoOutGetFlipStatus shortly after -- fixed by the separate
videoout-agc-hle-misc change; combined, the title renders and presents a
real GPU frame.

Regression-checked on Demon's Souls, Astro Bot, Outer Wilds, Cult of the
Lamb, Ghost of Tsushima, GTA, Minecraft, Quake: 0 calls to any export
touched by this change on any of them, byte-for-byte identical to
upstream/main.

src/SharpEmu.Libs/Acm/AcmExports.cs

@@ -99,6 +99,50 @@ public static class AcmExports
         return CompleteBatchStart(ctx, context, infoCount, errorAddress, batchAddress);
     }
 
+    // DSP batch submission and synchronization. The emulator runs no ACM DSP
+    // jobs (FFT/panner/reverb output stays silent), but Scream's workers trap
+    // with int 0x41/0x42 asserts whenever a submission call reports failure,
+    // so the whole batch surface must report success.
+    [SysAbiExport(
+        Nid = "WeZOIm8+8WI",
+        ExportName = "sceAcmBatchInitialize",
+        Target = Generation.Gen4 | Generation.Gen5,
+        LibraryName = "libSceAcm")]
+    public static int AcmBatchInitialize(CpuContext ctx) =>
+        ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK);
+
+    [SysAbiExport(
+        Nid = "Mk1xvQXIdkk",
+        ExportName = "sceAcmBatchInitializeLite",
+        Target = Generation.Gen4 | Generation.Gen5,
+        LibraryName = "libSceAcm")]
+    public static int AcmBatchInitializeLite(CpuContext ctx) =>
+        ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK);
+
+    [SysAbiExport(
+        Nid = "A5NXCXK5Gfc",
+        ExportName = "sceAcmBatchStart",
+        Target = Generation.Gen4 | Generation.Gen5,
+        LibraryName = "libSceAcm")]
+    public static int AcmBatchStart(CpuContext ctx) =>
+        ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK);
+
+    [SysAbiExport(
+        Nid = "S3BPrjCfZ90",
+        ExportName = "sceAcmBatchStartMultiple",
+        Target = Generation.Gen4 | Generation.Gen5,
+        LibraryName = "libSceAcm")]
+    public static int AcmBatchStartMultiple(CpuContext ctx) =>
+        ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK);
+
+    [SysAbiExport(
+        Nid = "uqDIauipRbo",
+        ExportName = "sceAcmBatchProcess",
+        Target = Generation.Gen4 | Generation.Gen5,
+        LibraryName = "libSceAcm")]
+    public static int AcmBatchProcess(CpuContext ctx) =>
+        ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK);
+
     [SysAbiExport(
         Nid = "RLN3gRlXJLE",
         ExportName = "sceAcmBatchWait",

src/SharpEmu.Libs/Audio/AjmExports.cs

@@ -21,7 +21,18 @@ public static class AjmExports
     private const int OrbisAjmErrorJobCreation = unchecked((int)0x80930012);
     private const ulong MaxSilentPcmBytes = 1 << 20;
     private const uint Atrac9CodecType = 1;
-    private const uint MaxCodecType = 25;
+    // instanceId packs codecType into the high bits and the instance slot
+    // into the low InstanceIdSlotBits bits (see AjmInstanceCreate's
+    // `(codecType << InstanceIdSlotBits) | instanceSlot` and the
+    // `& InstanceIdSlotMask` unpacks in AjmInstanceDestroy/GetError).
+    private const int InstanceIdSlotBits = 14;
+    private const uint InstanceIdSlotMask = (1u << InstanceIdSlotBits) - 1;
+    // Registration is pure bookkeeping (a HashSet.Add), so the only real
+    // constraint is that codecType must not overflow the 32-bit instanceId
+    // once shifted left by InstanceIdSlotBits -- not any hardcoded list of
+    // known Sony codec ids, which a retail title's Gen5 codec type (e.g. 24)
+    // can legitimately fall outside of.
+    private const uint MaxCodecType = 1u << (32 - InstanceIdSlotBits);
     private const int MaxInstanceIndex = 0x2FFF;
     private const int MaxDecodeBufferBytes = 64 * 1024 * 1024;
 
@@ -119,9 +130,12 @@ public static class AjmExports
         LibraryName = "libSceAjm")]
     public static int AjmFinalize(CpuContext ctx)
     {
-        Contexts.TryRemove(unchecked((uint)ctx[CpuRegister.Rdi]), out _);
+        if (!Contexts.TryRemove(unchecked((uint)ctx[CpuRegister.Rdi]), out _))
-        ctx[CpuRegister.Rax] = 0;
+        {
-        return 0;
+            return ctx.SetReturn(OrbisAjmErrorInvalidContext);
+        }
+
+        return ctx.SetReturn(0);
     }
 
     [SysAbiExport(
@@ -273,7 +287,7 @@ public static class AjmExports
             }
             while (state.InstancesBySlot.ContainsKey(instanceSlot));
 
-            instanceId = (codecType << 14) | instanceSlot;
+            instanceId = (codecType << InstanceIdSlotBits) | instanceSlot;
             Span<byte> value = stackalloc byte[sizeof(uint)];
             BinaryPrimitives.WriteUInt32LittleEndian(value, instanceId);
             if (!ctx.Memory.TryWrite(outputAddress, value))
@@ -314,7 +328,7 @@ public static class AjmExports
             return ctx.SetReturn(OrbisAjmErrorInvalidContext);
         }
 
-        var instanceSlot = instanceId & 0x3FFF;
+        var instanceSlot = instanceId & InstanceIdSlotMask;
         lock (state.Gate)
         {
             if (instanceSlot == 0 || !state.InstancesBySlot.Remove(instanceSlot))
@@ -334,8 +348,21 @@ public static class AjmExports
         LibraryName = "libSceAjm")]
     public static int AjmModuleUnregister(CpuContext ctx)
     {
-        ctx[CpuRegister.Rax] = 0;
+        var contextId = unchecked((uint)ctx[CpuRegister.Rdi]);
-        return 0;
+        var codecType = unchecked((uint)ctx[CpuRegister.Rsi]);
+        if (!Contexts.TryGetValue(contextId, out var state))
+        {
+            return ctx.SetReturn(OrbisAjmErrorInvalidContext);
+        }
+
+        bool removed;
+        lock (state.Gate)
+        {
+            removed = state.RegisteredCodecs.Remove(codecType);
+        }
+
+        Trace($"module_unregister context={contextId} codec={codecType} was_registered={removed}");
+        return ctx.SetReturn(0);
     }
 
     [SysAbiExport(
@@ -667,8 +694,8 @@ public static class AjmExports
     private static bool TryGetInstance(uint instanceId, out AjmInstanceState instance)
     {
         instance = null!;
-        var codec = instanceId >> 14;
+        var codec = instanceId >> InstanceIdSlotBits;
-        var slot = instanceId & 0x3FFF;
+        var slot = instanceId & InstanceIdSlotMask;
         if (slot == 0)
         {
             return false;

src/SharpEmu.Libs/Audio/AudioOut2Exports.cs

@@ -163,6 +163,33 @@ public static class AudioOut2Exports
         return (int)OrbisGen2Result.ORBIS_GEN2_OK;
     }
 
+    // Ghost of Yotei calls this with flags=0 during Scream startup and never
+    // checks the result before continuing into its mastering path; the actual
+    // mastering chain lives in the host mixer, so accepting the request is
+    // sufficient.
+    [SysAbiExport(
+        Nid = "XHl38ZNknbs",
+        ExportName = "sceAudioOut2MasteringInit",
+        Target = Generation.Gen5,
+        LibraryName = "libSceAudioOut2")]
+    public static int AudioOut2MasteringInit(CpuContext ctx)
+    {
+        return SetReturn(ctx, 0);
+    }
+
+    // 3D-audio object latency hint; the host mixer has no object pipeline to
+    // tune, but failure here makes Yotei tear down its whole ACM context and
+    // abort audio arena bring-up.
+    [SysAbiExport(
+        Nid = "TViD1EZXkNI",
+        ExportName = "sceAudioOut2Set3DLatency",
+        Target = Generation.Gen5,
+        LibraryName = "libSceAudioOut2")]
+    public static int AudioOut2Set3DLatency(CpuContext ctx)
+    {
+        return SetReturn(ctx, 0);
+    }
+
     [SysAbiExport(
         Nid = "t5YrizufpQc",
         ExportName = "sceAudioOut2ContextResetParam",

tests/SharpEmu.Libs.Tests/Audio/AjmExportsTests.cs

@@ -83,6 +83,30 @@ public sealed class AjmExportsTests : IDisposable
         Assert.Equal(InvalidContext, RegisterCodec(contextId + 1, 1));
     }
 
+    [Fact]
+    public void ModuleUnregister_RemovesRegisteredCodecAndRejectsUnknownContext()
+    {
+        var contextId = Initialize();
+
+        Assert.Equal(0, RegisterCodec(contextId, 1));
+        Assert.Equal(0, UnregisterCodec(contextId, 1));
+        // The codec is actually gone, not just a no-op stub: it's unusable
+        // for a new instance, and re-registering no longer hits
+        // CodecAlreadyRegistered.
+        Assert.Equal(CodecNotRegistered, CreateInstance(contextId, 1, 0x401, InstanceAddress));
+        Assert.Equal(0, RegisterCodec(contextId, 1));
+
+        Assert.Equal(InvalidContext, UnregisterCodec(contextId + 1, 1));
+    }
+
+    [Fact]
+    public void ModuleUnregister_UnknownCodecIsToleratedAsANoOp()
+    {
+        var contextId = Initialize();
+
+        Assert.Equal(0, UnregisterCodec(contextId, 1));
+    }
+
     [Fact]
     public void MemoryRegistration_TracksValidContextAndToleratesRepeatedUnregister()
     {
@@ -353,6 +377,13 @@ public sealed class AjmExportsTests : IDisposable
         return AjmExports.AjmModuleRegister(_ctx);
     }
 
+    private int UnregisterCodec(uint contextId, uint codecType)
+    {
+        _ctx[CpuRegister.Rdi] = contextId;
+        _ctx[CpuRegister.Rsi] = codecType;
+        return AjmExports.AjmModuleUnregister(_ctx);
+    }
+
     private int RegisterMemory(uint contextId, ulong address, ulong pages)
     {
         _ctx[CpuRegister.Rdi] = contextId;