[AGC] Fix arena/fence tracking stalls and recover lost GPU progress (#770)

* [AGC] Fix arena/fence tracking stalls and recover lost GPU progress

Fix several AGC synchronization issues that could leave GPU work
unsubmitted or fences permanently unsatisfied:

- Recover fence writes missed during arena transitions by tracking
  closed arena tails and resweeping unresolved fence locations.
- Fix chunk submission accounting so only actually parsed ranges are
  considered submitted, avoiding missed orphan recovery.
- Extend arena sweeps to follow chained command ranges without
  duplicate submissions.
- Preserve GPU state tracking across native worker memory wrappers by
  using canonical guest memory identity.
- Fix builder-ring orphan submissions and stale ring-tail waits.
- Improve frame cycling by handling arena reuse, cursor regression,
  and transiently unavailable builder headers.
- Record conditional DCB execution (IT_COND_EXEC) instead of
  silently dropping the packet.

These changes improve AGC progress tracking and prevent GPU fence
starvation/deadlocks on titles relying on builder arenas and indirect
command chains.

* chore: trigger CI

.gitignore

@@ -42,4 +42,6 @@ ehthumbs.db
 
 .vs/
 .idea/
-.vscode/
+.vscode/
+clean_test/
+.debug/

Directory.Build.props

@@ -9,7 +9,7 @@ SPDX-License-Identifier: GPL-2.0-or-later
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
-    <SharpEmuVersion>0.0.3-hotfix-2</SharpEmuVersion>
+    <SharpEmuVersion>0.0.3-release.2</SharpEmuVersion>
     <Version>$(SharpEmuVersion)</Version>
 
     <RepoRoot>$([MSBuild]::NormalizeDirectory('$(MSBuildThisFileDirectory)'))</RepoRoot>

src/SharpEmu.Libs/Agc/GpuWaitRegistry.cs

@@ -60,6 +60,19 @@ internal static class GpuWaitRegistry
     // address) so distinct guest processes never alias.
     private static readonly Dictionary<(object, ulong), ulong> _lastProduced = new();
 
+    // Unwraps to the shared root: per-thread TrackedCpuMemory decorators
+    // over ONE virtual memory are not reference-equal, so a raw-reference
+    // filter would make waits invisible across threads.
+    private static object? Canonicalize(object? memory)
+    {
+        while (memory is SharpEmu.HLE.ICpuMemoryWrapper wrapper)
+        {
+            memory = wrapper.Inner;
+        }
+
+        return memory;
+    }
+
     public static int Count
     {
         get
@@ -79,6 +92,7 @@ internal static class GpuWaitRegistry
 
     public static int CountForMemory(object memory)
     {
+        memory = Canonicalize(memory)!;
         lock (_gate)
         {
             var total = 0;
@@ -155,6 +169,7 @@ internal static class GpuWaitRegistry
     public static void Register(ulong address, WaitingDcb waiter)
     {
         waiter.WaitAddress = address;
+        waiter.Memory = Canonicalize(waiter.Memory);
         lock (_gate)
         {
             if (!_waiters.TryGetValue(address, out var list))
@@ -177,6 +192,7 @@ internal static class GpuWaitRegistry
         object memory,
         Func<ulong, bool, ulong?> readValue)
     {
+        memory = Canonicalize(memory)!;
         List<WaitingDcb>? woken = null;
         lock (_gate)
         {
@@ -237,6 +253,7 @@ internal static class GpuWaitRegistry
         long nowTicks,
         long maxAgeTicks)
     {
+        memory = Canonicalize(memory)!;
         List<WaitingDcb>? stale = null;
         lock (_gate)
         {
@@ -273,6 +290,7 @@ internal static class GpuWaitRegistry
         ulong start,
         ulong length)
     {
+        memory = Canonicalize(memory)!;
         var matches = new List<(ulong Address, int Count)>();
         if (length == 0)
         {
@@ -355,6 +373,56 @@ internal static class GpuWaitRegistry
         return latchedAny;
     }
 
+    /// <summary>
+    /// Every registered waiter, for the flip-stall watchdog. Not filtered by
+    /// memory identity — the watchdog wants a whole-process view.
+    /// </summary>
+    public static List<WaitingDcb> SnapshotAll()
+    {
+        var snapshot = new List<WaitingDcb>();
+        lock (_gate)
+        {
+            foreach (var (_, list) in _waiters)
+            {
+                snapshot.AddRange(list);
+            }
+        }
+
+        return snapshot;
+    }
+
+    /// <summary>
+    /// Removes the waiter at <paramref name="address"/> whose State is
+    /// <paramref name="state"/> — used when a new submission supersedes a
+    /// ring-tail park that would otherwise pin the queue forever.
+    /// </summary>
+    public static bool TryRemoveByState(object state, ulong address)
+    {
+        lock (_gate)
+        {
+            if (!_waiters.TryGetValue(address, out var list))
+            {
+                return false;
+            }
+
+            for (var i = list.Count - 1; i >= 0; i--)
+            {
+                if (ReferenceEquals(list[i].State, state))
+                {
+                    list.RemoveAt(i);
+                    if (list.Count == 0)
+                    {
+                        _waiters.Remove(address);
+                    }
+
+                    return true;
+                }
+            }
+
+            return false;
+        }
+    }
+
     /// <summary>
     /// Removes and returns waiters carrying a <see cref="WaitingDcb.RetryDeadlineTicks"/>
     /// that has elapsed. Used for indirect-dispatch dimension retries: the caller
@@ -564,6 +632,21 @@ internal static class GpuWaitRegistry
         return broken;
     }
 
+    // Under orphan force-submit, producers can run ahead of waiter
+    // registration and pass an equal-compare value before it's ever seen.
+    // Treat == as "reached or passed" only in that mode, so other titles
+    // keep exact hardware semantics. SHARPEMU_GPU_WAIT_EQ_EXACT=1 restores
+    // strict equality for A/B.
+    private static readonly bool _equalCompareExact =
+        string.Equals(
+            Environment.GetEnvironmentVariable("SHARPEMU_GPU_WAIT_EQ_EXACT"),
+            "1",
+            StringComparison.Ordinal) ||
+        !string.Equals(
+            Environment.GetEnvironmentVariable("SHARPEMU_FORCE_SUBMIT_ORPHAN_PREAMBLES"),
+            "1",
+            StringComparison.Ordinal);
+
     public static bool Compare(in WaitingDcb waiter, ulong value)
     {
         var masked = value & waiter.Mask;
@@ -573,7 +656,7 @@ internal static class GpuWaitRegistry
             0 => true,
             1 => masked < reference,
             2 => masked <= reference,
-            3 => masked == reference,
+            3 => _equalCompareExact ? masked == reference : masked >= reference,
             4 => masked != reference,
             5 => masked >= reference,
             6 => masked > reference,

src/SharpEmu.Libs/Audio/AudioOut2Exports.cs

@@ -921,6 +921,7 @@ public static class AudioOut2Exports
 
                 if (mixedPorts == 0)
                 {
+                    TraceSubmitSkipped(context, frames, "no-ports");
                     return false;
                 }
 
@@ -942,6 +943,7 @@ public static class AudioOut2Exports
                 var backend = ResolveContextBackend(context, out var backendName);
                 if (backend is null)
                 {
+                    TraceSubmitSkipped(context, frames, "no-backend");
                     return false;
                 }
 
@@ -1212,4 +1214,14 @@ public static class AudioOut2Exports
             Console.Error.WriteLine($"[LOADER][TRACE] audio_out2.{message}");
         }
     }
+
+    private static void TraceSubmitSkipped(ContextState context, int frames, string reason)
+    {
+        var n = Interlocked.Increment(ref _submitSkipTraceCount);
+        if (n <= 8 || n % 500 == 0)
+        {
+            TraceAudioOut2(
+                $"context-submit-skip#{n} handle=0x{context.Handle:X} frames={frames} reason={reason}");
+        }
+    }
 }

src/SharpEmu.Libs/Gpu/GuestDataPool.cs

@@ -23,6 +23,10 @@ internal static class GuestDataPool
 
     public static void Trim() => ((BoundedByteArrayPool)Shared).Trim();
 
+    /// <summary>Outstanding lease count and idle cached bytes, for leak diagnostics.</summary>
+    public static (int LeaseCount, ulong CachedBytes) DiagnosticStats() =>
+        ((BoundedByteArrayPool)Shared).Stats();
+
     private sealed class BoundedByteArrayPool : ArrayPool<byte>
     {
         private readonly object _gate = new();
@@ -119,6 +123,14 @@ internal static class GuestDataPool
             }
         }
 
+        public (int LeaseCount, ulong CachedBytes) Stats()
+        {
+            lock (_gate)
+            {
+                return (_leases.Count, _cachedBytes);
+            }
+        }
+
         private int GetAllocationLength(int minimumLength)
         {
             if (minimumLength <= 16)

src/SharpEmu.Libs/Kernel/KernelExports.cs

@@ -466,4 +466,45 @@ public static class KernelExports
         ctx[CpuRegister.Rax] = unchecked((ulong)(-1L));
         return (int)OrbisGen2Result.ORBIS_GEN2_OK;
     }
+
+    [SysAbiExport(
+    Nid = "tU5e3f9gSiU",
+    ExportName = "sceKernelIsTrinityMode",
+    Target = Generation.Gen4 | Generation.Gen5,
+    LibraryName = "libKernel")]
+    public static int KernelIsTrinityMode(CpuContext ctx)
+    {
+        ctx[CpuRegister.Rax] = 0;
+
+        return (int)OrbisGen2Result.ORBIS_GEN2_OK;
+    }
+
+    [SysAbiExport(
+    Nid = "DLORcroUqbc",
+    ExportName = "sceKernelGetOpenPsId",
+    Target = Generation.Gen4 | Generation.Gen5,
+    LibraryName = "libKernel")]
+    public static int KernelGetOpenPsId(CpuContext ctx)
+    {
+        ulong bufferPtr = ctx[CpuRegister.Rdi];
+
+        if (bufferPtr == 0)
+        {
+            ctx[CpuRegister.Rax] = unchecked((ulong)(int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT);
+
+            return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT;
+        }
+
+        Span<byte> openPsId = stackalloc byte[16];
+
+        if (!ctx.Memory.TryWrite(bufferPtr, openPsId))
+        {
+            ctx[CpuRegister.Rax] = unchecked((ulong)(int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT);
+            return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT;
+        }
+
+        ctx[CpuRegister.Rax] = 0;
+
+        return (int)OrbisGen2Result.ORBIS_GEN2_OK;
+    }
 }

src/SharpEmu.Libs/Np/NpTrophy2Exports.cs

@@ -99,6 +99,15 @@ public static class NpTrophy2Exports
     public static int NpTrophy2GetTrophyInfo(CpuContext ctx) =>
         SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND);
 
+    [SysAbiExport(
+        Nid = "y3zHpdZO6ME",
+        ExportName = "sceNpTrophy2GetTrophyInfoArray",
+        Target = Generation.Gen5,
+        LibraryName = "libSceNpTrophy2")]
+    public static int NpTrophy2GetTrophyInfoArray(CpuContext ctx) =>
+        SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND);
+
+
     private static int WriteIdAndReturn(CpuContext ctx, ulong outAddress, ref int nextId)
     {
         if (outAddress == 0)

src/SharpEmu.Libs/SharpEmu.Libs.csproj

@@ -24,6 +24,7 @@ SPDX-License-Identifier: GPL-2.0-or-later
 
   <ItemGroup>
     <InternalsVisibleTo Include="SharpEmu.Libs.Tests" />
+    <!-- SharpEmu.Core's stall watchdog reads GpuWaitRegistry for diagnostics. -->
     <InternalsVisibleTo Include="SharpEmu.Core" />
   </ItemGroup>
 

src/SharpEmu.Libs/VideoOut/VideoOutExports.cs

@@ -1300,10 +1300,12 @@ public static class VideoOutExports
         var submitted = Interlocked.Exchange(ref _submittedFrameCount, 0);
         var presentedCount = Interlocked.Exchange(ref _presentedFrameCount, 0);
         var (draws, drawMs, pipelines, spirvCompiles) = GuestGpu.Current.ReadAndResetPerfCounters();
+        var (poolLeases, poolCachedBytes) = SharpEmu.Libs.Gpu.GuestDataPool.DiagnosticStats();
         Console.Error.WriteLine(
             $"[LOADER][PERF] videoout submitted_fps={submitted / elapsedSeconds:F1} " +
             $"presented_fps={presentedCount / elapsedSeconds:F1} " +
-            $"draws={draws} draw_ms={drawMs:F0} pipelines={pipelines} spirv={spirvCompiles}");
+            $"draws={draws} draw_ms={drawMs:F0} pipelines={pipelines} spirv={spirvCompiles} " +
+            $"pool_leases={poolLeases} pool_cached_mb={poolCachedBytes / 1024.0 / 1024.0:F1}");
     }
 
     private static readonly bool _flipPacingDisabled = string.Equals(