[shader decoder] correct RDNA2 operands, fixing synchronization problems

src/SharpEmu.Libs/Agc/AgcExports.cs

@@ -77,8 +77,11 @@ public static class AgcExports
     private const uint CbColor0Attrib3 = 0x3B8;
     private const int ColorTargetCount = 8;
     private const uint PsTextureUserDataRegister = 0xC;
+    private const uint VsUserDataRegister = 0x4C;
+    private const uint GsUserDataRegister = 0x8C;
     private const uint EsUserDataRegister = 0xCC;
     private const uint ComputeUserDataRegister = 0x240;
+    private const uint NggUserDataScalarRegisterBase = 8;
     private const uint Gen5TextureFormatR8G8B8A8Unorm = 56;
     private const uint Gen5TextureFormatR16G16B16A16Float = 71;
     private const uint Gen5TextureType2D = 9;
@@ -2873,9 +2876,10 @@ public static class AgcExports
                 exportShaderAddress,
                 exportShaderHeader,
                 state.ShRegisters,
-                EsUserDataRegister,
+                SelectExportUserDataRegister(state.ShRegisters),
                 out var exportState,
-                out error) ||
+                out error,
+                userDataScalarRegisterBase: NggUserDataScalarRegisterBase) ||
             !Gen5ShaderScalarEvaluator.TryEvaluate(
                 ctx,
                 exportState,
@@ -2904,11 +2908,13 @@ public static class AgcExports
                 HasPixelColorExport(pixelState, target.Slot))
             .ToArray();
         var outputKind = GetPixelOutputKind(renderTargets.FirstOrDefault().NumberType);
+        var exportStateFingerprint = ComputeShaderStateFingerprint(exportEvaluation);
+        var pixelStateFingerprint = ComputeShaderStateFingerprint(pixelEvaluation);
         var shaderKey = (
             exportShaderAddress,
-            ComputeShaderStateFingerprint(exportEvaluation),
+            exportStateFingerprint,
             pixelShaderAddress,
-            ComputeShaderStateFingerprint(pixelEvaluation),
+            pixelStateFingerprint,
             outputKind);
         (byte[] Vertex, byte[] Pixel) compiled;
         lock (_submitTraceGate)
@@ -2943,6 +2949,18 @@ public static class AgcExports
             }
 
             compiled = (vertexShader.Spirv, pixelShader.Spirv);
+            DumpSpirv(
+                "vs",
+                exportShaderAddress,
+                exportStateFingerprint,
+                compiled.Vertex,
+                exportState.Program);
+            DumpSpirv(
+                "ps",
+                pixelShaderAddress,
+                pixelStateFingerprint,
+                compiled.Pixel,
+                pixelState.Program);
             lock (_submitTraceGate)
             {
                 _graphicsSpirvCache.TryAdd(shaderKey, compiled);
@@ -3004,7 +3022,9 @@ public static class AgcExports
         var byteOffset = checked((ulong)state.DrawIndexOffset * (uint)bytesPerIndex);
         var byteCount = checked((int)(indexCount * (uint)bytesPerIndex));
         var data = new byte[byteCount];
-        return ctx.Memory.TryRead(state.IndexBufferAddress + byteOffset, data)
+        var address = state.IndexBufferAddress + byteOffset;
+        return (ctx.Memory.TryRead(address, data) ||
+                KernelMemoryCompatExports.TryReadTrackedLibcHeap(address, data))
             ? new VulkanGuestIndexBuffer(data, is32Bit)
             : null;
     }
@@ -3144,12 +3164,22 @@ public static class AgcExports
                     $"fmt{texture.Format}/num{texture.NumberType}/tile{texture.TileMode}" +
                     $"/storage={binding.IsStorage}{target}/{probe}{writer}";
             }));
+        var buffers = string.Join(
+            ',',
+            draw.GlobalMemoryBindings.Select((binding, index) =>
+                $"{index}:0x{binding.BaseAddress:X16}:{binding.Data.Length}:" +
+                Convert.ToHexString(binding.Data.AsSpan(0, Math.Min(binding.Data.Length, 32)))));
+        var indices = draw.IndexBuffer is { } indexBuffer
+            ? $"{(indexBuffer.Is32Bit ? 32 : 16)}:" +
+              Convert.ToHexString(indexBuffer.Data.AsSpan(0, Math.Min(indexBuffer.Data.Length, 32)))
+            : "none";
         TraceAgcShader(
             $"agc.shader_draw es=0x{draw.ExportShaderAddress:X16} " +
             $"ps=0x{draw.PixelShaderAddress:X16} spirv={draw.PixelSpirv.Length} " +
             $"primitive=0x{draw.PrimitiveType:X} " +
             $"ps_ena=0x{psInputEna:X8} ps_addr=0x{psInputAddr:X8} " +
-            $"targets=[{targets}] textures=[{textures}]");
+            $"targets=[{targets}] textures=[{textures}] " +
+            $"buffers=[{buffers}] indices=[{indices}]");
     }
 
     private static IReadOnlyList<VulkanGuestDrawTexture> CreateVulkanGuestDrawTextures(
@@ -3511,6 +3541,12 @@ public static class AgcExports
                     out computeError))
             {
                 computeSpirv = compiledCompute.Spirv;
+                DumpSpirv(
+                    "cs",
+                    shaderAddress,
+                    shaderKey.Item2,
+                    computeSpirv,
+                    shaderState.Program);
             }
 
             if (computeSpirv is not null)
@@ -3527,6 +3563,7 @@ public static class AgcExports
                 var globalMemoryBuffers =
                     CreateVulkanGuestMemoryBuffers(evaluation.GlobalMemoryBindings);
                 VulkanVideoPresenter.SubmitComputeDispatch(
+                    shaderAddress,
                     computeSpirv,
                     textures,
                     globalMemoryBuffers,
@@ -3601,6 +3638,62 @@ public static class AgcExports
     private static string DescribeRegister(uint? register) =>
         register.HasValue ? $"s{register.Value}" : "-";
 
+    private static uint SelectExportUserDataRegister(
+        IReadOnlyDictionary<uint, uint> registers)
+    {
+        if (HasUserDataRange(registers, GsUserDataRegister))
+        {
+            return GsUserDataRegister;
+        }
+
+        if (HasUserDataRange(registers, EsUserDataRegister))
+        {
+            return EsUserDataRegister;
+        }
+
+        if (HasUserDataRange(registers, VsUserDataRegister))
+        {
+            return VsUserDataRegister;
+        }
+
+        var esValues = CountUserDataValues(registers, EsUserDataRegister);
+        var vsValues = CountUserDataValues(registers, VsUserDataRegister);
+        return esValues == 0 && vsValues != 0
+            ? VsUserDataRegister
+            : EsUserDataRegister;
+    }
+
+    private static bool HasUserDataRange(
+        IReadOnlyDictionary<uint, uint> registers,
+        uint startRegister)
+    {
+        for (var index = 0u; index < 16; index++)
+        {
+            if (registers.ContainsKey(startRegister + index))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private static int CountUserDataValues(
+        IReadOnlyDictionary<uint, uint> registers,
+        uint startRegister)
+    {
+        var count = 0;
+        for (var index = 0u; index < 16; index++)
+        {
+            count += registers.TryGetValue(startRegister + index, out var value) &&
+                     value != 0
+                ? 1
+                : 0;
+        }
+
+        return count;
+    }
+
     private static uint GetComputeLocalSize(
         IReadOnlyDictionary<uint, uint> registers,
         uint register)
@@ -3798,7 +3891,31 @@ public static class AgcExports
             }
         }
 
-        return $"probe={reads}/{sampleCount}:{nonzero}:0x{hash:X16}";
+        var bytesPerTexel = GetTextureBytesPerTexel(texture.Format);
+        var texels = bytesPerTexel is > 0 and <= 16
+            ? string.Join(
+                '/',
+                ProbeTextureTexel(ctx, texture.Address, (int)bytesPerTexel),
+                ProbeTextureTexel(
+                    ctx,
+                    texture.Address +
+                    (((ulong)(texture.Height / 2) * texture.Width) + (texture.Width / 2)) *
+                    bytesPerTexel,
+                    (int)bytesPerTexel),
+                ProbeTextureTexel(
+                    ctx,
+                    texture.Address + totalBytes - bytesPerTexel,
+                    (int)bytesPerTexel))
+            : "unsupported";
+        return $"probe={reads}/{sampleCount}:{nonzero}:0x{hash:X16}:texels={texels}";
+    }
+
+    private static string ProbeTextureTexel(CpuContext ctx, ulong address, int size)
+    {
+        var texel = new byte[size];
+        return ctx.Memory.TryRead(address, texel)
+            ? Convert.ToHexString(texel)
+            : "unreadable";
     }
 
     private static ulong GetTextureBytesPerTexel(uint format) =>
@@ -3912,9 +4029,10 @@ public static class AgcExports
                         exportShaderAddress,
                         exportShaderHeader,
                         state.ShRegisters,
-                        EsUserDataRegister,
+                        SelectExportUserDataRegister(state.ShRegisters),
                         out var exportState,
-                        out _) &&
+                        out _,
+                        userDataScalarRegisterBase: NggUserDataScalarRegisterBase) &&
                     Gen5ShaderTranslator.TryCreateState(
                         ctx,
                         pixelShaderAddress,
@@ -4882,6 +5000,46 @@ public static class AgcExports
             ? "none"
             : string.Join(',', values.Select(static value => $"{value:X8}"));
 
+    private static void DumpSpirv(
+        string stage,
+        ulong shaderAddress,
+        ulong stateFingerprint,
+        byte[] spirv,
+        Gen5ShaderProgram program)
+    {
+        if (spirv.Length == 0 ||
+            !string.Equals(
+                Environment.GetEnvironmentVariable("SHARPEMU_DUMP_SPIRV"),
+                "1",
+                StringComparison.Ordinal))
+        {
+            return;
+        }
+
+        var directory = Path.Combine(AppContext.BaseDirectory, "shader-dumps");
+        Directory.CreateDirectory(directory);
+        var name = $"{shaderAddress:X16}-{stateFingerprint:X16}.{stage}";
+        File.WriteAllBytes(Path.Combine(directory, $"{name}.spv"), spirv);
+
+        var lines = new List<string>(program.Instructions.Count + 2)
+        {
+            $"address=0x{program.Address:X16}",
+            "pc words opcode destinations <- sources control",
+        };
+        foreach (var instruction in program.Instructions)
+        {
+            lines.Add(
+                $"0x{instruction.Pc:X4} " +
+                $"{string.Join('_', instruction.Words.Select(static word => $"{word:X8}"))} " +
+                $"{instruction.Opcode} " +
+                $"{string.Join(',', instruction.Destinations)} <- " +
+                $"{string.Join(',', instruction.Sources)} " +
+                $"{instruction.Control}");
+        }
+
+        File.WriteAllLines(Path.Combine(directory, $"{name}.ir.txt"), lines);
+    }
+
     private static void TraceCreateShader(ulong destinationAddress, ulong headerAddress, ulong codeAddress, string detail)
     {
         var isOk = string.Equals(detail, "ok", StringComparison.Ordinal);

src/SharpEmu.Libs/Agc/Gen5ShaderIr.cs

@@ -131,7 +131,8 @@ internal sealed record Gen5ShaderState(
     Gen5ShaderProgram Program,
     IReadOnlyList<uint> UserData,
     Gen5ShaderMetadata? Metadata,
-    Gen5ComputeSystemRegisters? ComputeSystemRegisters = null);
+    Gen5ComputeSystemRegisters? ComputeSystemRegisters = null,
+    uint UserDataScalarRegisterBase = 0);
 
 internal readonly record struct Gen5Operand(Gen5OperandKind Kind, uint Value)
 {

src/SharpEmu.Libs/Agc/Gen5ShaderScalarEvaluator.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 using SharpEmu.HLE;
+using SharpEmu.Libs.Kernel;
 using System.Buffers.Binary;
 using System.Numerics;
 
@@ -45,9 +46,13 @@ internal static class Gen5ShaderScalarEvaluator
         evaluation = default!;
         error = string.Empty;
         var scalarRegisters = new uint[ScalarRegisterCount];
-        for (var index = 0; index < state.UserData.Count && index < scalarRegisters.Length; index++)
+        for (var index = 0;
+             index < state.UserData.Count &&
+             state.UserDataScalarRegisterBase + (uint)index < scalarRegisters.Length;
+             index++)
         {
-            scalarRegisters[index] = state.UserData[index];
+            scalarRegisters[state.UserDataScalarRegisterBase + (uint)index] =
+                state.UserData[index];
         }
 
         if (state.ComputeSystemRegisters is { } computeSystemRegisters)
@@ -259,10 +264,16 @@ internal static class Gen5ShaderScalarEvaluator
                             bufferDescriptor.SizeBytes,
                             out var data))
                     {
+                        var descriptorWords = string.Join(
+                            ':',
+                            Enumerable.Range(0, 4).Select(index =>
+                                $"{scalarRegisters[bufferMemory.ScalarResource + (uint)index]:X8}"));
                         error =
                             $"buffer-memory-read-failed pc=0x{instruction.Pc:X} " +
                             $"address=0x{bufferDescriptor.BaseAddress:X16} " +
-                            $"bytes={bufferDescriptor.SizeBytes}";
+                            $"bytes={bufferDescriptor.SizeBytes} " +
+                            $"stride={bufferDescriptor.Stride} records={bufferDescriptor.NumRecords} " +
+                            $"s{bufferMemory.ScalarResource}=[{descriptorWords}]";
                         return false;
                     }
 
@@ -471,12 +482,24 @@ internal static class Gen5ShaderScalarEvaluator
             return false;
         }
 
-        data = GC.AllocateUninitializedArray<byte>((int)cappedSize);
+        var candidateSize = (int)cappedSize;
-        if (ctx.Memory.TryRead(baseAddress, data))
+        while (candidateSize >= sizeof(uint))
+        {
+            data = GC.AllocateUninitializedArray<byte>(candidateSize);
+            if (ctx.Memory.TryRead(baseAddress, data) ||
+                KernelMemoryCompatExports.TryReadTrackedLibcHeap(baseAddress, data))
             {
                 return true;
             }
 
+            if (candidateSize == sizeof(uint))
+            {
+                break;
+            }
+
+            candidateSize = Math.Max(candidateSize / 2, sizeof(uint));
+        }
+
         data = [];
         return false;
     }
@@ -582,6 +605,49 @@ internal static class Gen5ShaderScalarEvaluator
             return true;
         }
 
+        if (instruction.Opcode is "SBfeU64" or "SBfeI64")
+        {
+            if (instruction.Sources.Count < 2 ||
+                destination.Value >= ScalarRegisterCount - 1 ||
+                !TryEvaluateScalarOperand64(
+                    instruction.Sources[0],
+                    registers,
+                    execMask,
+                    out var source) ||
+                !TryEvaluateScalarOperand(
+                    instruction.Sources[1],
+                    registers,
+                    out var control))
+            {
+                error = $"scalar-source64 pc=0x{instruction.Pc:X} op={instruction.Opcode}";
+                return false;
+            }
+
+            var offset = (int)control & 63;
+            var width = Math.Min(((int)control >> 16) & 0x7F, 64 - offset);
+            ulong value;
+            if (width == 0)
+            {
+                value = 0;
+            }
+            else
+            {
+                value = source >> offset;
+                if (width < 64)
+                {
+                    value &= ulong.MaxValue >> (64 - width);
+                    if (instruction.Opcode == "SBfeI64")
+                    {
+                        value = unchecked((ulong)((long)(value << (64 - width)) >> (64 - width)));
+                    }
+                }
+            }
+
+            WriteScalarPair(registers, destination.Value, value, ref execMask);
+            scalarConditionCode = value != 0;
+            return true;
+        }
+
         if (instruction.Opcode is
             "SCselectB64" or
             "SAndB64" or

src/SharpEmu.Libs/Agc/Gen5ShaderTranslator.cs

@@ -113,7 +113,8 @@ internal static class Gen5ShaderTranslator
         uint userDataBaseRegister,
         out Gen5ShaderState state,
         out string error,
-        Gen5ComputeSystemRegisters? computeSystemRegisters = null)
+        Gen5ComputeSystemRegisters? computeSystemRegisters = null,
+        uint userDataScalarRegisterBase = 0)
     {
         state = default!;
         if (!TryDecodeProgram(ctx, shaderAddress, out var program, out error))
@@ -134,7 +135,12 @@ internal static class Gen5ShaderTranslator
             shaderRegisters.TryGetValue(userDataBaseRegister + index, out userData[index]);
         }
 
-        state = new Gen5ShaderState(program, userData, metadata, computeSystemRegisters);
+        state = new Gen5ShaderState(
+            program,
+            userData,
+            metadata,
+            computeSystemRegisters,
+            userDataScalarRegisterBase);
         return true;
     }
 
@@ -179,7 +185,9 @@ internal static class Gen5ShaderTranslator
             : string.Empty;
         if (state.Metadata is not { } metadata)
         {
-            return $"ud[{userData}]{systemRegisters} metadata=missing";
+            return
+                $"ud_base=s{state.UserDataScalarRegisterBase} ud[{userData}]" +
+                $"{systemRegisters} metadata=missing";
         }
 
         var direct = string.Join(
@@ -191,7 +199,8 @@ internal static class Gen5ShaderTranslator
                 $"{resource.Kind}[{resource.Slot}]@{resource.OffsetDwords}" +
                 (resource.SizeFlag ? "+" : string.Empty)));
         return
-            $"ud[{userData}]{systemRegisters} metadata[eud={metadata.ExtendedUserDataSizeDwords}," +
+            $"ud_base=s{state.UserDataScalarRegisterBase} ud[{userData}]" +
+            $"{systemRegisters} metadata[eud={metadata.ExtendedUserDataSizeDwords}," +
             $"srt={metadata.ShaderResourceTableSizeDwords},direct={direct},resources={resources}]";
     }
 
@@ -668,9 +677,8 @@ internal static class Gen5ShaderTranslator
         error = string.Empty;
         name = opcode switch
         {
-            0x00 => "VCndmaskB32",
+            0x01 => "VCndmaskB32",
-            0x01 => "VReadlaneB32",
+            0x02 => "VDot2cF32F16",
-            0x02 => "VWritelaneB32",
             0x03 => "VAddF32",
             0x04 => "VSubF32",
             0x05 => "VSubrevF32",
@@ -810,7 +818,7 @@ internal static class Gen5ShaderTranslator
             }
             : opcode switch
         {
-            0x101 => "VReadlaneB32",
+            0x101 => "VCndmaskB32",
             0x103 => "VAddF32",
             0x104 => "VSubF32",
             0x108 => "VMulF32",
@@ -1246,17 +1254,16 @@ internal static class Gen5ShaderTranslator
                 var scalarOffset = (extra >> 25) & 0x7F;
                 var offset = SignExtend(extra & 0x1FFFFF, 21);
                 var count = ScalarLoadDwordCount(opcode);
-                var dynamicOffsetRegister = scalarOffset <= 105 || scalarOffset == 124
+                sources =
-                    ? scalarOffset
+                [
-                    : (uint?)null;
+                    Gen5Operand.Scalar(scalarBase),
-                sources = dynamicOffsetRegister.HasValue
+                    Gen5Operand.Scalar(scalarOffset),
-                    ? [Gen5Operand.Scalar(scalarBase), Gen5Operand.Scalar(dynamicOffsetRegister.Value)]
+                ];
-                    : [Gen5Operand.Scalar(scalarBase)];
                 destinations = Enumerable
                     .Range((int)scalarDestination, checked((int)count))
                     .Select(index => Gen5Operand.Scalar((uint)index))
                     .ToArray();
-                control = new Gen5ScalarMemoryControl(count, offset, dynamicOffsetRegister);
+                control = new Gen5ScalarMemoryControl(count, offset, scalarOffset);
                 break;
             }
             case Gen5ShaderEncoding.Vop1:

src/SharpEmu.Libs/Agc/Gen5SpirvTranslator.Alu.cs

@@ -38,7 +38,9 @@ internal static partial class Gen5SpirvTranslator
                     break;
                 case "VCndmaskB32":
                 {
-                    var condition = Load(_boolType, _vcc);
+                    var condition = instruction.Sources.Count > 2
+                        ? IsCurrentLaneSet(GetRawSource64(instruction, 2))
+                        : Load(_boolType, _vcc);
                     result = _module.AddInstruction(
                         SpirvOp.Select,
                         _uintType,
@@ -857,7 +859,7 @@ internal static partial class Gen5SpirvTranslator
                 condition = _module.AddInstruction(operation, _boolType, left, right);
             }
 
-            Store(_vcc, condition);
+            StoreWaveMask(106, condition);
             if (opcode.StartsWith("VCmpx", StringComparison.Ordinal))
             {
                 var active = _module.AddInstruction(
@@ -865,7 +867,7 @@ internal static partial class Gen5SpirvTranslator
                     _boolType,
                     Load(_boolType, _exec),
                     condition);
-                Store(_exec, active);
+                StoreWaveMask(126, active);
             }
 
             return true;
@@ -930,7 +932,7 @@ internal static partial class Gen5SpirvTranslator
             }
 
             if (instruction.Opcode.EndsWith("B64", StringComparison.Ordinal) ||
-                instruction.Opcode == "SWqmB64")
+                instruction.Opcode is "SWqmB64" or "SBfeU64" or "SBfeI64")
             {
                 return TryEmitScalar64(instruction, destination, out error);
             }
@@ -1312,7 +1314,7 @@ internal static partial class Gen5SpirvTranslator
             var left = GetRawSource64(instruction, 0);
             if (instruction.Opcode.EndsWith("SaveexecB64", StringComparison.Ordinal))
             {
-                var oldExec = LoadS64(126);
+                var oldExec = BooleanToLaneMask(Load(_boolType, _exec));
                 var notLeft = _module.AddInstruction(SpirvOp.Not, _ulongType, left);
                 var newExec = instruction.Opcode switch
                 {
@@ -1345,7 +1347,6 @@ internal static partial class Gen5SpirvTranslator
 
                 StoreS64(destination, oldExec);
                 StoreS64(126, newExec);
-                Store(_exec, IsNotZero64(newExec));
                 Store(_scc, IsNotZero64(newExec));
                 return true;
             }
@@ -1374,6 +1375,49 @@ internal static partial class Gen5SpirvTranslator
                 return true;
             }
 
+            if (instruction.Opcode is "SBfeU64" or "SBfeI64")
+            {
+                if (instruction.Sources.Count < 2)
+                {
+                    error = "missing scalar 64-bit bitfield source";
+                    return false;
+                }
+
+                var control = GetRawSource(instruction, 1);
+                var offset = BitwiseAnd(control, UInt(63));
+                var requestedWidth = BitwiseAnd(
+                    ShiftRightLogical(control, UInt(16)),
+                    UInt(0x7F));
+                var remaining = _module.AddInstruction(
+                    SpirvOp.ISub,
+                    _uintType,
+                    UInt(64),
+                    offset);
+                var width = Ext(
+                    38,
+                    _uintType,
+                    requestedWidth,
+                    remaining);
+                var extracted = instruction.Opcode == "SBfeI64"
+                    ? Bitcast(
+                        _ulongType,
+                        _module.AddInstruction(
+                            SpirvOp.BitFieldSExtract,
+                            _longType,
+                            Bitcast(_longType, left),
+                            offset,
+                            width))
+                    : _module.AddInstruction(
+                        SpirvOp.BitFieldUExtract,
+                        _ulongType,
+                        left,
+                        offset,
+                        width);
+                StoreS64(destination, extracted);
+                Store(_scc, IsNotZero64(extracted));
+                return true;
+            }
+
             uint value;
             if (instruction.Opcode is "SMovB64" or "SWqmB64")
             {
@@ -1451,11 +1495,6 @@ internal static partial class Gen5SpirvTranslator
             }
 
             StoreS64(destination, value);
-            if (destination == 126)
-            {
-                Store(_exec, IsNotZero64(value));
-            }
-
             return true;
         }
 
@@ -1472,14 +1511,6 @@ internal static partial class Gen5SpirvTranslator
             uint value = operand.Kind switch
             {
                 Gen5OperandKind.VectorRegister => LoadV(operand.Value),
-                Gen5OperandKind.ScalarRegister when operand.Value == 106 =>
-                    _module.AddInstruction(
-                        SpirvOp.Select,
-                        _uintType,
-                        Load(_boolType, _vcc),
-                        UInt(1),
-                        UInt(0)),
-                Gen5OperandKind.ScalarRegister when operand.Value == 107 => UInt(0),
                 Gen5OperandKind.ScalarRegister => LoadS(operand.Value),
                 Gen5OperandKind.LiteralConstant => UInt(operand.Value),
                 Gen5OperandKind.EncodedConstant when TryDecodeInlineConstant(
@@ -1881,7 +1912,7 @@ internal static partial class Gen5SpirvTranslator
                 _boolType,
                 _module.AddInstruction(SpirvOp.ULessThan, _boolType, partial, left),
                 _module.AddInstruction(SpirvOp.ULessThan, _boolType, result, partial));
-            Store(_vcc, carry);
+            StoreWaveMask(106, carry);
             return result;
         }
 
@@ -1912,7 +1943,7 @@ internal static partial class Gen5SpirvTranslator
                     _boolType,
                     partial,
                     borrowIn));
-            Store(_vcc, borrow);
+            StoreWaveMask(106, borrow);
             return result;
         }
 
@@ -1932,13 +1963,13 @@ internal static partial class Gen5SpirvTranslator
                         UInt(0)));
                 if (register == 106)
                 {
-                    Store(_vcc, carry);
+                    StoreWaveMask(106, carry);
                 }
 
                 return;
             }
 
-            Store(_vcc, carry);
+            StoreWaveMask(106, carry);
         }
 
         private uint EmitPermlane16(

src/SharpEmu.Libs/Agc/Gen5SpirvTranslator.cs

@@ -102,6 +102,7 @@ internal static partial class Gen5SpirvTranslator
         private uint _boolType;
         private uint _uintType;
         private uint _intType;
+        private uint _longType;
         private uint _ulongType;
         private uint _floatType;
         private uint _vec2Type;
@@ -297,17 +298,26 @@ internal static partial class Gen5SpirvTranslator
             _module.AddCapability(SpirvCapability.Shader);
             _module.AddCapability(SpirvCapability.Int64);
             _module.AddCapability(SpirvCapability.ImageQuery);
-            if (UsesSubgroupShuffle())
+            if (UsesSubgroupOperations())
             {
                 _module.AddCapability(SpirvCapability.GroupNonUniform);
+                if (UsesSubgroupShuffle())
+                {
                     _module.AddCapability(SpirvCapability.GroupNonUniformShuffle);
                 }
 
+                if (UsesWaveControl())
+                {
+                    _module.AddCapability(SpirvCapability.GroupNonUniformVote);
+                }
+            }
+
             _glsl = _module.ImportExtInst("GLSL.std.450");
             _voidType = _module.TypeVoid();
             _boolType = _module.TypeBool();
             _uintType = _module.TypeInt(32, signed: false);
             _intType = _module.TypeInt(32, signed: true);
+            _longType = _module.TypeInt(64, signed: true);
             _ulongType = _module.TypeInt(64, signed: false);
             _floatType = _module.TypeFloat(32);
             _vec2Type = _module.TypeVector(_floatType, 2);
@@ -557,7 +567,7 @@ internal static partial class Gen5SpirvTranslator
 
         private void DeclareStageInterface()
         {
-            if (UsesSubgroupShuffle())
+            if (UsesSubgroupOperations())
             {
                 var subgroupPointer =
                     _module.TypePointer(SpirvStorageClass.Input, _uintType);
@@ -706,8 +716,16 @@ internal static partial class Gen5SpirvTranslator
             }
 
             Store(_scc, _module.ConstantBool(false));
+            if (_subgroupInvocationIdInput != 0)
+            {
+                StoreWaveMask(106, _module.ConstantBool(false));
+                StoreWaveMask(126, _module.ConstantBool(true));
+            }
+            else
+            {
                 Store(_vcc, _module.ConstantBool(false));
                 Store(_exec, _module.ConstantBool(true));
+            }
             Store(_programCounter, UInt(0));
             Store(_programActive, _module.ConstantBool(true));
 
@@ -923,10 +941,10 @@ internal static partial class Gen5SpirvTranslator
             {
                 "SCbranchScc0" => LogicalNot(Load(_boolType, _scc)),
                 "SCbranchScc1" => Load(_boolType, _scc),
-                "SCbranchVccz" => LogicalNot(Load(_boolType, _vcc)),
+                "SCbranchVccz" => LogicalNot(SubgroupAny(Load(_boolType, _vcc))),
-                "SCbranchVccnz" => Load(_boolType, _vcc),
+                "SCbranchVccnz" => SubgroupAny(Load(_boolType, _vcc)),
-                "SCbranchExecz" => LogicalNot(Load(_boolType, _exec)),
+                "SCbranchExecz" => LogicalNot(SubgroupAny(Load(_boolType, _exec))),
-                "SCbranchExecnz" => Load(_boolType, _exec),
+                "SCbranchExecnz" => SubgroupAny(Load(_boolType, _exec)),
                 _ => 0,
             };
             return condition != 0;
@@ -1296,6 +1314,8 @@ internal static partial class Gen5SpirvTranslator
             var dwordAddress = ShiftRightLogical(byteAddress, UInt(2));
 
             if (instruction.Opcode == "BufferAtomicAdd")
+            {
+                EmitExecConditional(() =>
                 {
                     var original = _module.AddInstruction(
                         SpirvOp.AtomicIAdd,
@@ -1308,19 +1328,26 @@ internal static partial class Gen5SpirvTranslator
                     {
                         StoreV(control.VectorData, original);
                     }
+                });
 
                 return true;
             }
 
             if (instruction.Opcode.StartsWith("BufferStoreDword", StringComparison.Ordinal))
+            {
+                EmitExecConditional(() =>
                 {
                     for (uint index = 0; index < control.DwordCount; index++)
                     {
                         var address = index == 0
                             ? dwordAddress
                             : IAdd(dwordAddress, UInt(index));
-                    StoreBufferWord(bindingIndex, address, LoadV(control.VectorData + index));
+                        StoreBufferWord(
+                            bindingIndex,
+                            address,
+                            LoadV(control.VectorData + index));
                     }
+                });
 
                 return true;
             }
@@ -1449,11 +1476,12 @@ internal static partial class Gen5SpirvTranslator
                 }
                 else
                 {
-                    _module.AddStatement(
+                    EmitExecConditional(
+                        () => _module.AddStatement(
                             SpirvOp.ImageWrite,
                             imageObject,
                             coordinates,
-                        texel);
+                            texel));
                 }
 
                 return true;
@@ -1774,6 +1802,11 @@ internal static partial class Gen5SpirvTranslator
                 _boolType,
                 lowerInRange,
                 upperInRange);
+            inRange = _module.AddInstruction(
+                SpirvOp.LogicalAnd,
+                _boolType,
+                Load(_boolType, _exec),
+                inRange);
             var writeLabel = _module.AllocateId();
             var mergeLabel = _module.AllocateId();
             _module.AddStatement(SpirvOp.SelectionMerge, mergeLabel, 0);
@@ -1900,6 +1933,12 @@ internal static partial class Gen5SpirvTranslator
                     SpirvOp.CompositeConstruct,
                     outputType,
                     values);
+                vector = _module.AddInstruction(
+                    SpirvOp.Select,
+                    outputType,
+                    Load(_boolType, _exec),
+                    vector,
+                    Load(outputType, _pixelOutput));
                 Store(_pixelOutput, vector);
                 return true;
             }
@@ -1945,6 +1984,12 @@ internal static partial class Gen5SpirvTranslator
                 SpirvOp.CompositeConstruct,
                 _vec4Type,
                 components);
+            outputValue = _module.AddInstruction(
+                SpirvOp.Select,
+                _vec4Type,
+                Load(_boolType, _exec),
+                outputValue,
+                Load(_vec4Type, outputVariable));
             Store(outputVariable, outputValue);
             return true;
         }
@@ -2009,7 +2054,23 @@ internal static partial class Gen5SpirvTranslator
 
         private uint LoadV(uint register) => Load(_uintType, VectorPointer(register));
 
-        private void StoreS(uint register, uint value) => Store(ScalarPointer(register), value);
+        private void StoreS(uint register, uint value)
+        {
+            Store(ScalarPointer(register), value);
+            if (_subgroupInvocationIdInput == 0)
+            {
+                return;
+            }
+
+            if (register is 106 or 107)
+            {
+                Store(_vcc, IsCurrentLaneSet(LoadS64(106)));
+            }
+            else if (register is 126 or 127)
+            {
+                Store(_exec, IsCurrentLaneSet(LoadS64(126)));
+            }
+        }
 
         private void StoreV(uint register, uint value, bool guardWithExec = true)
         {
@@ -2059,6 +2120,61 @@ internal static partial class Gen5SpirvTranslator
         private uint LogicalNot(uint value) =>
             _module.AddInstruction(SpirvOp.LogicalNot, _boolType, value);
 
+        private uint SubgroupAny(uint condition) =>
+            _module.AddInstruction(
+                SpirvOp.GroupNonUniformAny,
+                _boolType,
+                UInt(3),
+                condition);
+
+        private uint CurrentLaneBit()
+        {
+            var lane = _module.AddInstruction(
+                SpirvOp.UConvert,
+                _ulongType,
+                Load(_uintType, _subgroupInvocationIdInput));
+            return _module.AddInstruction(
+                SpirvOp.ShiftLeftLogical,
+                _ulongType,
+                _module.Constant64(_ulongType, 1),
+                lane);
+        }
+
+        private uint BooleanToLaneMask(uint condition) =>
+            _module.AddInstruction(
+                SpirvOp.Select,
+                _ulongType,
+                condition,
+                CurrentLaneBit(),
+                _module.Constant64(_ulongType, 0));
+
+        private uint IsCurrentLaneSet(uint mask) =>
+            IsNotZero64(
+                _module.AddInstruction(
+                    SpirvOp.BitwiseAnd,
+                    _ulongType,
+                    mask,
+                    CurrentLaneBit()));
+
+        private void StoreWaveMask(uint register, uint condition) =>
+            StoreS64(register, BooleanToLaneMask(condition));
+
+        private void EmitExecConditional(Action emit)
+        {
+            var activeLabel = _module.AllocateId();
+            var mergeLabel = _module.AllocateId();
+            _module.AddStatement(SpirvOp.SelectionMerge, mergeLabel, 0);
+            _module.AddStatement(
+                SpirvOp.BranchConditional,
+                Load(_boolType, _exec),
+                activeLabel,
+                mergeLabel);
+            _module.AddLabel(activeLabel);
+            emit();
+            _module.AddStatement(SpirvOp.Branch, mergeLabel);
+            _module.AddLabel(mergeLabel);
+        }
+
         private bool UsesLds() =>
             _state.Program.Instructions.Any(instruction =>
                 instruction.Control is Gen5DataShareControl);
@@ -2067,6 +2183,22 @@ internal static partial class Gen5SpirvTranslator
             _state.Program.Instructions.Any(instruction =>
                 instruction.Opcode is "VPermlane16B32" or "VPermlanex16B32");
 
+        private bool UsesWaveControl() =>
+            _state.Program.Instructions.Any(instruction =>
+                instruction.Opcode.Contains("Saveexec", StringComparison.Ordinal) ||
+                instruction.Opcode.StartsWith("SCbranchExec", StringComparison.Ordinal) ||
+                instruction.Opcode.StartsWith("SCbranchVcc", StringComparison.Ordinal) ||
+                instruction.Opcode.StartsWith("VCmpx", StringComparison.Ordinal) ||
+                instruction.Sources.Any(IsWaveMaskOperand) ||
+                instruction.Destinations.Any(IsWaveMaskOperand));
+
+        private bool UsesSubgroupOperations() =>
+            UsesSubgroupShuffle() || UsesWaveControl();
+
+        private static bool IsWaveMaskOperand(Gen5Operand operand) =>
+            operand.Kind == Gen5OperandKind.ScalarRegister &&
+            operand.Value is 106 or 107 or 126 or 127;
+
         private static bool TryGetVectorDestination(
             Gen5ShaderInstruction instruction,
             out uint destination)

src/SharpEmu.Libs/Agc/SpirvModuleBuilder.cs

@@ -177,6 +177,7 @@ internal enum SpirvCapability : uint
     StorageImageReadWithoutFormat = 55,
     StorageImageWriteWithoutFormat = 56,
     GroupNonUniform = 61,
+    GroupNonUniformVote = 62,
     GroupNonUniformBallot = 64,
     GroupNonUniformShuffle = 65,
     RuntimeDescriptorArray = 5302,

src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs

@@ -5310,6 +5310,37 @@ public static class KernelMemoryCompatExports
         }
     }
 
+    internal static bool TryReadTrackedLibcHeap(
+        ulong address,
+        Span<byte> destination)
+    {
+        if (destination.IsEmpty)
+        {
+            return true;
+        }
+
+        var length = (ulong)destination.Length;
+        lock (_libcAllocGate)
+        {
+            foreach (var (allocationAddress, allocation) in _libcAllocations)
+            {
+                var allocationSize = (ulong)allocation.Size;
+                var offset = address >= allocationAddress
+                    ? address - allocationAddress
+                    : ulong.MaxValue;
+                if (offset > allocationSize ||
+                    length > allocationSize - offset)
+                {
+                    continue;
+                }
+
+                return TryReadHostMemory(address, destination);
+            }
+        }
+
+        return false;
+    }
+
     private static bool TryAllocateLibcHeap(ulong requestedSize, nuint alignment, bool zeroFill, out ulong address)
     {
         address = 0;

src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.cs

@@ -66,6 +66,7 @@ internal sealed record VulkanOffscreenGuestDraw(
     bool PublishTarget);
 
 internal sealed record VulkanComputeGuestDispatch(
+    ulong ShaderAddress,
     byte[] ComputeSpirv,
     IReadOnlyList<VulkanGuestDrawTexture> Textures,
     IReadOnlyList<VulkanGuestMemoryBuffer> GlobalMemoryBuffers,
@@ -434,6 +435,7 @@ internal static unsafe class VulkanVideoPresenter
     }
 
     public static void SubmitComputeDispatch(
+        ulong shaderAddress,
         byte[] computeSpirv,
         IReadOnlyList<VulkanGuestDrawTexture> textures,
         IReadOnlyList<VulkanGuestMemoryBuffer> globalMemoryBuffers,
@@ -459,6 +461,7 @@ internal static unsafe class VulkanVideoPresenter
 
             EnqueueGuestWorkLocked(
                 new VulkanComputeGuestDispatch(
+                    shaderAddress,
                     computeSpirv,
                     textures.ToArray(),
                     globalMemoryBuffers.ToArray(),
@@ -845,7 +848,8 @@ internal static unsafe class VulkanVideoPresenter
             Fence Fence,
             CommandBuffer CommandBuffer,
             TranslatedDrawResources Resources,
-            IReadOnlyList<GuestImageResource> TraceImages);
+            IReadOnlyList<GuestImageResource> TraceImages,
+            string DebugName);
 
         public Presenter(uint width, uint height)
         {
@@ -1056,8 +1060,10 @@ internal static unsafe class VulkanVideoPresenter
         {
             var storage = dispatch.Textures.FirstOrDefault(texture => texture.IsStorage && texture.Address != 0);
             return storage is null
-                ? $"SharpEmu compute {dispatch.GroupCountX}x{dispatch.GroupCountY}x{dispatch.GroupCountZ}"
+                ? $"SharpEmu compute cs=0x{dispatch.ShaderAddress:X16} " +
-                : $"SharpEmu compute storage=0x{storage.Address:X16} " +
+                  $"{dispatch.GroupCountX}x{dispatch.GroupCountY}x{dispatch.GroupCountZ}"
+                : $"SharpEmu compute cs=0x{dispatch.ShaderAddress:X16} " +
+                  $"storage=0x{storage.Address:X16} " +
                   $"{storage.Width}x{storage.Height} fmt{storage.Format} " +
                   $"{dispatch.GroupCountX}x{dispatch.GroupCountY}x{dispatch.GroupCountZ}";
         }
@@ -1365,7 +1371,8 @@ internal static unsafe class VulkanVideoPresenter
                     fence,
                     commandBuffer,
                     resources,
-                    traceImages));
+                    traceImages,
+                    resources.DebugName));
         }
 
         private void EnsureGuestSubmissionCapacity()
@@ -1382,14 +1389,13 @@ internal static unsafe class VulkanVideoPresenter
             if (waitForOldest && _pendingGuestSubmissions.TryPeek(out var oldest))
             {
                 var fence = oldest.Fence;
-                Check(
+                var result = _vk.WaitForFences(
-                    _vk.WaitForFences(
                     _device,
                     1,
                     &fence,
                     true,
-                        ulong.MaxValue),
+                    ulong.MaxValue);
-                    "vkWaitForFences(guest)");
+                Check(result, $"vkWaitForFences(guest: {oldest.DebugName})");
             }
 
             while (_pendingGuestSubmissions.TryPeek(out var submission))
@@ -1400,7 +1406,7 @@ internal static unsafe class VulkanVideoPresenter
                     break;
                 }
 
-                Check(status, "vkGetFenceStatus(guest)");
+                Check(status, $"vkGetFenceStatus(guest: {submission.DebugName})");
                 _pendingGuestSubmissions.Dequeue();
 
                 foreach (var image in submission.TraceImages)
@@ -2156,6 +2162,13 @@ internal static unsafe class VulkanVideoPresenter
                     AttachmentCount = 1,
                     PAttachments = &colorBlendAttachment,
                 };
+                var dynamicStateValue = DynamicState.Scissor;
+                var dynamicState = new PipelineDynamicStateCreateInfo
+                {
+                    SType = StructureType.PipelineDynamicStateCreateInfo,
+                    DynamicStateCount = 1,
+                    PDynamicStates = &dynamicStateValue,
+                };
                 var pipelineInfo = new GraphicsPipelineCreateInfo
                 {
                     SType = StructureType.GraphicsPipelineCreateInfo,
@@ -2167,6 +2180,7 @@ internal static unsafe class VulkanVideoPresenter
                     PRasterizationState = &rasterization,
                     PMultisampleState = &multisample,
                     PColorBlendState = &colorBlend,
+                    PDynamicState = &dynamicState,
                     Layout = resources.PipelineLayout,
                     RenderPass = renderPass,
                     Subpass = 0,
@@ -2893,11 +2907,7 @@ internal static unsafe class VulkanVideoPresenter
                         null);
                 }
 
-                _vk.CmdDispatch(
+                RecordChunkedComputeDispatch(_commandBuffer, work);
-                    _commandBuffer,
-                    work.GroupCountX,
-                    work.GroupCountY,
-                    work.GroupCountZ);
                 RecordStorageImagesForRead(resources, PipelineStageFlags.ComputeShaderBit);
                 EndDebugLabel(_commandBuffer);
 
@@ -2911,7 +2921,7 @@ internal static unsafe class VulkanVideoPresenter
                 TraceVulkanShader(
                     $"vk.compute_dispatch groups={work.GroupCountX}x" +
                     $"{work.GroupCountY}x{work.GroupCountZ} " +
-                    $"textures={work.Textures.Count}");
+                    $"textures={work.Textures.Count} cs=0x{work.ShaderAddress:X16}");
             }
             catch (Exception exception)
             {
@@ -2937,6 +2947,44 @@ internal static unsafe class VulkanVideoPresenter
             }
         }
 
+        private void RecordChunkedComputeDispatch(
+            CommandBuffer commandBuffer,
+            VulkanComputeGuestDispatch work)
+        {
+            const uint maxWorkgroupsPerCommand = 4096;
+            var yChunk = Math.Max(
+                1u,
+                Math.Min(
+                    work.GroupCountY,
+                    maxWorkgroupsPerCommand / Math.Max(work.GroupCountX, 1u)));
+            var commandCount = 0u;
+
+            for (var z = 0u; z < work.GroupCountZ; z++)
+            {
+                for (var y = 0u; y < work.GroupCountY; y += yChunk)
+                {
+                    var countY = Math.Min(yChunk, work.GroupCountY - y);
+                    _vk.CmdDispatchBase(
+                        commandBuffer,
+                        0,
+                        y,
+                        z,
+                        work.GroupCountX,
+                        countY,
+                        1);
+                    commandCount++;
+                }
+            }
+
+            if (commandCount > 1)
+            {
+                TraceVulkanShader(
+                    $"vk.compute_chunked cs=0x{work.ShaderAddress:X16} " +
+                    $"groups={work.GroupCountX}x{work.GroupCountY}x{work.GroupCountZ} " +
+                    $"commands={commandCount} y_chunk={yChunk}");
+            }
+        }
+
         private void ExecuteOffscreenDraw(VulkanOffscreenGuestDraw work)
         {
             var format = GetRenderTargetFormat(work.Target.Format, work.Target.NumberType);
@@ -4294,6 +4342,18 @@ internal static unsafe class VulkanVideoPresenter
                     null);
             }
 
+            const uint maxPixelsPerDraw = 512 * 512;
+            var rowsPerDraw = Math.Max(
+                1u,
+                Math.Min(extent.Height, maxPixelsPerDraw / Math.Max(extent.Width, 1u)));
+            var drawCount = 0u;
+            for (var y = 0u; y < extent.Height; y += rowsPerDraw)
+            {
+                var scissor = new Rect2D(
+                    new Offset2D(0, checked((int)y)),
+                    new Extent2D(extent.Width, Math.Min(rowsPerDraw, extent.Height - y)));
+                _vk.CmdSetScissor(_commandBuffer, 0, 1, &scissor);
+
                 if (resources.IndexBuffer.Handle != 0)
                 {
                     _vk.CmdBindIndexBuffer(
@@ -4318,6 +4378,16 @@ internal static unsafe class VulkanVideoPresenter
                         0,
                         0);
                 }
+
+                drawCount++;
+            }
+
+            if (drawCount > 1)
+            {
+                TraceVulkanShader(
+                    $"vk.graphics_chunked target={extent.Width}x{extent.Height} " +
+                    $"draws={drawCount} rows={rowsPerDraw} name={resources.DebugName}");
+            }
             _vk.CmdEndRenderPass(_commandBuffer);
         }