diff --git a/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs b/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs index 671f3d57..43ac6d67 100644 --- a/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs +++ b/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs @@ -401,6 +401,9 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime } Environment.SetEnvironmentVariable(app0VariableName, app0Root); + // Overlap the cooked-id APR walk with module load so the first ReadFile + // miss mid-boot does not stall on a cold USB index. + SharpEmu.Libs.Ampr.AmprFileRegistry.BeginApp0IndexPreload(app0Root); return new App0BindingScope(app0VariableName); } diff --git a/src/SharpEmu.Libs/Ampr/AmprExports.cs b/src/SharpEmu.Libs/Ampr/AmprExports.cs index dc09061a..7d2b9c4d 100644 --- a/src/SharpEmu.Libs/Ampr/AmprExports.cs +++ b/src/SharpEmu.Libs/Ampr/AmprExports.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.HLE; +using SharpEmu.Libs.Agc; using SharpEmu.Libs.Kernel; using System.Buffers; using System.Buffers.Binary; @@ -25,7 +26,6 @@ public static class AmprExports private const uint KernelEventQueueRecordType = 2; private const uint WriteAddressRecordType = 3; private static readonly ConcurrentDictionary _commandBuffers = new(); - private static readonly ConcurrentDictionary> _hostFileCache = new(StringComparer.OrdinalIgnoreCase); private static readonly bool _traceAmpr = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AMPR"), "1", StringComparison.Ordinal); private static readonly bool _traceAmprReads = @@ -40,7 +40,7 @@ public static class AmprExports public ulong CommandCount; } - private sealed class CachedHostFile + private sealed class CachedHostFile : IDisposable { public CachedHostFile(string path) { @@ -55,8 +55,26 @@ public static class AmprExports public SafeFileHandle Handle { get; } public long Length { get; } + + public void Dispose() => Handle.Dispose(); } + private sealed class CachedHostFileEntry + { + public required string Path { get; init; } + public required CachedHostFile File { get; init; } + } + + // Keep a bounded LRU of open host files. An unbounded cache exhausts the + // process FD limit (~10k on macOS) during large asset storms, after + // which every new open throws IOException and surfaces as NOT_FOUND — the + // guest then reports InvalidFileFourCC on empty buffers. + private const int MaxCachedHostFiles = 1536; + private static readonly object _hostFileCacheGate = new(); + private static readonly Dictionary> _hostFileByPath = + new(StringComparer.OrdinalIgnoreCase); + private static readonly LinkedList _hostFileLru = new(); + [SysAbiExport( Nid = "8aI7R7WaOlc", ExportName = "sceAmprCommandBufferConstructor", @@ -80,6 +98,7 @@ public static class AmprExports } TraceAmpr(ctx, "ctor", commandBuffer, buffer, size); + TryPreindexApp0(); ctx[CpuRegister.Rax] = commandBuffer; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -107,6 +126,7 @@ public static class AmprExports } TraceAmpr(ctx, "apr_ctor", commandBuffer, aux0, aux1); + TryPreindexApp0(); ctx[CpuRegister.Rax] = commandBuffer; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -271,8 +291,19 @@ public static class AmprExports if (!AmprFileRegistry.TryGetHostPath(fileId, out var hostPath)) { - TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead: 0, hostPath, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + // Cooked Insomniac content ids are FNV("/app0/...") and may never + // pass through APR resolve. Index app0 once, then retry the lookup. + var app0Root = KernelMemoryCompatExports.ResolveGuestPath("$/"); + if (!string.IsNullOrEmpty(app0Root)) + { + AmprFileRegistry.EnsureApp0Indexed(app0Root); + } + + if (!AmprFileRegistry.TryGetHostPath(fileId, out hostPath)) + { + TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead: 0, hostPath, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } } // Offset -1 means "continue after the previous read of this file id". @@ -779,7 +810,9 @@ public static class AmprExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - const int ChunkSize = 1024 * 1024; + // 4 MiB chunks cut syscall/Rosetta round-trips on DeS' large sequential + // APR reads without blowing the ArrayPool for small probes. + const int ChunkSize = 4 * 1024 * 1024; var buffer = ArrayPool.Shared.Rent((int)Math.Min((ulong)ChunkSize, size)); try @@ -857,27 +890,100 @@ public static class AmprExports cachePath = hostPath; } - var lazy = _hostFileCache.GetOrAdd( - cachePath, - static path => new Lazy(() => new CachedHostFile(path), isThreadSafe: true)); + lock (_hostFileCacheGate) + { + if (_hostFileByPath.TryGetValue(cachePath, out var existing)) + { + _hostFileLru.Remove(existing); + _hostFileLru.AddFirst(existing); + file = existing.Value.File; + return true; + } + } + CachedHostFile opened; try { - file = lazy.Value; - return true; + opened = new CachedHostFile(cachePath); } catch (UnauthorizedAccessException) { - _hostFileCache.TryRemove(cachePath, out _); result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; return false; } catch (IOException) { - _hostFileCache.TryRemove(cachePath, out _); - result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; - return false; + // Likely EMFILE from a prior unbounded cache, or a transient miss. + // Evict everything we hold and retry once so a full FD table can + // recover without restarting the process. + EvictAllCachedHostFiles(); + try + { + opened = new CachedHostFile(cachePath); + } + catch (UnauthorizedAccessException) + { + result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; + return false; + } + catch (IOException) + { + result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + return false; + } } + + lock (_hostFileCacheGate) + { + if (_hostFileByPath.TryGetValue(cachePath, out var raced)) + { + opened.Dispose(); + _hostFileLru.Remove(raced); + _hostFileLru.AddFirst(raced); + file = raced.Value.File; + return true; + } + + while (_hostFileByPath.Count >= MaxCachedHostFiles) + { + EvictLeastRecentlyUsedHostFileLocked(); + } + + var entry = new CachedHostFileEntry { Path = cachePath, File = opened }; + var node = _hostFileLru.AddFirst(entry); + _hostFileByPath[cachePath] = node; + file = opened; + return true; + } + } + + private static void EvictAllCachedHostFiles() + { + List doomed; + lock (_hostFileCacheGate) + { + doomed = _hostFileLru.Select(entry => entry.File).ToList(); + _hostFileLru.Clear(); + _hostFileByPath.Clear(); + } + + foreach (var cached in doomed) + { + cached.Dispose(); + } + } + + private static void EvictLeastRecentlyUsedHostFileLocked() + { + var last = _hostFileLru.Last; + if (last is null) + { + return; + } + + _hostFileLru.RemoveLast(); + _hostFileByPath.Remove(last.Value.Path); + last.Value.File.Dispose(); } private static bool AppendReadFileRecord( @@ -1004,6 +1110,19 @@ public static class AmprExports return false; } + // GPU WAIT_REG_MEM often watches these APR completion labels + // (e.g. 0x20505xxx DEADBEEF/counter fences). Without + // RecordProduced the wait sits producerless after the guest recycles + // the dword, and CollectDeadlockBroken cannot replay the wake. + _ = GpuWaitRegistry.RecordProduced(ctx.Memory, address, value); + if ((address & 4ul) == 0) + { + // 32-bit GPU waits use the low dword; also latch that view when the + // address is dword-aligned so a u32 compare against ref sees it. + _ = GpuWaitRegistry.RecordProduced( + ctx.Memory, address, unchecked((uint)value)); + } + TraceAmpr(ctx, "complete_write_address", address, value, 0); return true; } @@ -1021,6 +1140,15 @@ public static class AmprExports return true; } + private static void TryPreindexApp0() + { + var app0Root = KernelMemoryCompatExports.ResolveGuestPath("$/"); + if (!string.IsNullOrEmpty(app0Root)) + { + AmprFileRegistry.EnsureApp0Indexed(app0Root); + } + } + private static void TraceAmpr(CpuContext ctx, string operation, ulong commandBuffer, ulong arg0, ulong arg1) { if (!_traceAmpr) diff --git a/src/SharpEmu.Libs/Ampr/AmprFileRegistry.cs b/src/SharpEmu.Libs/Ampr/AmprFileRegistry.cs index d808b9a8..4c9233ca 100644 --- a/src/SharpEmu.Libs/Ampr/AmprFileRegistry.cs +++ b/src/SharpEmu.Libs/Ampr/AmprFileRegistry.cs @@ -2,15 +2,35 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System.Collections.Concurrent; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Text; namespace SharpEmu.Libs.Ampr; internal static class AmprFileRegistry { - private static readonly ConcurrentDictionary _hostPathsById = new(); + private const uint CacheMagicV2 = 0x32495041u; // 'API2' + private const uint CacheVersionV2 = 2; + private const uint CacheMagicV3 = 0x33495041u; // 'API3' + private const uint CacheVersionV3 = 3; + + private static readonly ConcurrentDictionary _hostPathsById = new( + concurrencyLevel: Math.Max(4, Environment.ProcessorCount), + capacity: 1_048_576); + private static readonly object _indexGate = new(); + private static string? _indexedApp0Root; + private static string? _indexingApp0Root; + private static int _preloadStarted; public static uint Register(string guestPath, string hostPath) { + if (TryGetApp0Relative(guestPath, out var relative) && relative.Length != 0) + { + RegisterApp0Relative(relative, hostPath); + return ComputeFileId("$/" + relative); + } + var id = ComputeFileId(guestPath); _hostPathsById[id] = hostPath; return id; @@ -21,18 +41,559 @@ internal static class AmprFileRegistry return _hostPathsById.TryGetValue(id, out hostPath!); } + /// Test hook: wipe registry state between cases. + internal static void ClearForTests() + { + lock (_indexGate) + { + _hostPathsById.Clear(); + _indexedApp0Root = null; + _indexingApp0Root = null; + _preloadStarted = 0; + } + } + + /// Test hook for the allocation-free alias publisher. + internal static void RegisterApp0RelativeForTests(string relative, string hostPath) => + RegisterApp0Relative(relative, hostPath); + + /// + /// Kick off on a background thread as soon as + /// the host knows app0. otherwise pays the full tree walk on the + /// first cooked-id APR miss mid-boot (~8s under Rosetta for DeS). + /// + public static void BeginApp0IndexPreload(string? app0Root) + { + if (string.IsNullOrWhiteSpace(app0Root) || !Directory.Exists(app0Root)) + { + return; + } + + if (Interlocked.Exchange(ref _preloadStarted, 1) != 0) + { + return; + } + + var root = app0Root; + ThreadPool.UnsafeQueueUserWorkItem( + static state => + { + try + { + EnsureApp0Indexed((string)state!); + } + catch (Exception exception) + { + Console.Error.WriteLine( + $"[LOADER][WARN] ampr.app0_index_preload_failed: {exception.Message}"); + } + }, + root); + } + + /// + /// Indexes every file under app0 under both $/ and /app0/ FNV + /// ids. Cooked asset tables ship precomputed ids; without this walk, a title + /// that never resolves those paths through APR leaves ReadFile permanently + /// NOT_FOUND. + /// + public static void EnsureApp0Indexed(string app0Root) + { + if (string.IsNullOrWhiteSpace(app0Root) || !Directory.Exists(app0Root)) + { + return; + } + + var normalizedRoot = Path.GetFullPath(app0Root); + var stopwatch = Stopwatch.StartNew(); + + lock (_indexGate) + { + while (true) + { + if (string.Equals(_indexedApp0Root, normalizedRoot, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + if (_indexingApp0Root is not null) + { + // Another thread (preload) owns the walk — wait instead of + // stacking a second 8s index on the guest APR miss path. + if (string.Equals( + _indexingApp0Root, + normalizedRoot, + StringComparison.OrdinalIgnoreCase)) + { + Monitor.Wait(_indexGate); + continue; + } + + Monitor.Wait(_indexGate, 50); + continue; + } + + _indexingApp0Root = normalizedRoot; + break; + } + } + + try + { + var cachePathV3 = GetIndexCachePath(normalizedRoot, version: 3); + var cachePathV2 = GetIndexCachePath(normalizedRoot, version: 2); + if (TryLoadIndexCache(normalizedRoot, cachePathV3, preferV3: true, out var cachedFiles)) + { + lock (_indexGate) + { + _indexedApp0Root = normalizedRoot; + } + + Console.Error.WriteLine( + $"[LOADER][INFO] ampr.app0_index_cache_hit root={normalizedRoot} " + + $"files={cachedFiles} ids={_hostPathsById.Count} " + + $"elapsed_ms={stopwatch.Elapsed.TotalMilliseconds:F1}"); + return; + } + + if (TryLoadIndexCache(normalizedRoot, cachePathV2, preferV3: false, out cachedFiles)) + { + lock (_indexGate) + { + _indexedApp0Root = normalizedRoot; + } + + // Promote v2 (rehash-on-load) to v3 (precomputed ids) so the + // next boot skips the Rosetta FNV storm. + TrySaveIndexCache(normalizedRoot, cachePathV3, cachedFiles); + Console.Error.WriteLine( + $"[LOADER][INFO] ampr.app0_index_cache_hit root={normalizedRoot} " + + $"files={cachedFiles} ids={_hostPathsById.Count} " + + $"elapsed_ms={stopwatch.Elapsed.TotalMilliseconds:F1} upgraded=v3"); + return; + } + + var relatives = new List(256 * 1024); + foreach (var hostPath in Directory.EnumerateFiles( + normalizedRoot, + "*", + SearchOption.AllDirectories)) + { + var relative = Path.GetRelativePath(normalizedRoot, hostPath) + .Replace('\\', '/'); + if (string.IsNullOrEmpty(relative) || + relative.StartsWith("..", StringComparison.Ordinal)) + { + continue; + } + + relatives.Add(relative); + } + + // Hash + dictionary fill dominates under Rosetta once the walk is + // done; parallelize across cores without re-walking the tree. + Parallel.ForEach( + relatives, + new ParallelOptions + { + MaxDegreeOfParallelism = Math.Max(2, Environment.ProcessorCount - 1), + }, + relative => + { + var hostPath = Path.Combine(normalizedRoot, relative.Replace('/', Path.DirectorySeparatorChar)); + RegisterApp0Relative(relative, hostPath); + }); + + lock (_indexGate) + { + _indexedApp0Root = normalizedRoot; + } + + TrySaveIndexCache(normalizedRoot, cachePathV3, relatives.Count); + Console.Error.WriteLine( + $"[LOADER][INFO] ampr.app0_indexed root={normalizedRoot} " + + $"files={relatives.Count} ids={_hostPathsById.Count} " + + $"elapsed_ms={stopwatch.Elapsed.TotalMilliseconds:F1}"); + } + finally + { + lock (_indexGate) + { + _indexingApp0Root = null; + Monitor.PulseAll(_indexGate); + } + } + } + + /// + /// Registers the four Insomniac path aliases for one app0-relative file + /// without allocating intermediate guest-path strings. + /// + private static void RegisterApp0Relative(string relative, string hostPath) + { + // "$/" + relative + Publish(FnvContinueAscii(FnvContinueAscii(OffsetBasis, (byte)'$'), (byte)'/'), relative, hostPath); + // "/app0/" + relative + Publish(FnvContinueAsciiPrefix(OffsetBasis, "/app0/"u8), relative, hostPath); + // "app0/" + relative + Publish(FnvContinueAsciiPrefix(OffsetBasis, "app0/"u8), relative, hostPath); + // bare relative + Publish(OffsetBasis, relative, hostPath); + } + + private static void Publish(uint hash, string relative, string hostPath) + { + _hostPathsById[FnvContinueUtf8(hash, relative)] = hostPath; + } + internal static uint ComputeFileId(string guestPath) { - var bytes = System.Text.Encoding.UTF8.GetBytes(guestPath); + return FnvContinueUtf8(OffsetBasis, guestPath); + } - const uint offsetBasis = 2166136261; - const uint prime = 16777619; - - var hash = offsetBasis; - foreach (var b in bytes) + internal static IEnumerable EnumerateApp0PathAliases(string guestPath) + { + if (string.IsNullOrEmpty(guestPath)) { - hash ^= b; - hash *= prime; + yield break; + } + + if (!TryGetApp0Relative(guestPath, out var relative) || + string.IsNullOrEmpty(relative)) + { + yield break; + } + + yield return "$/" + relative; + yield return "/app0/" + relative; + yield return "app0/" + relative; + yield return relative; + } + + private static bool TryGetApp0Relative(string guestPath, out string relative) + { + relative = string.Empty; + var normalized = guestPath.Replace('\\', '/'); + + if (normalized.StartsWith("$/", StringComparison.Ordinal)) + { + relative = normalized[2..].TrimStart('/'); + return relative.Length != 0; + } + + if (normalized.StartsWith("/app0/", StringComparison.OrdinalIgnoreCase)) + { + relative = normalized["/app0/".Length..].TrimStart('/'); + return relative.Length != 0; + } + + if (normalized.StartsWith("app0/", StringComparison.OrdinalIgnoreCase)) + { + relative = normalized["app0/".Length..].TrimStart('/'); + return relative.Length != 0; + } + + // Bare relative paths are treated as app0-relative by ResolveGuestPath. + if (!normalized.StartsWith('/') && + !Path.IsPathFullyQualified(guestPath)) + { + relative = normalized.TrimStart('/'); + return relative.Length != 0; + } + + return false; + } + + private static string GetIndexCachePath(string normalizedRoot, int version) + { + var overrideDir = Environment.GetEnvironmentVariable("SHARPEMU_AMPR_INDEX_CACHE"); + var cacheDir = !string.IsNullOrWhiteSpace(overrideDir) + ? overrideDir + : Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "SharpEmu", + "ampr-index"); + Directory.CreateDirectory(cacheDir); + + var rootHash = ComputeFileId(normalizedRoot.ToLowerInvariant()); + return Path.Combine(cacheDir, $"app0-{rootHash:x8}.v{version}.idx"); + } + + private static bool TryLoadIndexCache( + string normalizedRoot, + string cachePath, + bool preferV3, + out int fileCount) + { + fileCount = 0; + try + { + if (!File.Exists(cachePath)) + { + return false; + } + + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_AMPR_REINDEX"), + "1", + StringComparison.Ordinal)) + { + return false; + } + + using var stream = File.OpenRead(cachePath); + using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: false); + var magic = reader.ReadUInt32(); + var version = reader.ReadUInt32(); + var isV3 = magic == CacheMagicV3 && version == CacheVersionV3; + var isV2 = magic == CacheMagicV2 && version == CacheVersionV2; + if (preferV3) + { + if (!isV3) + { + return false; + } + } + else if (!isV2) + { + return false; + } + + var root = reader.ReadString(); + if (!string.Equals(root, normalizedRoot, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var expectedFiles = reader.ReadInt32(); + var expectedParamTicks = reader.ReadInt64(); + var actualParamTicks = GetParamJsonWriteTicks(normalizedRoot); + if (actualParamTicks == 0 || actualParamTicks != expectedParamTicks) + { + return false; + } + + if (expectedFiles < 0 || expectedFiles > 8_000_000) + { + return false; + } + + if (isV3) + { + // Precomputed FNV ids — no Rosetta hash storm on every boot. + var entries = new (string Relative, uint Id0, uint Id1, uint Id2, uint Id3)[expectedFiles]; + for (var i = 0; i < expectedFiles; i++) + { + entries[i] = ( + reader.ReadString(), + reader.ReadUInt32(), + reader.ReadUInt32(), + reader.ReadUInt32(), + reader.ReadUInt32()); + } + + Parallel.ForEach( + entries, + new ParallelOptions + { + MaxDegreeOfParallelism = Math.Max(2, Environment.ProcessorCount - 1), + }, + entry => + { + if (string.IsNullOrEmpty(entry.Relative) || + entry.Relative.Contains("..", StringComparison.Ordinal)) + { + return; + } + + var hostPath = normalizedRoot.EndsWith(Path.DirectorySeparatorChar) + ? normalizedRoot + entry.Relative.Replace('/', Path.DirectorySeparatorChar) + : normalizedRoot + Path.DirectorySeparatorChar + + entry.Relative.Replace('/', Path.DirectorySeparatorChar); + _hostPathsById[entry.Id0] = hostPath; + _hostPathsById[entry.Id1] = hostPath; + _hostPathsById[entry.Id2] = hostPath; + _hostPathsById[entry.Id3] = hostPath; + }); + } + else + { + var relatives = new string[expectedFiles]; + for (var i = 0; i < expectedFiles; i++) + { + relatives[i] = reader.ReadString(); + } + + Parallel.ForEach( + relatives, + new ParallelOptions + { + MaxDegreeOfParallelism = Math.Max(2, Environment.ProcessorCount - 1), + }, + relative => + { + if (string.IsNullOrEmpty(relative) || + relative.Contains("..", StringComparison.Ordinal)) + { + return; + } + + var hostPath = Path.Combine( + normalizedRoot, + relative.Replace('/', Path.DirectorySeparatorChar)); + RegisterApp0Relative(relative, hostPath); + }); + } + + fileCount = expectedFiles; + return true; + } + catch (Exception exception) + { + _hostPathsById.Clear(); + Console.Error.WriteLine( + $"[LOADER][WARN] ampr.app0_index_cache_load_failed: {exception.Message}"); + return false; + } + } + + private static void TrySaveIndexCache( + string normalizedRoot, + string cachePath, + int fileCount) + { + try + { + var paramTicks = GetParamJsonWriteTicks(normalizedRoot); + if (paramTicks == 0 || fileCount <= 0) + { + return; + } + + var relatives = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var hostPath in _hostPathsById.Values) + { + var relative = Path.GetRelativePath(normalizedRoot, hostPath) + .Replace('\\', '/'); + if (string.IsNullOrEmpty(relative) || + relative.StartsWith("..", StringComparison.Ordinal)) + { + continue; + } + + relatives.Add(relative); + } + + var tempPath = cachePath + ".tmp"; + using (var stream = File.Create(tempPath)) + using (var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: false)) + { + writer.Write(CacheMagicV3); + writer.Write(CacheVersionV3); + writer.Write(normalizedRoot); + writer.Write(relatives.Count); + writer.Write(paramTicks); + foreach (var relative in relatives) + { + writer.Write(relative); + // Mirror RegisterApp0Relative id order: $/ /app0/ app0/ bare. + writer.Write(ComputeApp0AliasIds(relative, out var id1, out var id2, out var id3)); + writer.Write(id1); + writer.Write(id2); + writer.Write(id3); + } + } + + File.Move(tempPath, cachePath, overwrite: true); + Console.Error.WriteLine( + $"[LOADER][INFO] ampr.app0_index_cache_saved path={cachePath} " + + $"files={relatives.Count}"); + } + catch (Exception exception) + { + Console.Error.WriteLine( + $"[LOADER][WARN] ampr.app0_index_cache_save_failed: {exception.Message}"); + } + } + + private static uint ComputeApp0AliasIds( + string relative, + out uint app0Slash, + out uint app0, + out uint bare) + { + var dollar = FnvContinueUtf8( + FnvContinueAscii(FnvContinueAscii(OffsetBasis, (byte)'$'), (byte)'/'), + relative); + app0Slash = FnvContinueUtf8(FnvContinueAsciiPrefix(OffsetBasis, "/app0/"u8), relative); + app0 = FnvContinueUtf8(FnvContinueAsciiPrefix(OffsetBasis, "app0/"u8), relative); + bare = FnvContinueUtf8(OffsetBasis, relative); + return dollar; + } + + /// + /// Cheap dump fingerprint. Full-tree walks are too expensive for cache + /// validation; param.json changes with title updates. Force a rebuild with + /// SHARPEMU_AMPR_REINDEX=1 after manual dump edits. + /// + private static long GetParamJsonWriteTicks(string normalizedRoot) + { + try + { + var paramPath = Path.Combine(normalizedRoot, "sce_sys", "param.json"); + return File.Exists(paramPath) + ? File.GetLastWriteTimeUtc(paramPath).Ticks + : 0; + } + catch + { + return 0; + } + } + + private const uint OffsetBasis = 2166136261; + private const uint FnvPrime = 16777619; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint FnvContinueAscii(uint hash, byte value) + { + hash ^= value; + return hash * FnvPrime; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint FnvContinueAsciiPrefix(uint hash, ReadOnlySpan ascii) + { + foreach (var value in ascii) + { + hash ^= value; + hash *= FnvPrime; + } + + return hash; + } + + private static uint FnvContinueUtf8(uint hash, string text) + { + // Game asset paths are overwhelmingly ASCII; avoid Encoding.GetBytes + // allocations on the 223k-file DeS index hot path. + Span utf8Scratch = stackalloc byte[4]; + for (var i = 0; i < text.Length; i++) + { + var ch = text[i]; + if (ch < 0x80) + { + hash ^= (byte)ch; + hash *= FnvPrime; + continue; + } + + var written = Encoding.UTF8.GetBytes(text.AsSpan(i, 1), utf8Scratch); + for (var b = 0; b < written; b++) + { + hash ^= utf8Scratch[b]; + hash *= FnvPrime; + } } return hash; diff --git a/src/SharpEmu.Libs/SharpEmu.Libs.csproj b/src/SharpEmu.Libs/SharpEmu.Libs.csproj index ed4d1e4b..05ccb8a4 100644 --- a/src/SharpEmu.Libs/SharpEmu.Libs.csproj +++ b/src/SharpEmu.Libs/SharpEmu.Libs.csproj @@ -24,6 +24,7 @@ SPDX-License-Identifier: GPL-2.0-or-later + diff --git a/tests/SharpEmu.Libs.Tests/Ampr/AmprFileRegistryTests.cs b/tests/SharpEmu.Libs.Tests/Ampr/AmprFileRegistryTests.cs new file mode 100644 index 00000000..f05f1b62 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Ampr/AmprFileRegistryTests.cs @@ -0,0 +1,78 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Ampr; +using Xunit; + +namespace SharpEmu.Libs.Tests.Ampr; + +public class AmprFileRegistryTests +{ + [Fact] + public void ComputeFileId_matches_utf8_fnv1a() + { + const string relative = "CoreData/foo/bar.bin"; + Assert.Equal(FnvUtf8("$/" + relative), AmprFileRegistry.ComputeFileId("$/" + relative)); + Assert.Equal(FnvUtf8("/app0/" + relative), AmprFileRegistry.ComputeFileId("/app0/" + relative)); + Assert.Equal(FnvUtf8("app0/" + relative), AmprFileRegistry.ComputeFileId("app0/" + relative)); + Assert.Equal(FnvUtf8(relative), AmprFileRegistry.ComputeFileId(relative)); + } + + [Fact] + public void RegisterApp0Relative_publishes_same_ids_as_string_hashes() + { + AmprFileRegistry.ClearForTests(); + const string relative = "misc/loadouts/test.txt"; + var host = Path.Combine(Path.GetTempPath(), "sharpemu-ampr-test", relative); + AmprFileRegistry.RegisterApp0RelativeForTests(relative, host); + + Assert.True(AmprFileRegistry.TryGetHostPath( + AmprFileRegistry.ComputeFileId("$/" + relative), out var a)); + Assert.True(AmprFileRegistry.TryGetHostPath( + AmprFileRegistry.ComputeFileId("/app0/" + relative), out var b)); + Assert.True(AmprFileRegistry.TryGetHostPath( + AmprFileRegistry.ComputeFileId("app0/" + relative), out var c)); + Assert.True(AmprFileRegistry.TryGetHostPath( + AmprFileRegistry.ComputeFileId(relative), out var d)); + Assert.Equal(host, a); + Assert.Equal(host, b); + Assert.Equal(host, c); + Assert.Equal(host, d); + } + + [Fact] + public void Register_publishes_all_app0_path_aliases() + { + AmprFileRegistry.ClearForTests(); + const string relative = "scripts/cp11/cp11main.script"; + var host = Path.Combine(Path.GetTempPath(), "sharpemu-ampr-test2", relative); + AmprFileRegistry.Register("$/" + relative, host); + + Assert.True(AmprFileRegistry.TryGetHostPath( + AmprFileRegistry.ComputeFileId("$/" + relative), out var a)); + Assert.True(AmprFileRegistry.TryGetHostPath( + AmprFileRegistry.ComputeFileId("/app0/" + relative), out var b)); + Assert.True(AmprFileRegistry.TryGetHostPath( + AmprFileRegistry.ComputeFileId("app0/" + relative), out var c)); + Assert.True(AmprFileRegistry.TryGetHostPath( + AmprFileRegistry.ComputeFileId(relative), out var d)); + Assert.Equal(host, a); + Assert.Equal(host, b); + Assert.Equal(host, c); + Assert.Equal(host, d); + } + + private static uint FnvUtf8(string text) + { + const uint offset = 2166136261; + const uint prime = 16777619; + var hash = offset; + foreach (var b in System.Text.Encoding.UTF8.GetBytes(text)) + { + hash ^= b; + hash *= prime; + } + + return hash; + } +} diff --git a/tests/SharpEmu.Libs.Tests/Ampr/AprStreamingContractTests.cs b/tests/SharpEmu.Libs.Tests/Ampr/AprStreamingContractTests.cs index c303c8d1..a3e3be8e 100644 --- a/tests/SharpEmu.Libs.Tests/Ampr/AprStreamingContractTests.cs +++ b/tests/SharpEmu.Libs.Tests/Ampr/AprStreamingContractTests.cs @@ -318,6 +318,53 @@ public sealed class AprStreamingContractTests return BinaryPrimitives.ReadUInt32LittleEndian(bytes); } + [Fact] + public void Register_AlsoPublishesApp0AndDollarPathAliases() + { + // Resolve may register "$/asset.bin" while cooked tables look up + // FNV("/app0/asset.bin"). Both ids must map to the same host file. + var hostPath = Path.Combine(Path.GetTempPath(), $"sharpemu-apr-alias-{Guid.NewGuid():N}.bin"); + File.WriteAllBytes(hostPath, [1, 2, 3]); + try + { + var dollarId = AmprFileRegistry.Register("$/weapons/demo.cani", hostPath); + Assert.True(AmprFileRegistry.TryGetHostPath(dollarId, out var viaDollar)); + Assert.Equal(hostPath, viaDollar); + + var app0Id = AmprFileRegistry.ComputeFileId("/app0/weapons/demo.cani"); + Assert.NotEqual(dollarId, app0Id); + Assert.True(AmprFileRegistry.TryGetHostPath(app0Id, out var viaApp0)); + Assert.Equal(hostPath, viaApp0); + } + finally + { + File.Delete(hostPath); + } + } + + [Fact] + public void EnsureApp0Indexed_PublishesCookedApp0FileIds() + { + var mountRoot = Path.Combine( + Path.GetTempPath(), + $"sharpemu-apr-index-{Guid.NewGuid():N}"); + var relativeDir = Path.Combine(mountRoot, "weapons"); + Directory.CreateDirectory(relativeDir); + var hostPath = Path.Combine(relativeDir, "demo.cani"); + File.WriteAllBytes(hostPath, [9, 8, 7]); + try + { + AmprFileRegistry.EnsureApp0Indexed(mountRoot); + var cookedId = AmprFileRegistry.ComputeFileId("/app0/weapons/demo.cani"); + Assert.True(AmprFileRegistry.TryGetHostPath(cookedId, out var resolved)); + Assert.Equal(Path.GetFullPath(hostPath), Path.GetFullPath(resolved)); + } + finally + { + Directory.Delete(mountRoot, recursive: true); + } + } + private static ulong ReadUInt64(FakeCpuMemory memory, ulong address) { Span bytes = stackalloc byte[sizeof(ulong)];