Ampr: cooked-id index preload and host FD LRU cache (#725)

Preload the app0 APR index during bind and bound the open host-file cache so streaming reads stay responsive under large title archives.
This commit is contained in:
kuba
2026-07-31 11:13:33 +02:00
committed by GitHub
parent 532251c0c3
commit 93c9f14081
6 changed files with 841 additions and 23 deletions
@@ -401,6 +401,9 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime
}
Environment.SetEnvironmentVariable(app0VariableName, app0Root);
// Overlap the cooked-id APR walk with module load so the first ReadFile
// miss mid-boot does not stall on a cold USB index.
SharpEmu.Libs.Ampr.AmprFileRegistry.BeginApp0IndexPreload(app0Root);
return new App0BindingScope(app0VariableName);
}
+142 -14
View File
@@ -2,6 +2,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
using SharpEmu.HLE;
using SharpEmu.Libs.Agc;
using SharpEmu.Libs.Kernel;
using System.Buffers;
using System.Buffers.Binary;
@@ -25,7 +26,6 @@ public static class AmprExports
private const uint KernelEventQueueRecordType = 2;
private const uint WriteAddressRecordType = 3;
private static readonly ConcurrentDictionary<ulong, CommandBufferState> _commandBuffers = new();
private static readonly ConcurrentDictionary<string, Lazy<CachedHostFile>> _hostFileCache = new(StringComparer.OrdinalIgnoreCase);
private static readonly bool _traceAmpr =
string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AMPR"), "1", StringComparison.Ordinal);
private static readonly bool _traceAmprReads =
@@ -40,7 +40,7 @@ public static class AmprExports
public ulong CommandCount;
}
private sealed class CachedHostFile
private sealed class CachedHostFile : IDisposable
{
public CachedHostFile(string path)
{
@@ -55,8 +55,26 @@ public static class AmprExports
public SafeFileHandle Handle { get; }
public long Length { get; }
public void Dispose() => Handle.Dispose();
}
private sealed class CachedHostFileEntry
{
public required string Path { get; init; }
public required CachedHostFile File { get; init; }
}
// Keep a bounded LRU of open host files. An unbounded cache exhausts the
// process FD limit (~10k on macOS) during large asset storms, after
// which every new open throws IOException and surfaces as NOT_FOUND — the
// guest then reports InvalidFileFourCC on empty buffers.
private const int MaxCachedHostFiles = 1536;
private static readonly object _hostFileCacheGate = new();
private static readonly Dictionary<string, LinkedListNode<CachedHostFileEntry>> _hostFileByPath =
new(StringComparer.OrdinalIgnoreCase);
private static readonly LinkedList<CachedHostFileEntry> _hostFileLru = new();
[SysAbiExport(
Nid = "8aI7R7WaOlc",
ExportName = "sceAmprCommandBufferConstructor",
@@ -80,6 +98,7 @@ public static class AmprExports
}
TraceAmpr(ctx, "ctor", commandBuffer, buffer, size);
TryPreindexApp0();
ctx[CpuRegister.Rax] = commandBuffer;
return (int)OrbisGen2Result.ORBIS_GEN2_OK;
}
@@ -107,6 +126,7 @@ public static class AmprExports
}
TraceAmpr(ctx, "apr_ctor", commandBuffer, aux0, aux1);
TryPreindexApp0();
ctx[CpuRegister.Rax] = commandBuffer;
return (int)OrbisGen2Result.ORBIS_GEN2_OK;
}
@@ -271,8 +291,19 @@ public static class AmprExports
if (!AmprFileRegistry.TryGetHostPath(fileId, out var hostPath))
{
TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead: 0, hostPath, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND);
return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND;
// Cooked Insomniac content ids are FNV("/app0/...") and may never
// pass through APR resolve. Index app0 once, then retry the lookup.
var app0Root = KernelMemoryCompatExports.ResolveGuestPath("$/");
if (!string.IsNullOrEmpty(app0Root))
{
AmprFileRegistry.EnsureApp0Indexed(app0Root);
}
if (!AmprFileRegistry.TryGetHostPath(fileId, out hostPath))
{
TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead: 0, hostPath, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND);
return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND;
}
}
// Offset -1 means "continue after the previous read of this file id".
@@ -779,7 +810,9 @@ public static class AmprExports
return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT;
}
const int ChunkSize = 1024 * 1024;
// 4 MiB chunks cut syscall/Rosetta round-trips on DeS' large sequential
// APR reads without blowing the ArrayPool for small probes.
const int ChunkSize = 4 * 1024 * 1024;
var buffer = ArrayPool<byte>.Shared.Rent((int)Math.Min((ulong)ChunkSize, size));
try
@@ -857,27 +890,100 @@ public static class AmprExports
cachePath = hostPath;
}
var lazy = _hostFileCache.GetOrAdd(
cachePath,
static path => new Lazy<CachedHostFile>(() => new CachedHostFile(path), isThreadSafe: true));
lock (_hostFileCacheGate)
{
if (_hostFileByPath.TryGetValue(cachePath, out var existing))
{
_hostFileLru.Remove(existing);
_hostFileLru.AddFirst(existing);
file = existing.Value.File;
return true;
}
}
CachedHostFile opened;
try
{
file = lazy.Value;
return true;
opened = new CachedHostFile(cachePath);
}
catch (UnauthorizedAccessException)
{
_hostFileCache.TryRemove(cachePath, out _);
result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED;
return false;
}
catch (IOException)
{
_hostFileCache.TryRemove(cachePath, out _);
result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND;
return false;
// Likely EMFILE from a prior unbounded cache, or a transient miss.
// Evict everything we hold and retry once so a full FD table can
// recover without restarting the process.
EvictAllCachedHostFiles();
try
{
opened = new CachedHostFile(cachePath);
}
catch (UnauthorizedAccessException)
{
result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED;
return false;
}
catch (IOException)
{
result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND;
return false;
}
}
lock (_hostFileCacheGate)
{
if (_hostFileByPath.TryGetValue(cachePath, out var raced))
{
opened.Dispose();
_hostFileLru.Remove(raced);
_hostFileLru.AddFirst(raced);
file = raced.Value.File;
return true;
}
while (_hostFileByPath.Count >= MaxCachedHostFiles)
{
EvictLeastRecentlyUsedHostFileLocked();
}
var entry = new CachedHostFileEntry { Path = cachePath, File = opened };
var node = _hostFileLru.AddFirst(entry);
_hostFileByPath[cachePath] = node;
file = opened;
return true;
}
}
private static void EvictAllCachedHostFiles()
{
List<CachedHostFile> doomed;
lock (_hostFileCacheGate)
{
doomed = _hostFileLru.Select(entry => entry.File).ToList();
_hostFileLru.Clear();
_hostFileByPath.Clear();
}
foreach (var cached in doomed)
{
cached.Dispose();
}
}
private static void EvictLeastRecentlyUsedHostFileLocked()
{
var last = _hostFileLru.Last;
if (last is null)
{
return;
}
_hostFileLru.RemoveLast();
_hostFileByPath.Remove(last.Value.Path);
last.Value.File.Dispose();
}
private static bool AppendReadFileRecord(
@@ -1004,6 +1110,19 @@ public static class AmprExports
return false;
}
// GPU WAIT_REG_MEM often watches these APR completion labels
// (e.g. 0x20505xxx DEADBEEF/counter fences). Without
// RecordProduced the wait sits producerless after the guest recycles
// the dword, and CollectDeadlockBroken cannot replay the wake.
_ = GpuWaitRegistry.RecordProduced(ctx.Memory, address, value);
if ((address & 4ul) == 0)
{
// 32-bit GPU waits use the low dword; also latch that view when the
// address is dword-aligned so a u32 compare against ref sees it.
_ = GpuWaitRegistry.RecordProduced(
ctx.Memory, address, unchecked((uint)value));
}
TraceAmpr(ctx, "complete_write_address", address, value, 0);
return true;
}
@@ -1021,6 +1140,15 @@ public static class AmprExports
return true;
}
private static void TryPreindexApp0()
{
var app0Root = KernelMemoryCompatExports.ResolveGuestPath("$/");
if (!string.IsNullOrEmpty(app0Root))
{
AmprFileRegistry.EnsureApp0Indexed(app0Root);
}
}
private static void TraceAmpr(CpuContext ctx, string operation, ulong commandBuffer, ulong arg0, ulong arg1)
{
if (!_traceAmpr)
+570 -9
View File
@@ -2,15 +2,35 @@
// SPDX-License-Identifier: GPL-2.0-or-later
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Text;
namespace SharpEmu.Libs.Ampr;
internal static class AmprFileRegistry
{
private static readonly ConcurrentDictionary<uint, string> _hostPathsById = new();
private const uint CacheMagicV2 = 0x32495041u; // 'API2'
private const uint CacheVersionV2 = 2;
private const uint CacheMagicV3 = 0x33495041u; // 'API3'
private const uint CacheVersionV3 = 3;
private static readonly ConcurrentDictionary<uint, string> _hostPathsById = new(
concurrencyLevel: Math.Max(4, Environment.ProcessorCount),
capacity: 1_048_576);
private static readonly object _indexGate = new();
private static string? _indexedApp0Root;
private static string? _indexingApp0Root;
private static int _preloadStarted;
public static uint Register(string guestPath, string hostPath)
{
if (TryGetApp0Relative(guestPath, out var relative) && relative.Length != 0)
{
RegisterApp0Relative(relative, hostPath);
return ComputeFileId("$/" + relative);
}
var id = ComputeFileId(guestPath);
_hostPathsById[id] = hostPath;
return id;
@@ -21,18 +41,559 @@ internal static class AmprFileRegistry
return _hostPathsById.TryGetValue(id, out hostPath!);
}
/// <summary>Test hook: wipe registry state between cases.</summary>
internal static void ClearForTests()
{
lock (_indexGate)
{
_hostPathsById.Clear();
_indexedApp0Root = null;
_indexingApp0Root = null;
_preloadStarted = 0;
}
}
/// <summary>Test hook for the allocation-free alias publisher.</summary>
internal static void RegisterApp0RelativeForTests(string relative, string hostPath) =>
RegisterApp0Relative(relative, hostPath);
/// <summary>
/// Kick off <see cref="EnsureApp0Indexed"/> on a background thread as soon as
/// the host knows app0. otherwise pays the full tree walk on the
/// first cooked-id APR miss mid-boot (~8s under Rosetta for DeS).
/// </summary>
public static void BeginApp0IndexPreload(string? app0Root)
{
if (string.IsNullOrWhiteSpace(app0Root) || !Directory.Exists(app0Root))
{
return;
}
if (Interlocked.Exchange(ref _preloadStarted, 1) != 0)
{
return;
}
var root = app0Root;
ThreadPool.UnsafeQueueUserWorkItem(
static state =>
{
try
{
EnsureApp0Indexed((string)state!);
}
catch (Exception exception)
{
Console.Error.WriteLine(
$"[LOADER][WARN] ampr.app0_index_preload_failed: {exception.Message}");
}
},
root);
}
/// <summary>
/// Indexes every file under app0 under both <c>$/</c> and <c>/app0/</c> FNV
/// ids. Cooked asset tables ship precomputed ids; without this walk, a title
/// that never resolves those paths through APR leaves ReadFile permanently
/// NOT_FOUND.
/// </summary>
public static void EnsureApp0Indexed(string app0Root)
{
if (string.IsNullOrWhiteSpace(app0Root) || !Directory.Exists(app0Root))
{
return;
}
var normalizedRoot = Path.GetFullPath(app0Root);
var stopwatch = Stopwatch.StartNew();
lock (_indexGate)
{
while (true)
{
if (string.Equals(_indexedApp0Root, normalizedRoot, StringComparison.OrdinalIgnoreCase))
{
return;
}
if (_indexingApp0Root is not null)
{
// Another thread (preload) owns the walk — wait instead of
// stacking a second 8s index on the guest APR miss path.
if (string.Equals(
_indexingApp0Root,
normalizedRoot,
StringComparison.OrdinalIgnoreCase))
{
Monitor.Wait(_indexGate);
continue;
}
Monitor.Wait(_indexGate, 50);
continue;
}
_indexingApp0Root = normalizedRoot;
break;
}
}
try
{
var cachePathV3 = GetIndexCachePath(normalizedRoot, version: 3);
var cachePathV2 = GetIndexCachePath(normalizedRoot, version: 2);
if (TryLoadIndexCache(normalizedRoot, cachePathV3, preferV3: true, out var cachedFiles))
{
lock (_indexGate)
{
_indexedApp0Root = normalizedRoot;
}
Console.Error.WriteLine(
$"[LOADER][INFO] ampr.app0_index_cache_hit root={normalizedRoot} " +
$"files={cachedFiles} ids={_hostPathsById.Count} " +
$"elapsed_ms={stopwatch.Elapsed.TotalMilliseconds:F1}");
return;
}
if (TryLoadIndexCache(normalizedRoot, cachePathV2, preferV3: false, out cachedFiles))
{
lock (_indexGate)
{
_indexedApp0Root = normalizedRoot;
}
// Promote v2 (rehash-on-load) to v3 (precomputed ids) so the
// next boot skips the Rosetta FNV storm.
TrySaveIndexCache(normalizedRoot, cachePathV3, cachedFiles);
Console.Error.WriteLine(
$"[LOADER][INFO] ampr.app0_index_cache_hit root={normalizedRoot} " +
$"files={cachedFiles} ids={_hostPathsById.Count} " +
$"elapsed_ms={stopwatch.Elapsed.TotalMilliseconds:F1} upgraded=v3");
return;
}
var relatives = new List<string>(256 * 1024);
foreach (var hostPath in Directory.EnumerateFiles(
normalizedRoot,
"*",
SearchOption.AllDirectories))
{
var relative = Path.GetRelativePath(normalizedRoot, hostPath)
.Replace('\\', '/');
if (string.IsNullOrEmpty(relative) ||
relative.StartsWith("..", StringComparison.Ordinal))
{
continue;
}
relatives.Add(relative);
}
// Hash + dictionary fill dominates under Rosetta once the walk is
// done; parallelize across cores without re-walking the tree.
Parallel.ForEach(
relatives,
new ParallelOptions
{
MaxDegreeOfParallelism = Math.Max(2, Environment.ProcessorCount - 1),
},
relative =>
{
var hostPath = Path.Combine(normalizedRoot, relative.Replace('/', Path.DirectorySeparatorChar));
RegisterApp0Relative(relative, hostPath);
});
lock (_indexGate)
{
_indexedApp0Root = normalizedRoot;
}
TrySaveIndexCache(normalizedRoot, cachePathV3, relatives.Count);
Console.Error.WriteLine(
$"[LOADER][INFO] ampr.app0_indexed root={normalizedRoot} " +
$"files={relatives.Count} ids={_hostPathsById.Count} " +
$"elapsed_ms={stopwatch.Elapsed.TotalMilliseconds:F1}");
}
finally
{
lock (_indexGate)
{
_indexingApp0Root = null;
Monitor.PulseAll(_indexGate);
}
}
}
/// <summary>
/// Registers the four Insomniac path aliases for one app0-relative file
/// without allocating intermediate guest-path strings.
/// </summary>
private static void RegisterApp0Relative(string relative, string hostPath)
{
// "$/" + relative
Publish(FnvContinueAscii(FnvContinueAscii(OffsetBasis, (byte)'$'), (byte)'/'), relative, hostPath);
// "/app0/" + relative
Publish(FnvContinueAsciiPrefix(OffsetBasis, "/app0/"u8), relative, hostPath);
// "app0/" + relative
Publish(FnvContinueAsciiPrefix(OffsetBasis, "app0/"u8), relative, hostPath);
// bare relative
Publish(OffsetBasis, relative, hostPath);
}
private static void Publish(uint hash, string relative, string hostPath)
{
_hostPathsById[FnvContinueUtf8(hash, relative)] = hostPath;
}
internal static uint ComputeFileId(string guestPath)
{
var bytes = System.Text.Encoding.UTF8.GetBytes(guestPath);
return FnvContinueUtf8(OffsetBasis, guestPath);
}
const uint offsetBasis = 2166136261;
const uint prime = 16777619;
var hash = offsetBasis;
foreach (var b in bytes)
internal static IEnumerable<string> EnumerateApp0PathAliases(string guestPath)
{
if (string.IsNullOrEmpty(guestPath))
{
hash ^= b;
hash *= prime;
yield break;
}
if (!TryGetApp0Relative(guestPath, out var relative) ||
string.IsNullOrEmpty(relative))
{
yield break;
}
yield return "$/" + relative;
yield return "/app0/" + relative;
yield return "app0/" + relative;
yield return relative;
}
private static bool TryGetApp0Relative(string guestPath, out string relative)
{
relative = string.Empty;
var normalized = guestPath.Replace('\\', '/');
if (normalized.StartsWith("$/", StringComparison.Ordinal))
{
relative = normalized[2..].TrimStart('/');
return relative.Length != 0;
}
if (normalized.StartsWith("/app0/", StringComparison.OrdinalIgnoreCase))
{
relative = normalized["/app0/".Length..].TrimStart('/');
return relative.Length != 0;
}
if (normalized.StartsWith("app0/", StringComparison.OrdinalIgnoreCase))
{
relative = normalized["app0/".Length..].TrimStart('/');
return relative.Length != 0;
}
// Bare relative paths are treated as app0-relative by ResolveGuestPath.
if (!normalized.StartsWith('/') &&
!Path.IsPathFullyQualified(guestPath))
{
relative = normalized.TrimStart('/');
return relative.Length != 0;
}
return false;
}
private static string GetIndexCachePath(string normalizedRoot, int version)
{
var overrideDir = Environment.GetEnvironmentVariable("SHARPEMU_AMPR_INDEX_CACHE");
var cacheDir = !string.IsNullOrWhiteSpace(overrideDir)
? overrideDir
: Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"SharpEmu",
"ampr-index");
Directory.CreateDirectory(cacheDir);
var rootHash = ComputeFileId(normalizedRoot.ToLowerInvariant());
return Path.Combine(cacheDir, $"app0-{rootHash:x8}.v{version}.idx");
}
private static bool TryLoadIndexCache(
string normalizedRoot,
string cachePath,
bool preferV3,
out int fileCount)
{
fileCount = 0;
try
{
if (!File.Exists(cachePath))
{
return false;
}
if (string.Equals(
Environment.GetEnvironmentVariable("SHARPEMU_AMPR_REINDEX"),
"1",
StringComparison.Ordinal))
{
return false;
}
using var stream = File.OpenRead(cachePath);
using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: false);
var magic = reader.ReadUInt32();
var version = reader.ReadUInt32();
var isV3 = magic == CacheMagicV3 && version == CacheVersionV3;
var isV2 = magic == CacheMagicV2 && version == CacheVersionV2;
if (preferV3)
{
if (!isV3)
{
return false;
}
}
else if (!isV2)
{
return false;
}
var root = reader.ReadString();
if (!string.Equals(root, normalizedRoot, StringComparison.OrdinalIgnoreCase))
{
return false;
}
var expectedFiles = reader.ReadInt32();
var expectedParamTicks = reader.ReadInt64();
var actualParamTicks = GetParamJsonWriteTicks(normalizedRoot);
if (actualParamTicks == 0 || actualParamTicks != expectedParamTicks)
{
return false;
}
if (expectedFiles < 0 || expectedFiles > 8_000_000)
{
return false;
}
if (isV3)
{
// Precomputed FNV ids — no Rosetta hash storm on every boot.
var entries = new (string Relative, uint Id0, uint Id1, uint Id2, uint Id3)[expectedFiles];
for (var i = 0; i < expectedFiles; i++)
{
entries[i] = (
reader.ReadString(),
reader.ReadUInt32(),
reader.ReadUInt32(),
reader.ReadUInt32(),
reader.ReadUInt32());
}
Parallel.ForEach(
entries,
new ParallelOptions
{
MaxDegreeOfParallelism = Math.Max(2, Environment.ProcessorCount - 1),
},
entry =>
{
if (string.IsNullOrEmpty(entry.Relative) ||
entry.Relative.Contains("..", StringComparison.Ordinal))
{
return;
}
var hostPath = normalizedRoot.EndsWith(Path.DirectorySeparatorChar)
? normalizedRoot + entry.Relative.Replace('/', Path.DirectorySeparatorChar)
: normalizedRoot + Path.DirectorySeparatorChar +
entry.Relative.Replace('/', Path.DirectorySeparatorChar);
_hostPathsById[entry.Id0] = hostPath;
_hostPathsById[entry.Id1] = hostPath;
_hostPathsById[entry.Id2] = hostPath;
_hostPathsById[entry.Id3] = hostPath;
});
}
else
{
var relatives = new string[expectedFiles];
for (var i = 0; i < expectedFiles; i++)
{
relatives[i] = reader.ReadString();
}
Parallel.ForEach(
relatives,
new ParallelOptions
{
MaxDegreeOfParallelism = Math.Max(2, Environment.ProcessorCount - 1),
},
relative =>
{
if (string.IsNullOrEmpty(relative) ||
relative.Contains("..", StringComparison.Ordinal))
{
return;
}
var hostPath = Path.Combine(
normalizedRoot,
relative.Replace('/', Path.DirectorySeparatorChar));
RegisterApp0Relative(relative, hostPath);
});
}
fileCount = expectedFiles;
return true;
}
catch (Exception exception)
{
_hostPathsById.Clear();
Console.Error.WriteLine(
$"[LOADER][WARN] ampr.app0_index_cache_load_failed: {exception.Message}");
return false;
}
}
private static void TrySaveIndexCache(
string normalizedRoot,
string cachePath,
int fileCount)
{
try
{
var paramTicks = GetParamJsonWriteTicks(normalizedRoot);
if (paramTicks == 0 || fileCount <= 0)
{
return;
}
var relatives = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var hostPath in _hostPathsById.Values)
{
var relative = Path.GetRelativePath(normalizedRoot, hostPath)
.Replace('\\', '/');
if (string.IsNullOrEmpty(relative) ||
relative.StartsWith("..", StringComparison.Ordinal))
{
continue;
}
relatives.Add(relative);
}
var tempPath = cachePath + ".tmp";
using (var stream = File.Create(tempPath))
using (var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: false))
{
writer.Write(CacheMagicV3);
writer.Write(CacheVersionV3);
writer.Write(normalizedRoot);
writer.Write(relatives.Count);
writer.Write(paramTicks);
foreach (var relative in relatives)
{
writer.Write(relative);
// Mirror RegisterApp0Relative id order: $/ /app0/ app0/ bare.
writer.Write(ComputeApp0AliasIds(relative, out var id1, out var id2, out var id3));
writer.Write(id1);
writer.Write(id2);
writer.Write(id3);
}
}
File.Move(tempPath, cachePath, overwrite: true);
Console.Error.WriteLine(
$"[LOADER][INFO] ampr.app0_index_cache_saved path={cachePath} " +
$"files={relatives.Count}");
}
catch (Exception exception)
{
Console.Error.WriteLine(
$"[LOADER][WARN] ampr.app0_index_cache_save_failed: {exception.Message}");
}
}
private static uint ComputeApp0AliasIds(
string relative,
out uint app0Slash,
out uint app0,
out uint bare)
{
var dollar = FnvContinueUtf8(
FnvContinueAscii(FnvContinueAscii(OffsetBasis, (byte)'$'), (byte)'/'),
relative);
app0Slash = FnvContinueUtf8(FnvContinueAsciiPrefix(OffsetBasis, "/app0/"u8), relative);
app0 = FnvContinueUtf8(FnvContinueAsciiPrefix(OffsetBasis, "app0/"u8), relative);
bare = FnvContinueUtf8(OffsetBasis, relative);
return dollar;
}
/// <summary>
/// Cheap dump fingerprint. Full-tree walks are too expensive for cache
/// validation; param.json changes with title updates. Force a rebuild with
/// SHARPEMU_AMPR_REINDEX=1 after manual dump edits.
/// </summary>
private static long GetParamJsonWriteTicks(string normalizedRoot)
{
try
{
var paramPath = Path.Combine(normalizedRoot, "sce_sys", "param.json");
return File.Exists(paramPath)
? File.GetLastWriteTimeUtc(paramPath).Ticks
: 0;
}
catch
{
return 0;
}
}
private const uint OffsetBasis = 2166136261;
private const uint FnvPrime = 16777619;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint FnvContinueAscii(uint hash, byte value)
{
hash ^= value;
return hash * FnvPrime;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint FnvContinueAsciiPrefix(uint hash, ReadOnlySpan<byte> ascii)
{
foreach (var value in ascii)
{
hash ^= value;
hash *= FnvPrime;
}
return hash;
}
private static uint FnvContinueUtf8(uint hash, string text)
{
// Game asset paths are overwhelmingly ASCII; avoid Encoding.GetBytes
// allocations on the 223k-file DeS index hot path.
Span<byte> utf8Scratch = stackalloc byte[4];
for (var i = 0; i < text.Length; i++)
{
var ch = text[i];
if (ch < 0x80)
{
hash ^= (byte)ch;
hash *= FnvPrime;
continue;
}
var written = Encoding.UTF8.GetBytes(text.AsSpan(i, 1), utf8Scratch);
for (var b = 0; b < written; b++)
{
hash ^= utf8Scratch[b];
hash *= FnvPrime;
}
}
return hash;
+1
View File
@@ -24,6 +24,7 @@ SPDX-License-Identifier: GPL-2.0-or-later
<ItemGroup>
<InternalsVisibleTo Include="SharpEmu.Libs.Tests" />
<InternalsVisibleTo Include="SharpEmu.Core" />
</ItemGroup>
<ItemGroup>
@@ -0,0 +1,78 @@
// Copyright (C) 2026 SharpEmu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
using SharpEmu.Libs.Ampr;
using Xunit;
namespace SharpEmu.Libs.Tests.Ampr;
public class AmprFileRegistryTests
{
[Fact]
public void ComputeFileId_matches_utf8_fnv1a()
{
const string relative = "CoreData/foo/bar.bin";
Assert.Equal(FnvUtf8("$/" + relative), AmprFileRegistry.ComputeFileId("$/" + relative));
Assert.Equal(FnvUtf8("/app0/" + relative), AmprFileRegistry.ComputeFileId("/app0/" + relative));
Assert.Equal(FnvUtf8("app0/" + relative), AmprFileRegistry.ComputeFileId("app0/" + relative));
Assert.Equal(FnvUtf8(relative), AmprFileRegistry.ComputeFileId(relative));
}
[Fact]
public void RegisterApp0Relative_publishes_same_ids_as_string_hashes()
{
AmprFileRegistry.ClearForTests();
const string relative = "misc/loadouts/test.txt";
var host = Path.Combine(Path.GetTempPath(), "sharpemu-ampr-test", relative);
AmprFileRegistry.RegisterApp0RelativeForTests(relative, host);
Assert.True(AmprFileRegistry.TryGetHostPath(
AmprFileRegistry.ComputeFileId("$/" + relative), out var a));
Assert.True(AmprFileRegistry.TryGetHostPath(
AmprFileRegistry.ComputeFileId("/app0/" + relative), out var b));
Assert.True(AmprFileRegistry.TryGetHostPath(
AmprFileRegistry.ComputeFileId("app0/" + relative), out var c));
Assert.True(AmprFileRegistry.TryGetHostPath(
AmprFileRegistry.ComputeFileId(relative), out var d));
Assert.Equal(host, a);
Assert.Equal(host, b);
Assert.Equal(host, c);
Assert.Equal(host, d);
}
[Fact]
public void Register_publishes_all_app0_path_aliases()
{
AmprFileRegistry.ClearForTests();
const string relative = "scripts/cp11/cp11main.script";
var host = Path.Combine(Path.GetTempPath(), "sharpemu-ampr-test2", relative);
AmprFileRegistry.Register("$/" + relative, host);
Assert.True(AmprFileRegistry.TryGetHostPath(
AmprFileRegistry.ComputeFileId("$/" + relative), out var a));
Assert.True(AmprFileRegistry.TryGetHostPath(
AmprFileRegistry.ComputeFileId("/app0/" + relative), out var b));
Assert.True(AmprFileRegistry.TryGetHostPath(
AmprFileRegistry.ComputeFileId("app0/" + relative), out var c));
Assert.True(AmprFileRegistry.TryGetHostPath(
AmprFileRegistry.ComputeFileId(relative), out var d));
Assert.Equal(host, a);
Assert.Equal(host, b);
Assert.Equal(host, c);
Assert.Equal(host, d);
}
private static uint FnvUtf8(string text)
{
const uint offset = 2166136261;
const uint prime = 16777619;
var hash = offset;
foreach (var b in System.Text.Encoding.UTF8.GetBytes(text))
{
hash ^= b;
hash *= prime;
}
return hash;
}
}
@@ -318,6 +318,53 @@ public sealed class AprStreamingContractTests
return BinaryPrimitives.ReadUInt32LittleEndian(bytes);
}
[Fact]
public void Register_AlsoPublishesApp0AndDollarPathAliases()
{
// Resolve may register "$/asset.bin" while cooked tables look up
// FNV("/app0/asset.bin"). Both ids must map to the same host file.
var hostPath = Path.Combine(Path.GetTempPath(), $"sharpemu-apr-alias-{Guid.NewGuid():N}.bin");
File.WriteAllBytes(hostPath, [1, 2, 3]);
try
{
var dollarId = AmprFileRegistry.Register("$/weapons/demo.cani", hostPath);
Assert.True(AmprFileRegistry.TryGetHostPath(dollarId, out var viaDollar));
Assert.Equal(hostPath, viaDollar);
var app0Id = AmprFileRegistry.ComputeFileId("/app0/weapons/demo.cani");
Assert.NotEqual(dollarId, app0Id);
Assert.True(AmprFileRegistry.TryGetHostPath(app0Id, out var viaApp0));
Assert.Equal(hostPath, viaApp0);
}
finally
{
File.Delete(hostPath);
}
}
[Fact]
public void EnsureApp0Indexed_PublishesCookedApp0FileIds()
{
var mountRoot = Path.Combine(
Path.GetTempPath(),
$"sharpemu-apr-index-{Guid.NewGuid():N}");
var relativeDir = Path.Combine(mountRoot, "weapons");
Directory.CreateDirectory(relativeDir);
var hostPath = Path.Combine(relativeDir, "demo.cani");
File.WriteAllBytes(hostPath, [9, 8, 7]);
try
{
AmprFileRegistry.EnsureApp0Indexed(mountRoot);
var cookedId = AmprFileRegistry.ComputeFileId("/app0/weapons/demo.cani");
Assert.True(AmprFileRegistry.TryGetHostPath(cookedId, out var resolved));
Assert.Equal(Path.GetFullPath(hostPath), Path.GetFullPath(resolved));
}
finally
{
Directory.Delete(mountRoot, recursive: true);
}
}
private static ulong ReadUInt64(FakeCpuMemory memory, ulong address)
{
Span<byte> bytes = stackalloc byte[sizeof(ulong)];