mirror of
https://github.com/par274/sharpemu.git
synced 2026-07-31 23:19:44 +08:00
Ampr: cooked-id index preload and host FD LRU cache (#725)
Preload the app0 APR index during bind and bound the open host-file cache so streaming reads stay responsive under large title archives.
This commit is contained in:
@@ -401,6 +401,9 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime
|
||||
}
|
||||
|
||||
Environment.SetEnvironmentVariable(app0VariableName, app0Root);
|
||||
// Overlap the cooked-id APR walk with module load so the first ReadFile
|
||||
// miss mid-boot does not stall on a cold USB index.
|
||||
SharpEmu.Libs.Ampr.AmprFileRegistry.BeginApp0IndexPreload(app0Root);
|
||||
return new App0BindingScope(app0VariableName);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
using SharpEmu.HLE;
|
||||
using SharpEmu.Libs.Agc;
|
||||
using SharpEmu.Libs.Kernel;
|
||||
using System.Buffers;
|
||||
using System.Buffers.Binary;
|
||||
@@ -25,7 +26,6 @@ public static class AmprExports
|
||||
private const uint KernelEventQueueRecordType = 2;
|
||||
private const uint WriteAddressRecordType = 3;
|
||||
private static readonly ConcurrentDictionary<ulong, CommandBufferState> _commandBuffers = new();
|
||||
private static readonly ConcurrentDictionary<string, Lazy<CachedHostFile>> _hostFileCache = new(StringComparer.OrdinalIgnoreCase);
|
||||
private static readonly bool _traceAmpr =
|
||||
string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AMPR"), "1", StringComparison.Ordinal);
|
||||
private static readonly bool _traceAmprReads =
|
||||
@@ -40,7 +40,7 @@ public static class AmprExports
|
||||
public ulong CommandCount;
|
||||
}
|
||||
|
||||
private sealed class CachedHostFile
|
||||
private sealed class CachedHostFile : IDisposable
|
||||
{
|
||||
public CachedHostFile(string path)
|
||||
{
|
||||
@@ -55,8 +55,26 @@ public static class AmprExports
|
||||
|
||||
public SafeFileHandle Handle { get; }
|
||||
public long Length { get; }
|
||||
|
||||
public void Dispose() => Handle.Dispose();
|
||||
}
|
||||
|
||||
private sealed class CachedHostFileEntry
|
||||
{
|
||||
public required string Path { get; init; }
|
||||
public required CachedHostFile File { get; init; }
|
||||
}
|
||||
|
||||
// Keep a bounded LRU of open host files. An unbounded cache exhausts the
|
||||
// process FD limit (~10k on macOS) during large asset storms, after
|
||||
// which every new open throws IOException and surfaces as NOT_FOUND — the
|
||||
// guest then reports InvalidFileFourCC on empty buffers.
|
||||
private const int MaxCachedHostFiles = 1536;
|
||||
private static readonly object _hostFileCacheGate = new();
|
||||
private static readonly Dictionary<string, LinkedListNode<CachedHostFileEntry>> _hostFileByPath =
|
||||
new(StringComparer.OrdinalIgnoreCase);
|
||||
private static readonly LinkedList<CachedHostFileEntry> _hostFileLru = new();
|
||||
|
||||
[SysAbiExport(
|
||||
Nid = "8aI7R7WaOlc",
|
||||
ExportName = "sceAmprCommandBufferConstructor",
|
||||
@@ -80,6 +98,7 @@ public static class AmprExports
|
||||
}
|
||||
|
||||
TraceAmpr(ctx, "ctor", commandBuffer, buffer, size);
|
||||
TryPreindexApp0();
|
||||
ctx[CpuRegister.Rax] = commandBuffer;
|
||||
return (int)OrbisGen2Result.ORBIS_GEN2_OK;
|
||||
}
|
||||
@@ -107,6 +126,7 @@ public static class AmprExports
|
||||
}
|
||||
|
||||
TraceAmpr(ctx, "apr_ctor", commandBuffer, aux0, aux1);
|
||||
TryPreindexApp0();
|
||||
ctx[CpuRegister.Rax] = commandBuffer;
|
||||
return (int)OrbisGen2Result.ORBIS_GEN2_OK;
|
||||
}
|
||||
@@ -271,8 +291,19 @@ public static class AmprExports
|
||||
|
||||
if (!AmprFileRegistry.TryGetHostPath(fileId, out var hostPath))
|
||||
{
|
||||
TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead: 0, hostPath, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND);
|
||||
return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND;
|
||||
// Cooked Insomniac content ids are FNV("/app0/...") and may never
|
||||
// pass through APR resolve. Index app0 once, then retry the lookup.
|
||||
var app0Root = KernelMemoryCompatExports.ResolveGuestPath("$/");
|
||||
if (!string.IsNullOrEmpty(app0Root))
|
||||
{
|
||||
AmprFileRegistry.EnsureApp0Indexed(app0Root);
|
||||
}
|
||||
|
||||
if (!AmprFileRegistry.TryGetHostPath(fileId, out hostPath))
|
||||
{
|
||||
TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead: 0, hostPath, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND);
|
||||
return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND;
|
||||
}
|
||||
}
|
||||
|
||||
// Offset -1 means "continue after the previous read of this file id".
|
||||
@@ -779,7 +810,9 @@ public static class AmprExports
|
||||
return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT;
|
||||
}
|
||||
|
||||
const int ChunkSize = 1024 * 1024;
|
||||
// 4 MiB chunks cut syscall/Rosetta round-trips on DeS' large sequential
|
||||
// APR reads without blowing the ArrayPool for small probes.
|
||||
const int ChunkSize = 4 * 1024 * 1024;
|
||||
var buffer = ArrayPool<byte>.Shared.Rent((int)Math.Min((ulong)ChunkSize, size));
|
||||
|
||||
try
|
||||
@@ -857,27 +890,100 @@ public static class AmprExports
|
||||
cachePath = hostPath;
|
||||
}
|
||||
|
||||
var lazy = _hostFileCache.GetOrAdd(
|
||||
cachePath,
|
||||
static path => new Lazy<CachedHostFile>(() => new CachedHostFile(path), isThreadSafe: true));
|
||||
lock (_hostFileCacheGate)
|
||||
{
|
||||
if (_hostFileByPath.TryGetValue(cachePath, out var existing))
|
||||
{
|
||||
_hostFileLru.Remove(existing);
|
||||
_hostFileLru.AddFirst(existing);
|
||||
file = existing.Value.File;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
CachedHostFile opened;
|
||||
try
|
||||
{
|
||||
file = lazy.Value;
|
||||
return true;
|
||||
opened = new CachedHostFile(cachePath);
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
_hostFileCache.TryRemove(cachePath, out _);
|
||||
result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED;
|
||||
return false;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
_hostFileCache.TryRemove(cachePath, out _);
|
||||
result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND;
|
||||
return false;
|
||||
// Likely EMFILE from a prior unbounded cache, or a transient miss.
|
||||
// Evict everything we hold and retry once so a full FD table can
|
||||
// recover without restarting the process.
|
||||
EvictAllCachedHostFiles();
|
||||
try
|
||||
{
|
||||
opened = new CachedHostFile(cachePath);
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED;
|
||||
return false;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
lock (_hostFileCacheGate)
|
||||
{
|
||||
if (_hostFileByPath.TryGetValue(cachePath, out var raced))
|
||||
{
|
||||
opened.Dispose();
|
||||
_hostFileLru.Remove(raced);
|
||||
_hostFileLru.AddFirst(raced);
|
||||
file = raced.Value.File;
|
||||
return true;
|
||||
}
|
||||
|
||||
while (_hostFileByPath.Count >= MaxCachedHostFiles)
|
||||
{
|
||||
EvictLeastRecentlyUsedHostFileLocked();
|
||||
}
|
||||
|
||||
var entry = new CachedHostFileEntry { Path = cachePath, File = opened };
|
||||
var node = _hostFileLru.AddFirst(entry);
|
||||
_hostFileByPath[cachePath] = node;
|
||||
file = opened;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static void EvictAllCachedHostFiles()
|
||||
{
|
||||
List<CachedHostFile> doomed;
|
||||
lock (_hostFileCacheGate)
|
||||
{
|
||||
doomed = _hostFileLru.Select(entry => entry.File).ToList();
|
||||
_hostFileLru.Clear();
|
||||
_hostFileByPath.Clear();
|
||||
}
|
||||
|
||||
foreach (var cached in doomed)
|
||||
{
|
||||
cached.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private static void EvictLeastRecentlyUsedHostFileLocked()
|
||||
{
|
||||
var last = _hostFileLru.Last;
|
||||
if (last is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_hostFileLru.RemoveLast();
|
||||
_hostFileByPath.Remove(last.Value.Path);
|
||||
last.Value.File.Dispose();
|
||||
}
|
||||
|
||||
private static bool AppendReadFileRecord(
|
||||
@@ -1004,6 +1110,19 @@ public static class AmprExports
|
||||
return false;
|
||||
}
|
||||
|
||||
// GPU WAIT_REG_MEM often watches these APR completion labels
|
||||
// (e.g. 0x20505xxx DEADBEEF/counter fences). Without
|
||||
// RecordProduced the wait sits producerless after the guest recycles
|
||||
// the dword, and CollectDeadlockBroken cannot replay the wake.
|
||||
_ = GpuWaitRegistry.RecordProduced(ctx.Memory, address, value);
|
||||
if ((address & 4ul) == 0)
|
||||
{
|
||||
// 32-bit GPU waits use the low dword; also latch that view when the
|
||||
// address is dword-aligned so a u32 compare against ref sees it.
|
||||
_ = GpuWaitRegistry.RecordProduced(
|
||||
ctx.Memory, address, unchecked((uint)value));
|
||||
}
|
||||
|
||||
TraceAmpr(ctx, "complete_write_address", address, value, 0);
|
||||
return true;
|
||||
}
|
||||
@@ -1021,6 +1140,15 @@ public static class AmprExports
|
||||
return true;
|
||||
}
|
||||
|
||||
private static void TryPreindexApp0()
|
||||
{
|
||||
var app0Root = KernelMemoryCompatExports.ResolveGuestPath("$/");
|
||||
if (!string.IsNullOrEmpty(app0Root))
|
||||
{
|
||||
AmprFileRegistry.EnsureApp0Indexed(app0Root);
|
||||
}
|
||||
}
|
||||
|
||||
private static void TraceAmpr(CpuContext ctx, string operation, ulong commandBuffer, ulong arg0, ulong arg1)
|
||||
{
|
||||
if (!_traceAmpr)
|
||||
|
||||
@@ -2,15 +2,35 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
|
||||
namespace SharpEmu.Libs.Ampr;
|
||||
|
||||
internal static class AmprFileRegistry
|
||||
{
|
||||
private static readonly ConcurrentDictionary<uint, string> _hostPathsById = new();
|
||||
private const uint CacheMagicV2 = 0x32495041u; // 'API2'
|
||||
private const uint CacheVersionV2 = 2;
|
||||
private const uint CacheMagicV3 = 0x33495041u; // 'API3'
|
||||
private const uint CacheVersionV3 = 3;
|
||||
|
||||
private static readonly ConcurrentDictionary<uint, string> _hostPathsById = new(
|
||||
concurrencyLevel: Math.Max(4, Environment.ProcessorCount),
|
||||
capacity: 1_048_576);
|
||||
private static readonly object _indexGate = new();
|
||||
private static string? _indexedApp0Root;
|
||||
private static string? _indexingApp0Root;
|
||||
private static int _preloadStarted;
|
||||
|
||||
public static uint Register(string guestPath, string hostPath)
|
||||
{
|
||||
if (TryGetApp0Relative(guestPath, out var relative) && relative.Length != 0)
|
||||
{
|
||||
RegisterApp0Relative(relative, hostPath);
|
||||
return ComputeFileId("$/" + relative);
|
||||
}
|
||||
|
||||
var id = ComputeFileId(guestPath);
|
||||
_hostPathsById[id] = hostPath;
|
||||
return id;
|
||||
@@ -21,18 +41,559 @@ internal static class AmprFileRegistry
|
||||
return _hostPathsById.TryGetValue(id, out hostPath!);
|
||||
}
|
||||
|
||||
/// <summary>Test hook: wipe registry state between cases.</summary>
|
||||
internal static void ClearForTests()
|
||||
{
|
||||
lock (_indexGate)
|
||||
{
|
||||
_hostPathsById.Clear();
|
||||
_indexedApp0Root = null;
|
||||
_indexingApp0Root = null;
|
||||
_preloadStarted = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test hook for the allocation-free alias publisher.</summary>
|
||||
internal static void RegisterApp0RelativeForTests(string relative, string hostPath) =>
|
||||
RegisterApp0Relative(relative, hostPath);
|
||||
|
||||
/// <summary>
|
||||
/// Kick off <see cref="EnsureApp0Indexed"/> on a background thread as soon as
|
||||
/// the host knows app0. otherwise pays the full tree walk on the
|
||||
/// first cooked-id APR miss mid-boot (~8s under Rosetta for DeS).
|
||||
/// </summary>
|
||||
public static void BeginApp0IndexPreload(string? app0Root)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(app0Root) || !Directory.Exists(app0Root))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (Interlocked.Exchange(ref _preloadStarted, 1) != 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var root = app0Root;
|
||||
ThreadPool.UnsafeQueueUserWorkItem(
|
||||
static state =>
|
||||
{
|
||||
try
|
||||
{
|
||||
EnsureApp0Indexed((string)state!);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Console.Error.WriteLine(
|
||||
$"[LOADER][WARN] ampr.app0_index_preload_failed: {exception.Message}");
|
||||
}
|
||||
},
|
||||
root);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Indexes every file under app0 under both <c>$/</c> and <c>/app0/</c> FNV
|
||||
/// ids. Cooked asset tables ship precomputed ids; without this walk, a title
|
||||
/// that never resolves those paths through APR leaves ReadFile permanently
|
||||
/// NOT_FOUND.
|
||||
/// </summary>
|
||||
public static void EnsureApp0Indexed(string app0Root)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(app0Root) || !Directory.Exists(app0Root))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var normalizedRoot = Path.GetFullPath(app0Root);
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
|
||||
lock (_indexGate)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if (string.Equals(_indexedApp0Root, normalizedRoot, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (_indexingApp0Root is not null)
|
||||
{
|
||||
// Another thread (preload) owns the walk — wait instead of
|
||||
// stacking a second 8s index on the guest APR miss path.
|
||||
if (string.Equals(
|
||||
_indexingApp0Root,
|
||||
normalizedRoot,
|
||||
StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Monitor.Wait(_indexGate);
|
||||
continue;
|
||||
}
|
||||
|
||||
Monitor.Wait(_indexGate, 50);
|
||||
continue;
|
||||
}
|
||||
|
||||
_indexingApp0Root = normalizedRoot;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var cachePathV3 = GetIndexCachePath(normalizedRoot, version: 3);
|
||||
var cachePathV2 = GetIndexCachePath(normalizedRoot, version: 2);
|
||||
if (TryLoadIndexCache(normalizedRoot, cachePathV3, preferV3: true, out var cachedFiles))
|
||||
{
|
||||
lock (_indexGate)
|
||||
{
|
||||
_indexedApp0Root = normalizedRoot;
|
||||
}
|
||||
|
||||
Console.Error.WriteLine(
|
||||
$"[LOADER][INFO] ampr.app0_index_cache_hit root={normalizedRoot} " +
|
||||
$"files={cachedFiles} ids={_hostPathsById.Count} " +
|
||||
$"elapsed_ms={stopwatch.Elapsed.TotalMilliseconds:F1}");
|
||||
return;
|
||||
}
|
||||
|
||||
if (TryLoadIndexCache(normalizedRoot, cachePathV2, preferV3: false, out cachedFiles))
|
||||
{
|
||||
lock (_indexGate)
|
||||
{
|
||||
_indexedApp0Root = normalizedRoot;
|
||||
}
|
||||
|
||||
// Promote v2 (rehash-on-load) to v3 (precomputed ids) so the
|
||||
// next boot skips the Rosetta FNV storm.
|
||||
TrySaveIndexCache(normalizedRoot, cachePathV3, cachedFiles);
|
||||
Console.Error.WriteLine(
|
||||
$"[LOADER][INFO] ampr.app0_index_cache_hit root={normalizedRoot} " +
|
||||
$"files={cachedFiles} ids={_hostPathsById.Count} " +
|
||||
$"elapsed_ms={stopwatch.Elapsed.TotalMilliseconds:F1} upgraded=v3");
|
||||
return;
|
||||
}
|
||||
|
||||
var relatives = new List<string>(256 * 1024);
|
||||
foreach (var hostPath in Directory.EnumerateFiles(
|
||||
normalizedRoot,
|
||||
"*",
|
||||
SearchOption.AllDirectories))
|
||||
{
|
||||
var relative = Path.GetRelativePath(normalizedRoot, hostPath)
|
||||
.Replace('\\', '/');
|
||||
if (string.IsNullOrEmpty(relative) ||
|
||||
relative.StartsWith("..", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
relatives.Add(relative);
|
||||
}
|
||||
|
||||
// Hash + dictionary fill dominates under Rosetta once the walk is
|
||||
// done; parallelize across cores without re-walking the tree.
|
||||
Parallel.ForEach(
|
||||
relatives,
|
||||
new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = Math.Max(2, Environment.ProcessorCount - 1),
|
||||
},
|
||||
relative =>
|
||||
{
|
||||
var hostPath = Path.Combine(normalizedRoot, relative.Replace('/', Path.DirectorySeparatorChar));
|
||||
RegisterApp0Relative(relative, hostPath);
|
||||
});
|
||||
|
||||
lock (_indexGate)
|
||||
{
|
||||
_indexedApp0Root = normalizedRoot;
|
||||
}
|
||||
|
||||
TrySaveIndexCache(normalizedRoot, cachePathV3, relatives.Count);
|
||||
Console.Error.WriteLine(
|
||||
$"[LOADER][INFO] ampr.app0_indexed root={normalizedRoot} " +
|
||||
$"files={relatives.Count} ids={_hostPathsById.Count} " +
|
||||
$"elapsed_ms={stopwatch.Elapsed.TotalMilliseconds:F1}");
|
||||
}
|
||||
finally
|
||||
{
|
||||
lock (_indexGate)
|
||||
{
|
||||
_indexingApp0Root = null;
|
||||
Monitor.PulseAll(_indexGate);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers the four Insomniac path aliases for one app0-relative file
|
||||
/// without allocating intermediate guest-path strings.
|
||||
/// </summary>
|
||||
private static void RegisterApp0Relative(string relative, string hostPath)
|
||||
{
|
||||
// "$/" + relative
|
||||
Publish(FnvContinueAscii(FnvContinueAscii(OffsetBasis, (byte)'$'), (byte)'/'), relative, hostPath);
|
||||
// "/app0/" + relative
|
||||
Publish(FnvContinueAsciiPrefix(OffsetBasis, "/app0/"u8), relative, hostPath);
|
||||
// "app0/" + relative
|
||||
Publish(FnvContinueAsciiPrefix(OffsetBasis, "app0/"u8), relative, hostPath);
|
||||
// bare relative
|
||||
Publish(OffsetBasis, relative, hostPath);
|
||||
}
|
||||
|
||||
private static void Publish(uint hash, string relative, string hostPath)
|
||||
{
|
||||
_hostPathsById[FnvContinueUtf8(hash, relative)] = hostPath;
|
||||
}
|
||||
|
||||
internal static uint ComputeFileId(string guestPath)
|
||||
{
|
||||
var bytes = System.Text.Encoding.UTF8.GetBytes(guestPath);
|
||||
return FnvContinueUtf8(OffsetBasis, guestPath);
|
||||
}
|
||||
|
||||
const uint offsetBasis = 2166136261;
|
||||
const uint prime = 16777619;
|
||||
|
||||
var hash = offsetBasis;
|
||||
foreach (var b in bytes)
|
||||
internal static IEnumerable<string> EnumerateApp0PathAliases(string guestPath)
|
||||
{
|
||||
if (string.IsNullOrEmpty(guestPath))
|
||||
{
|
||||
hash ^= b;
|
||||
hash *= prime;
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (!TryGetApp0Relative(guestPath, out var relative) ||
|
||||
string.IsNullOrEmpty(relative))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
yield return "$/" + relative;
|
||||
yield return "/app0/" + relative;
|
||||
yield return "app0/" + relative;
|
||||
yield return relative;
|
||||
}
|
||||
|
||||
private static bool TryGetApp0Relative(string guestPath, out string relative)
|
||||
{
|
||||
relative = string.Empty;
|
||||
var normalized = guestPath.Replace('\\', '/');
|
||||
|
||||
if (normalized.StartsWith("$/", StringComparison.Ordinal))
|
||||
{
|
||||
relative = normalized[2..].TrimStart('/');
|
||||
return relative.Length != 0;
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("/app0/", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
relative = normalized["/app0/".Length..].TrimStart('/');
|
||||
return relative.Length != 0;
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("app0/", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
relative = normalized["app0/".Length..].TrimStart('/');
|
||||
return relative.Length != 0;
|
||||
}
|
||||
|
||||
// Bare relative paths are treated as app0-relative by ResolveGuestPath.
|
||||
if (!normalized.StartsWith('/') &&
|
||||
!Path.IsPathFullyQualified(guestPath))
|
||||
{
|
||||
relative = normalized.TrimStart('/');
|
||||
return relative.Length != 0;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string GetIndexCachePath(string normalizedRoot, int version)
|
||||
{
|
||||
var overrideDir = Environment.GetEnvironmentVariable("SHARPEMU_AMPR_INDEX_CACHE");
|
||||
var cacheDir = !string.IsNullOrWhiteSpace(overrideDir)
|
||||
? overrideDir
|
||||
: Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
|
||||
"SharpEmu",
|
||||
"ampr-index");
|
||||
Directory.CreateDirectory(cacheDir);
|
||||
|
||||
var rootHash = ComputeFileId(normalizedRoot.ToLowerInvariant());
|
||||
return Path.Combine(cacheDir, $"app0-{rootHash:x8}.v{version}.idx");
|
||||
}
|
||||
|
||||
private static bool TryLoadIndexCache(
|
||||
string normalizedRoot,
|
||||
string cachePath,
|
||||
bool preferV3,
|
||||
out int fileCount)
|
||||
{
|
||||
fileCount = 0;
|
||||
try
|
||||
{
|
||||
if (!File.Exists(cachePath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (string.Equals(
|
||||
Environment.GetEnvironmentVariable("SHARPEMU_AMPR_REINDEX"),
|
||||
"1",
|
||||
StringComparison.Ordinal))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
using var stream = File.OpenRead(cachePath);
|
||||
using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: false);
|
||||
var magic = reader.ReadUInt32();
|
||||
var version = reader.ReadUInt32();
|
||||
var isV3 = magic == CacheMagicV3 && version == CacheVersionV3;
|
||||
var isV2 = magic == CacheMagicV2 && version == CacheVersionV2;
|
||||
if (preferV3)
|
||||
{
|
||||
if (!isV3)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else if (!isV2)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var root = reader.ReadString();
|
||||
if (!string.Equals(root, normalizedRoot, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var expectedFiles = reader.ReadInt32();
|
||||
var expectedParamTicks = reader.ReadInt64();
|
||||
var actualParamTicks = GetParamJsonWriteTicks(normalizedRoot);
|
||||
if (actualParamTicks == 0 || actualParamTicks != expectedParamTicks)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (expectedFiles < 0 || expectedFiles > 8_000_000)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isV3)
|
||||
{
|
||||
// Precomputed FNV ids — no Rosetta hash storm on every boot.
|
||||
var entries = new (string Relative, uint Id0, uint Id1, uint Id2, uint Id3)[expectedFiles];
|
||||
for (var i = 0; i < expectedFiles; i++)
|
||||
{
|
||||
entries[i] = (
|
||||
reader.ReadString(),
|
||||
reader.ReadUInt32(),
|
||||
reader.ReadUInt32(),
|
||||
reader.ReadUInt32(),
|
||||
reader.ReadUInt32());
|
||||
}
|
||||
|
||||
Parallel.ForEach(
|
||||
entries,
|
||||
new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = Math.Max(2, Environment.ProcessorCount - 1),
|
||||
},
|
||||
entry =>
|
||||
{
|
||||
if (string.IsNullOrEmpty(entry.Relative) ||
|
||||
entry.Relative.Contains("..", StringComparison.Ordinal))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var hostPath = normalizedRoot.EndsWith(Path.DirectorySeparatorChar)
|
||||
? normalizedRoot + entry.Relative.Replace('/', Path.DirectorySeparatorChar)
|
||||
: normalizedRoot + Path.DirectorySeparatorChar +
|
||||
entry.Relative.Replace('/', Path.DirectorySeparatorChar);
|
||||
_hostPathsById[entry.Id0] = hostPath;
|
||||
_hostPathsById[entry.Id1] = hostPath;
|
||||
_hostPathsById[entry.Id2] = hostPath;
|
||||
_hostPathsById[entry.Id3] = hostPath;
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
var relatives = new string[expectedFiles];
|
||||
for (var i = 0; i < expectedFiles; i++)
|
||||
{
|
||||
relatives[i] = reader.ReadString();
|
||||
}
|
||||
|
||||
Parallel.ForEach(
|
||||
relatives,
|
||||
new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = Math.Max(2, Environment.ProcessorCount - 1),
|
||||
},
|
||||
relative =>
|
||||
{
|
||||
if (string.IsNullOrEmpty(relative) ||
|
||||
relative.Contains("..", StringComparison.Ordinal))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var hostPath = Path.Combine(
|
||||
normalizedRoot,
|
||||
relative.Replace('/', Path.DirectorySeparatorChar));
|
||||
RegisterApp0Relative(relative, hostPath);
|
||||
});
|
||||
}
|
||||
|
||||
fileCount = expectedFiles;
|
||||
return true;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_hostPathsById.Clear();
|
||||
Console.Error.WriteLine(
|
||||
$"[LOADER][WARN] ampr.app0_index_cache_load_failed: {exception.Message}");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static void TrySaveIndexCache(
|
||||
string normalizedRoot,
|
||||
string cachePath,
|
||||
int fileCount)
|
||||
{
|
||||
try
|
||||
{
|
||||
var paramTicks = GetParamJsonWriteTicks(normalizedRoot);
|
||||
if (paramTicks == 0 || fileCount <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var relatives = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var hostPath in _hostPathsById.Values)
|
||||
{
|
||||
var relative = Path.GetRelativePath(normalizedRoot, hostPath)
|
||||
.Replace('\\', '/');
|
||||
if (string.IsNullOrEmpty(relative) ||
|
||||
relative.StartsWith("..", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
relatives.Add(relative);
|
||||
}
|
||||
|
||||
var tempPath = cachePath + ".tmp";
|
||||
using (var stream = File.Create(tempPath))
|
||||
using (var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: false))
|
||||
{
|
||||
writer.Write(CacheMagicV3);
|
||||
writer.Write(CacheVersionV3);
|
||||
writer.Write(normalizedRoot);
|
||||
writer.Write(relatives.Count);
|
||||
writer.Write(paramTicks);
|
||||
foreach (var relative in relatives)
|
||||
{
|
||||
writer.Write(relative);
|
||||
// Mirror RegisterApp0Relative id order: $/ /app0/ app0/ bare.
|
||||
writer.Write(ComputeApp0AliasIds(relative, out var id1, out var id2, out var id3));
|
||||
writer.Write(id1);
|
||||
writer.Write(id2);
|
||||
writer.Write(id3);
|
||||
}
|
||||
}
|
||||
|
||||
File.Move(tempPath, cachePath, overwrite: true);
|
||||
Console.Error.WriteLine(
|
||||
$"[LOADER][INFO] ampr.app0_index_cache_saved path={cachePath} " +
|
||||
$"files={relatives.Count}");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Console.Error.WriteLine(
|
||||
$"[LOADER][WARN] ampr.app0_index_cache_save_failed: {exception.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static uint ComputeApp0AliasIds(
|
||||
string relative,
|
||||
out uint app0Slash,
|
||||
out uint app0,
|
||||
out uint bare)
|
||||
{
|
||||
var dollar = FnvContinueUtf8(
|
||||
FnvContinueAscii(FnvContinueAscii(OffsetBasis, (byte)'$'), (byte)'/'),
|
||||
relative);
|
||||
app0Slash = FnvContinueUtf8(FnvContinueAsciiPrefix(OffsetBasis, "/app0/"u8), relative);
|
||||
app0 = FnvContinueUtf8(FnvContinueAsciiPrefix(OffsetBasis, "app0/"u8), relative);
|
||||
bare = FnvContinueUtf8(OffsetBasis, relative);
|
||||
return dollar;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cheap dump fingerprint. Full-tree walks are too expensive for cache
|
||||
/// validation; param.json changes with title updates. Force a rebuild with
|
||||
/// SHARPEMU_AMPR_REINDEX=1 after manual dump edits.
|
||||
/// </summary>
|
||||
private static long GetParamJsonWriteTicks(string normalizedRoot)
|
||||
{
|
||||
try
|
||||
{
|
||||
var paramPath = Path.Combine(normalizedRoot, "sce_sys", "param.json");
|
||||
return File.Exists(paramPath)
|
||||
? File.GetLastWriteTimeUtc(paramPath).Ticks
|
||||
: 0;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private const uint OffsetBasis = 2166136261;
|
||||
private const uint FnvPrime = 16777619;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static uint FnvContinueAscii(uint hash, byte value)
|
||||
{
|
||||
hash ^= value;
|
||||
return hash * FnvPrime;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static uint FnvContinueAsciiPrefix(uint hash, ReadOnlySpan<byte> ascii)
|
||||
{
|
||||
foreach (var value in ascii)
|
||||
{
|
||||
hash ^= value;
|
||||
hash *= FnvPrime;
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
private static uint FnvContinueUtf8(uint hash, string text)
|
||||
{
|
||||
// Game asset paths are overwhelmingly ASCII; avoid Encoding.GetBytes
|
||||
// allocations on the 223k-file DeS index hot path.
|
||||
Span<byte> utf8Scratch = stackalloc byte[4];
|
||||
for (var i = 0; i < text.Length; i++)
|
||||
{
|
||||
var ch = text[i];
|
||||
if (ch < 0x80)
|
||||
{
|
||||
hash ^= (byte)ch;
|
||||
hash *= FnvPrime;
|
||||
continue;
|
||||
}
|
||||
|
||||
var written = Encoding.UTF8.GetBytes(text.AsSpan(i, 1), utf8Scratch);
|
||||
for (var b = 0; b < written; b++)
|
||||
{
|
||||
hash ^= utf8Scratch[b];
|
||||
hash *= FnvPrime;
|
||||
}
|
||||
}
|
||||
|
||||
return hash;
|
||||
|
||||
@@ -24,6 +24,7 @@ SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="SharpEmu.Libs.Tests" />
|
||||
<InternalsVisibleTo Include="SharpEmu.Core" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
// Copyright (C) 2026 SharpEmu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
using SharpEmu.Libs.Ampr;
|
||||
using Xunit;
|
||||
|
||||
namespace SharpEmu.Libs.Tests.Ampr;
|
||||
|
||||
public class AmprFileRegistryTests
|
||||
{
|
||||
[Fact]
|
||||
public void ComputeFileId_matches_utf8_fnv1a()
|
||||
{
|
||||
const string relative = "CoreData/foo/bar.bin";
|
||||
Assert.Equal(FnvUtf8("$/" + relative), AmprFileRegistry.ComputeFileId("$/" + relative));
|
||||
Assert.Equal(FnvUtf8("/app0/" + relative), AmprFileRegistry.ComputeFileId("/app0/" + relative));
|
||||
Assert.Equal(FnvUtf8("app0/" + relative), AmprFileRegistry.ComputeFileId("app0/" + relative));
|
||||
Assert.Equal(FnvUtf8(relative), AmprFileRegistry.ComputeFileId(relative));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RegisterApp0Relative_publishes_same_ids_as_string_hashes()
|
||||
{
|
||||
AmprFileRegistry.ClearForTests();
|
||||
const string relative = "misc/loadouts/test.txt";
|
||||
var host = Path.Combine(Path.GetTempPath(), "sharpemu-ampr-test", relative);
|
||||
AmprFileRegistry.RegisterApp0RelativeForTests(relative, host);
|
||||
|
||||
Assert.True(AmprFileRegistry.TryGetHostPath(
|
||||
AmprFileRegistry.ComputeFileId("$/" + relative), out var a));
|
||||
Assert.True(AmprFileRegistry.TryGetHostPath(
|
||||
AmprFileRegistry.ComputeFileId("/app0/" + relative), out var b));
|
||||
Assert.True(AmprFileRegistry.TryGetHostPath(
|
||||
AmprFileRegistry.ComputeFileId("app0/" + relative), out var c));
|
||||
Assert.True(AmprFileRegistry.TryGetHostPath(
|
||||
AmprFileRegistry.ComputeFileId(relative), out var d));
|
||||
Assert.Equal(host, a);
|
||||
Assert.Equal(host, b);
|
||||
Assert.Equal(host, c);
|
||||
Assert.Equal(host, d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Register_publishes_all_app0_path_aliases()
|
||||
{
|
||||
AmprFileRegistry.ClearForTests();
|
||||
const string relative = "scripts/cp11/cp11main.script";
|
||||
var host = Path.Combine(Path.GetTempPath(), "sharpemu-ampr-test2", relative);
|
||||
AmprFileRegistry.Register("$/" + relative, host);
|
||||
|
||||
Assert.True(AmprFileRegistry.TryGetHostPath(
|
||||
AmprFileRegistry.ComputeFileId("$/" + relative), out var a));
|
||||
Assert.True(AmprFileRegistry.TryGetHostPath(
|
||||
AmprFileRegistry.ComputeFileId("/app0/" + relative), out var b));
|
||||
Assert.True(AmprFileRegistry.TryGetHostPath(
|
||||
AmprFileRegistry.ComputeFileId("app0/" + relative), out var c));
|
||||
Assert.True(AmprFileRegistry.TryGetHostPath(
|
||||
AmprFileRegistry.ComputeFileId(relative), out var d));
|
||||
Assert.Equal(host, a);
|
||||
Assert.Equal(host, b);
|
||||
Assert.Equal(host, c);
|
||||
Assert.Equal(host, d);
|
||||
}
|
||||
|
||||
private static uint FnvUtf8(string text)
|
||||
{
|
||||
const uint offset = 2166136261;
|
||||
const uint prime = 16777619;
|
||||
var hash = offset;
|
||||
foreach (var b in System.Text.Encoding.UTF8.GetBytes(text))
|
||||
{
|
||||
hash ^= b;
|
||||
hash *= prime;
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
}
|
||||
@@ -318,6 +318,53 @@ public sealed class AprStreamingContractTests
|
||||
return BinaryPrimitives.ReadUInt32LittleEndian(bytes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Register_AlsoPublishesApp0AndDollarPathAliases()
|
||||
{
|
||||
// Resolve may register "$/asset.bin" while cooked tables look up
|
||||
// FNV("/app0/asset.bin"). Both ids must map to the same host file.
|
||||
var hostPath = Path.Combine(Path.GetTempPath(), $"sharpemu-apr-alias-{Guid.NewGuid():N}.bin");
|
||||
File.WriteAllBytes(hostPath, [1, 2, 3]);
|
||||
try
|
||||
{
|
||||
var dollarId = AmprFileRegistry.Register("$/weapons/demo.cani", hostPath);
|
||||
Assert.True(AmprFileRegistry.TryGetHostPath(dollarId, out var viaDollar));
|
||||
Assert.Equal(hostPath, viaDollar);
|
||||
|
||||
var app0Id = AmprFileRegistry.ComputeFileId("/app0/weapons/demo.cani");
|
||||
Assert.NotEqual(dollarId, app0Id);
|
||||
Assert.True(AmprFileRegistry.TryGetHostPath(app0Id, out var viaApp0));
|
||||
Assert.Equal(hostPath, viaApp0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
File.Delete(hostPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureApp0Indexed_PublishesCookedApp0FileIds()
|
||||
{
|
||||
var mountRoot = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
$"sharpemu-apr-index-{Guid.NewGuid():N}");
|
||||
var relativeDir = Path.Combine(mountRoot, "weapons");
|
||||
Directory.CreateDirectory(relativeDir);
|
||||
var hostPath = Path.Combine(relativeDir, "demo.cani");
|
||||
File.WriteAllBytes(hostPath, [9, 8, 7]);
|
||||
try
|
||||
{
|
||||
AmprFileRegistry.EnsureApp0Indexed(mountRoot);
|
||||
var cookedId = AmprFileRegistry.ComputeFileId("/app0/weapons/demo.cani");
|
||||
Assert.True(AmprFileRegistry.TryGetHostPath(cookedId, out var resolved));
|
||||
Assert.Equal(Path.GetFullPath(hostPath), Path.GetFullPath(resolved));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(mountRoot, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private static ulong ReadUInt64(FakeCpuMemory memory, ulong address)
|
||||
{
|
||||
Span<byte> bytes = stackalloc byte[sizeof(ulong)];
|
||||
|
||||
Reference in New Issue
Block a user