diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d1ae32d3d..e7c7ac7e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -119,8 +119,30 @@ jobs: mkw_macos_context_abi_tests \ mkw_macos_host_context_tests \ mkw_macos_guest_flat_memory_tests \ - mkw_macos_external_audio_tests + mkw_macos_external_audio_tests \ + mkw_ax_mix_kernels_tests - name: Test execution substrate shell: bash run: ctest --test-dir build-macos --output-on-failure + + - name: Check ARM audio kernels with sanitizers + shell: bash + run: | + clang++ -std=c++17 -O2 -fno-fast-math \ + -fsanitize=address,undefined -fno-sanitize-recover=all \ + -DMKW_AX_MIX_REQUIRE_NEON=1 \ + -Iruntime/include -Iruntime/src/hle/audio \ + runtime/tests/ax_mix_kernels_tests.cpp -o build-macos/ax-mix-sanitized + build-macos/ax-mix-sanitized + build-macos/mkw_ax_mix_kernels_tests --benchmark 20000 + + - name: Build and test GX performance paths + shell: bash + run: | + cmake -S runtime -B build-macos -DMKW_BUILD_AURORA_TESTS=ON + cmake --build build-macos --parallel 3 --target \ + gx_fifo_tests shader_module_cache_tests gx_shader_layout_tests \ + mkw_gx_texture_binding_tests mkw_gx_sync_tests mkw_gx_dl_cache_tests + ctest --test-dir build-macos --output-on-failure \ + -R '^(GXFifoTest|RuntimeDirectDrawTest|ShaderModuleCacheTest|GxShader(Layout|Key|Arena|Mode|State)|mkw_gx_(texture_binding|sync|dl_cache)_tests)' diff --git a/aurora-main b/aurora-main index a222e54a4..cf312581c 160000 --- a/aurora-main +++ b/aurora-main @@ -1 +1 @@ -Subproject commit a222e54a4e53d55415de1de1e4fe9dece2705288 +Subproject commit cf312581c4c7cdcdf03f7d6f5e24875d9898c082 diff --git a/runtime/cmake/PlatformTests.cmake b/runtime/cmake/PlatformTests.cmake index 6f0dc65eb..5e634d23f 100644 --- a/runtime/cmake/PlatformTests.cmake +++ b/runtime/cmake/PlatformTests.cmake @@ -155,6 +155,7 @@ endif() if(TARGET gx_fifo_tests) target_sources(gx_fifo_tests PRIVATE "${MKW_TEST_RUNTIME_DIR}/tests/gx_fifo_invalidation_tests.cpp" + "${MKW_TEST_RUNTIME_DIR}/tests/gx_direct_draw_tests.cpp" "${MKW_TEST_RUNTIME_DIR}/src/hle/gx/gx_fifo.cpp") target_include_directories(gx_fifo_tests PRIVATE "${MKW_AURORA_DIR}/tests" "${MKW_TEST_RUNTIME_DIR}/include" @@ -164,7 +165,66 @@ if(TARGET gx_fifo_tests) if(MKW_PLATFORM_WINDOWS) target_link_libraries(gx_fifo_tests PRIVATE advapi32) endif() + + get_target_property(_mkw_gx_sources gx_fifo_tests SOURCES) + get_target_property(_mkw_gx_source_dir gx_fifo_tests SOURCE_DIR) + get_target_property(_mkw_gx_includes gx_fifo_tests INCLUDE_DIRECTORIES) + get_target_property(_mkw_gx_links gx_fifo_tests LINK_LIBRARIES) + set(_mkw_gx_encoder_sources) + foreach(_mkw_gx_source IN LISTS _mkw_gx_sources) + if(_mkw_gx_source MATCHES "(^|/)lib/") + get_filename_component(_mkw_gx_absolute "${_mkw_gx_source}" ABSOLUTE + BASE_DIR "${_mkw_gx_source_dir}") + list(APPEND _mkw_gx_encoder_sources "${_mkw_gx_absolute}") + endif() + endforeach() + + add_executable(mkw_gx_texture_binding_tests + "${MKW_TEST_RUNTIME_DIR}/tests/gx_texture_binding_tests.cpp" + "${MKW_AURORA_DIR}/tests/gx_test_stubs.cpp" + ${_mkw_gx_encoder_sources}) + add_executable(mkw_gx_sync_tests + "${MKW_TEST_RUNTIME_DIR}/tests/gx_worker_tests.cpp" + "${MKW_TEST_RUNTIME_DIR}/tests/gx_copy_tests.cpp" + "${MKW_AURORA_DIR}/tests/gx_test_stubs.cpp" + ${_mkw_gx_encoder_sources}) + target_compile_definitions(mkw_gx_sync_tests PRIVATE MKW_GX_COPY_TESTING=1) + foreach(_mkw_gx_test IN ITEMS mkw_gx_texture_binding_tests mkw_gx_sync_tests) + target_include_directories(${_mkw_gx_test} PRIVATE + ${_mkw_gx_includes} "${MKW_TEST_RUNTIME_DIR}/tests") + target_compile_definitions(${_mkw_gx_test} PRIVATE AURORA TARGET_PC=1) + target_compile_features(${_mkw_gx_test} PRIVATE cxx_std_20) + target_link_libraries(${_mkw_gx_test} PRIVATE ${_mkw_gx_links} Threads::Threads) + set_target_properties(${_mkw_gx_test} PROPERTIES UNITY_BUILD OFF) + aurora_copy_runtime_dlls(${_mkw_gx_test}) + add_test(NAME ${_mkw_gx_test} COMMAND ${_mkw_gx_test}) + set_tests_properties(${_mkw_gx_test} PROPERTIES TIMEOUT 120) + endforeach() + + add_executable(mkw_gx_dl_cache_tests "${MKW_TEST_RUNTIME_DIR}/tests/gx_dl_cache_tests.cpp") + target_include_directories(mkw_gx_dl_cache_tests PRIVATE + "${MKW_TEST_RUNTIME_DIR}/include" + "${MKW_TEST_RUNTIME_DIR}/third_party/toml11" + "${MKW_AURORA_DIR}/include") + target_compile_features(mkw_gx_dl_cache_tests PRIVATE cxx_std_20) + target_compile_definitions(mkw_gx_dl_cache_tests PRIVATE AURORA TARGET_PC=1 NOMINMAX) + target_link_libraries(mkw_gx_dl_cache_tests PRIVATE xxHash::xxhash ${AURORA_SDL3_TARGET}) + set_target_properties(mkw_gx_dl_cache_tests PROPERTIES UNITY_BUILD OFF) + add_test(NAME mkw_gx_dl_cache_tests COMMAND mkw_gx_dl_cache_tests) +endif() + +add_executable(mkw_ax_mix_kernels_tests "${MKW_TEST_RUNTIME_DIR}/tests/ax_mix_kernels_tests.cpp") +target_include_directories(mkw_ax_mix_kernels_tests PRIVATE + "${MKW_TEST_RUNTIME_DIR}/include" + "${MKW_TEST_RUNTIME_DIR}/src/hle/audio") +target_compile_features(mkw_ax_mix_kernels_tests PRIVATE cxx_std_17) +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") + target_compile_definitions(mkw_ax_mix_kernels_tests PRIVATE MKW_AX_MIX_REQUIRE_NEON=1) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|X86_64)$") + target_compile_options(mkw_ax_mix_kernels_tests PRIVATE -march=x86-64-${MKW_X86_CPU_PROFILE}) endif() +set_target_properties(mkw_ax_mix_kernels_tests PROPERTIES UNITY_BUILD OFF) +add_test(NAME mkw_ax_mix_kernels_tests COMMAND mkw_ax_mix_kernels_tests) add_executable(mkw_fiber_manager_tests "${MKW_TEST_RUNTIME_DIR}/tests/fiber_manager_tests.cpp" diff --git a/runtime/include/runtime_config.h b/runtime/include/runtime_config.h index d88ac372e..101c50f59 100644 --- a/runtime/include/runtime_config.h +++ b/runtime/include/runtime_config.h @@ -47,6 +47,7 @@ struct RuntimeUserConfig { std::optional displayMode; std::optional frameInterpolationFps; std::optional metalFxSpatialUpscaling; + std::optional shaderMode; std::optional skipUnreadyPipelines; std::optional vsync; std::optional gxWorker; @@ -294,6 +295,7 @@ inline void EnsureConfigFile() { "frame_interpolation_fps = 0\n" "display_mode = \"windowed\"\n" "graphics_api = \"auto\"\n" + "shader_mode = 'specialized'\n" "skip_unready_pipelines = true\n" "vsync = false\n" "# Runs GX command processing on its own thread. Read once, at startup.\n" @@ -485,6 +487,10 @@ inline RuntimeUserConfig ParseConfigDocument(const toml::value& document) { } config.metalFxSpatialUpscaling = FindConfigValue(document, "video", "metalfx_spatial_upscaling"); + if (auto value = FindConfigValue(document, "video", "shader_mode"); + value && (*value == "specialized" || *value == "hybrid")) { + config.shaderMode = *value; + } config.skipUnreadyPipelines = FindConfigValue(document, "video", "skip_unready_pipelines"); config.vsync = FindConfigValue(document, "video", "vsync"); config.gxWorker = FindConfigValue(document, "video", "gx_worker"); @@ -702,6 +708,12 @@ inline bool SetDisplayMode(std::string value) { return WriteSetting("video", "display_mode", FormatString(value)); } +inline bool SetShaderMode(std::string value) { + if (value != "specialized" && value != "hybrid") return false; + Mutable().shaderMode = value; + return WriteSetting("video", "shader_mode", FormatString(value)); +} + inline bool SetSkipUnreadyPipelines(bool value) { Mutable().skipUnreadyPipelines = value; return WriteSetting("video", "skip_unready_pipelines", value ? "true" : "false"); @@ -1036,6 +1048,8 @@ inline uint32_t FrameInterpolationFps(uint32_t fallback = 0) { return Get().frameInterpolationFps.value_or(fallback); } +inline std::string ShaderMode() { return Get().shaderMode.value_or("specialized"); } + // Whether to skip draws whose graphics pipeline has not finished compiling yet. inline bool SkipUnreadyPipelines(bool fallback = true) { return Get().skipUnreadyPipelines.value_or(fallback); diff --git a/runtime/src/hle/audio/ax_mix_kernels.h b/runtime/src/hle/audio/ax_mix_kernels.h index 687d6bab4..d51a68972 100644 --- a/runtime/src/hle/audio/ax_mix_kernels.h +++ b/runtime/src/hle/audio/ax_mix_kernels.h @@ -3,7 +3,7 @@ #include "isa/big_endian.h" // AX/DSP fixed-point mix kernels: the inner loops of the AXWii command-list mix in ax_mix.cpp. -// Each exists as a scalar reference (a literal transcription of the loop it replaced) and an AVX2 form. +// Each has a scalar reference and architecture-specific SIMD forms. #include #include @@ -16,8 +16,23 @@ #define MKW_AX_MIX_AVX2 0 #endif +#if !MKW_AX_MIX_AVX2 && defined(__aarch64__) && defined(__ARM_NEON) && !defined(__AARCH64EB__) +#include +#define MKW_AX_MIX_NEON 1 +#else +#define MKW_AX_MIX_NEON 0 +#endif + namespace AxMixKernels { +// Mix buses retain the low 32 bits, matching the SIMD adds. +inline int32_t AddWrap32(int32_t value, uint32_t addend) { + const uint32_t bits = static_cast(value) + addend; + int32_t result; + std::memcpy(&result, &bits, sizeof(result)); + return result; +} + inline int16_t ClampToS16(int32_t value) { if (value < -0x8000) { return static_cast(-0x8000); @@ -39,7 +54,7 @@ inline uint16_t MixAddRampScalar(int32_t* out, const int16_t* input, uint32_t co const int32_t scaled = (static_cast(input[i]) * static_cast(volume)) >> 15; const int16_t sample = ClampToS16(scaled); - out[i] += sample; + out[i] = AddWrap32(out[i], static_cast(sample)); volume = static_cast(volume + delta); last = sample; } @@ -90,7 +105,7 @@ inline uint16_t MixAddRampAvx2(int32_t* out, const int16_t* input, uint32_t coun const int32_t scaled = (static_cast(input[i]) * static_cast(volume)) >> 15; const int16_t sample = ClampToS16(scaled); - out[i] += sample; + out[i] = AddWrap32(out[i], static_cast(sample)); volume = static_cast(volume + delta); last = sample; } @@ -99,10 +114,46 @@ inline uint16_t MixAddRampAvx2(int32_t* out, const int16_t* input, uint32_t coun } #endif +#if MKW_AX_MIX_NEON +inline uint16x4_t Ramp4(uint16_t volume, uint16_t delta) { + static constexpr uint16_t lanes[] = {0, 1, 2, 3}; + return vmla_n_u16(vdup_n_u16(volume), vld1_u16(lanes), delta); +} + +inline int16x4_t ScaleBlock4(int16x4_t samples, uint16x4_t ramp) { + const int32x4_t gain = vreinterpretq_s32_u32(vmovl_u16(ramp)); + const int32x4_t product = vmulq_s32(vmovl_s16(samples), gain); + return vqmovn_s32(vshrq_n_s32(product, 15)); +} + +inline uint16_t MixAddRampNeon(int32_t* out, const int16_t* input, uint32_t count, + uint16_t volume, uint16_t delta, int16_t& dpop) { + uint32_t i = 0; + int16_t last = dpop; + if (count >= 4) { + uint16x4_t ramp = Ramp4(volume, delta); + const uint16x4_t step = vdup_n_u16(static_cast(delta * 4u)); + int16x4_t block = vdup_n_s16(0); + for (; count - i >= 4; i += 4) { + block = ScaleBlock4(vld1_s16(input + i), ramp); + vst1q_s32(out + i, vaddw_s16(vld1q_s32(out + i), block)); + ramp = vadd_u16(ramp, step); + } + last = vget_lane_s16(block, 3); + volume = vget_lane_u16(ramp, 0); + } + volume = MixAddRampScalar(out + i, input + i, count - i, volume, delta, last); + dpop = last; + return volume; +} +#endif + inline uint16_t MixAddRamp(int32_t* out, const int16_t* input, uint32_t count, uint16_t volume, uint16_t delta, int16_t& dpop) { #if MKW_AX_MIX_AVX2 return MixAddRampAvx2(out, input, count, volume, delta, dpop); +#elif MKW_AX_MIX_NEON + return MixAddRampNeon(out, input, count, volume, delta, dpop); #else return MixAddRampScalar(out, input, count, volume, delta, dpop); #endif @@ -162,10 +213,29 @@ inline uint16_t ScaleRampAvx2(int16_t* samples, uint32_t count, uint16_t volume, } #endif +#if MKW_AX_MIX_NEON +inline uint16_t ScaleRampNeon(int16_t* samples, uint32_t count, uint16_t volume, + uint16_t delta) { + uint32_t i = 0; + if (count >= 4) { + uint16x4_t ramp = Ramp4(volume, delta); + const uint16x4_t step = vdup_n_u16(static_cast(delta * 4u)); + for (; count - i >= 4; i += 4) { + vst1_s16(samples + i, ScaleBlock4(vld1_s16(samples + i), ramp)); + ramp = vadd_u16(ramp, step); + } + volume = vget_lane_u16(ramp, 0); + } + return ScaleRampScalar(samples + i, count - i, volume, delta); +} +#endif + inline uint16_t ScaleRamp(int16_t* samples, uint32_t count, uint16_t volume, uint16_t delta) { #if MKW_AX_MIX_AVX2 return ScaleRampAvx2(samples, count, volume, delta); +#elif MKW_AX_MIX_NEON + return ScaleRampNeon(samples, count, volume, delta); #else return ScaleRampScalar(samples, count, volume, delta); #endif @@ -179,8 +249,9 @@ inline uint16_t ScaleRamp(int16_t* samples, uint32_t count, uint16_t volume, inline void MixAccumRamp32Scalar(int32_t* dst, const int32_t* src, const uint16_t* ramp, uint32_t count) { for (uint32_t i = 0; i < count; ++i) { - dst[i] += static_cast( - (static_cast(src[i]) * static_cast(ramp[i])) >> 15); + const int64_t product = static_cast(src[i]) * ramp[i]; + const uint32_t scaled = static_cast(static_cast(product) >> 15); + dst[i] = AddWrap32(dst[i], scaled); } } @@ -208,16 +279,37 @@ inline void MixAccumRamp32Avx2(int32_t* dst, const int32_t* src, const uint16_t* _mm256_add_epi32(acc, result)); } for (; i < count; ++i) { - dst[i] += static_cast( - (static_cast(src[i]) * static_cast(ramp[i])) >> 15); + const int64_t product = static_cast(src[i]) * ramp[i]; + const uint32_t scaled = static_cast(static_cast(product) >> 15); + dst[i] = AddWrap32(dst[i], scaled); } } #endif +#if MKW_AX_MIX_NEON +inline void MixAccumRamp32Neon(int32_t* dst, const int32_t* src, const uint16_t* ramp, + uint32_t count) { + uint32_t i = 0; + for (; count - i >= 4; i += 4) { + const int32x4_t source = vld1q_s32(src + i); + const int32x4_t gain = vreinterpretq_s32_u32(vmovl_u16(vld1_u16(ramp + i))); + const int64x2_t lowProduct = vmull_s32(vget_low_s32(source), vget_low_s32(gain)); + const int64x2_t highProduct = vmull_s32(vget_high_s32(source), vget_high_s32(gain)); + // Keep bits 15..46, including products whose shifted value exceeds int32. + const int32x4_t scaled = vcombine_s32(vshrn_n_s64(lowProduct, 15), + vshrn_n_s64(highProduct, 15)); + vst1q_s32(dst + i, vaddq_s32(vld1q_s32(dst + i), scaled)); + } + MixAccumRamp32Scalar(dst + i, src + i, ramp + i, count - i); +} +#endif + inline void MixAccumRamp32(int32_t* dst, const int32_t* src, const uint16_t* ramp, uint32_t count) { #if MKW_AX_MIX_AVX2 MixAccumRamp32Avx2(dst, src, ramp, count); +#elif MKW_AX_MIX_NEON + MixAccumRamp32Neon(dst, src, ramp, count); #else MixAccumRamp32Scalar(dst, src, ramp, count); #endif @@ -270,9 +362,31 @@ inline void LoadBigEndian32Avx2(int32_t* dst, const uint8_t* src, size_t count) } #endif +#if MKW_AX_MIX_NEON +inline void StoreBigEndian32Neon(uint8_t* dst, const int32_t* src, size_t count) { + size_t i = 0; + for (; count - i >= 4; i += 4) { + const uint8x16_t bytes = vreinterpretq_u8_s32(vld1q_s32(src + i)); + vst1q_u8(dst + i * sizeof(uint32_t), vrev32q_u8(bytes)); + } + StoreBigEndian32Scalar(dst + i * sizeof(uint32_t), src + i, count - i); +} + +inline void LoadBigEndian32Neon(int32_t* dst, const uint8_t* src, size_t count) { + size_t i = 0; + for (; count - i >= 4; i += 4) { + const uint8x16_t bytes = vld1q_u8(src + i * sizeof(uint32_t)); + vst1q_s32(dst + i, vreinterpretq_s32_u8(vrev32q_u8(bytes))); + } + LoadBigEndian32Scalar(dst + i, src + i * sizeof(uint32_t), count - i); +} +#endif + inline void StoreBigEndian32(uint8_t* dst, const int32_t* src, size_t count) { #if MKW_AX_MIX_AVX2 StoreBigEndian32Avx2(dst, src, count); +#elif MKW_AX_MIX_NEON + StoreBigEndian32Neon(dst, src, count); #else StoreBigEndian32Scalar(dst, src, count); #endif @@ -281,6 +395,8 @@ inline void StoreBigEndian32(uint8_t* dst, const int32_t* src, size_t count) { inline void LoadBigEndian32(int32_t* dst, const uint8_t* src, size_t count) { #if MKW_AX_MIX_AVX2 LoadBigEndian32Avx2(dst, src, count); +#elif MKW_AX_MIX_NEON + LoadBigEndian32Neon(dst, src, count); #else LoadBigEndian32Scalar(dst, src, count); #endif diff --git a/runtime/src/hle/gx/gx_copy.cpp b/runtime/src/hle/gx/gx_copy.cpp index 2f13b3b50..56640605f 100644 --- a/runtime/src/hle/gx/gx_copy.cpp +++ b/runtime/src/hle/gx/gx_copy.cpp @@ -165,19 +165,13 @@ PPC_NATIVE_OVERRIDE_VOID(8016fc24, GX__SetDispCopyGamma_8016fc24, (uint32_t g), extern "C" void GX__CopyDisp_8016fc38(uint32_t da, uint32_t c) { EnsureAuroraFrameActive(); if (GxWorker::ShouldDefer()) { - // The worker drains like the inline path; EndAuroraFrame joins it before the next frame. + // The copy is ordered with draws in Aurora's FIFO; frame end drains it. GxWorker::Submit([da, c] { - GXDrawDone(); GXCopyDisp(GuestToHostPtr(da), (GXBool)c); }); } else { - // GX copies are FIFO-ordered on hardware. Drain submitted draws before - // resolving the EFB so high-level copies see the same contents. - GXDrawDone(); GXCopyDisp(GuestToHostPtr(da), (GXBool)c); - // Present immediately so post-copy draws don't leak into this frame. Join at the DONE phase - // (not the cheaper SEALED phase GXDrawDone waits for) because ImGui's draw lists, owned by - // Aurora's render worker, replay during encode; aurora_end_frame would join here anyway. + // ImGui draw lists remain owned by the render worker until encoding completes. aurora_wait_for_frame_worker(); } ++g_gxFrameCount; @@ -195,8 +189,8 @@ extern "C" void GX__CopyTex_8016fd74(uint32_t da, uint32_t c) { // Begin the frame before going inline so a deferred begin reaches the worker first. EnsureAuroraFrameActive(); GX_SYNC_INLINE(); - // Match GX FIFO ordering: texture copies observe all prior draws. - GXDrawDone(); + // Flush dirty state without inserting a completion token before the ordered copy. + GXFlush(); const uint16_t rawSrcLeft = g_texCopyState.srcLeft; const uint16_t rawSrcTop = g_texCopyState.srcTop; const uint16_t rawSrcWidth = g_texCopyState.srcWidth; diff --git a/runtime/src/hle/gx/gx_dl.cpp b/runtime/src/hle/gx/gx_dl.cpp index 71e18fbcd..255b19f94 100644 --- a/runtime/src/hle/gx/gx_dl.cpp +++ b/runtime/src/hle/gx/gx_dl.cpp @@ -1,10 +1,10 @@ #include "gx_internal.h" #include "gx_stream_common.h" #include "gx_cp_decode.h" +#include "gx_dl_cache.h" #include "isa/big_endian.h" #include -#include // The display-list scan cache validates a cached scan against the live guest // bytes with a 64-bit XXH3 digest (see GxDisplayListScanCache::CanReuse). @@ -502,10 +502,7 @@ struct DlScanCacheRecord { std::vector flattened{}; }; -struct DlScanCacheState { - std::unordered_map entries{}; - size_t storedCommandBytes = 0; -}; +using DlScanCacheState = GxDisplayListScanCache::BoundedCache; static uint64_t HashScanLayoutState() { if (!g_hleGxState.vtxLayoutHashDirty) { @@ -537,7 +534,7 @@ static uint64_t DlScanCacheKey(uint32_t listAddr, uint32_t nbytes, uint64_t layo } static DlScanCacheState& DlScanCache() { - static thread_local DlScanCacheState s_cache; + static thread_local DlScanCacheState s_cache{kDlScanCacheMaxEntries, kDlScanCacheMaxStoredBytes}; return s_cache; } @@ -560,9 +557,8 @@ static DlScanCacheProbe ProbeDlScanCache(const uint8_t* list, uint32_t listAddr, auto& s_cache = DlScanCache(); const uint64_t key = DlScanCacheKey(listAddr, nbytes, layoutHash); - const auto it = s_cache.entries.find(key); - if (it != s_cache.entries.end()) { - auto& record = it->second; + if (auto* found = s_cache.Find(key)) { + auto& record = *found; const auto& entry = record.result; const bool identityMatches = entry.listAddr == CanonicalizeGxMainRamAddress(listAddr) && entry.nbytes == nbytes && entry.layoutHash == layoutHash; @@ -586,6 +582,9 @@ static DlScanCacheProbe ProbeDlScanCache(const uint8_t* list, uint32_t listAddr, } } } + if (probe.record != nullptr) { + s_cache.Touch(key); + } return probe; } @@ -600,18 +599,6 @@ static void StoreDlScanCache(uint32_t listAddr, uint32_t nbytes, uint64_t layout uint32_t flattenedBytes) { auto& s_cache = DlScanCache(); const uint64_t key = DlScanCacheKey(listAddr, nbytes, layoutHash); - auto old = s_cache.entries.find(key); - bool replacing = old != s_cache.entries.end(); - size_t replacedBytes = replacing ? DlScanCacheRecordBytes(old->second) : 0; - const size_t incomingBytes = static_cast(flattenedBytes) + - cpWrites.size() * sizeof(DlCpWrite); - if ((!replacing && s_cache.entries.size() >= kDlScanCacheMaxEntries) || - s_cache.storedCommandBytes - replacedBytes + incomingBytes > kDlScanCacheMaxStoredBytes) { - s_cache.entries.clear(); - s_cache.storedCommandBytes = 0; - replacing = false; - replacedBytes = 0; - } DlScanCacheRecord record{}; record.result = entry; record.mayContainDraw = mayContainDraw; @@ -626,12 +613,8 @@ static void StoreDlScanCache(uint32_t listAddr, uint32_t nbytes, uint64_t layout if (flattened != nullptr && flattenedBytes != 0) { record.flattened.assign(flattened, flattened + flattenedBytes); } - auto [it, inserted] = s_cache.entries.insert_or_assign(key, std::move(record)); - (void)inserted; - s_cache.storedCommandBytes += DlScanCacheRecordBytes(it->second); - if (replacing) { - s_cache.storedCommandBytes -= replacedBytes; - } + const size_t incomingBytes = DlScanCacheRecordBytes(record); + s_cache.Insert(key, std::move(record), incomingBytes); } static void BuildScanVertexLayout(GXVtxFmt vtxfmt, std::array& indexedSteps, diff --git a/runtime/src/hle/gx/gx_dl_cache.h b/runtime/src/hle/gx/gx_dl_cache.h new file mode 100644 index 000000000..927534214 --- /dev/null +++ b/runtime/src/hle/gx/gx_dl_cache.h @@ -0,0 +1,121 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace GxDisplayListScanCache { + +template +class BoundedCache { + struct Entry { + uint64_t key; + Value value; + size_t bytes; + }; + using Entries = std::list; + using Iterator = typename Entries::iterator; + +public: + static constexpr size_t kMaxEvictionsPerStore = 64; + + BoundedCache(size_t maxEntries, size_t maxBytes) + : m_maxEntries(maxEntries), m_maxBytes(maxBytes) {} + BoundedCache(const BoundedCache&) = delete; + BoundedCache& operator=(const BoundedCache&) = delete; + + Value* Find(uint64_t key) { + const auto it = m_index.find(key); + return it == m_index.end() ? nullptr : &it->second->value; + } + + // Only validated hits refresh recency. + void Touch(uint64_t key) { + const auto it = m_index.find(key); + if (it != m_index.end()) { + m_entries.splice(m_entries.begin(), m_entries, it->second); + } + } + + bool Insert(uint64_t key, Value&& value, size_t bytes) { + static_assert(std::is_nothrow_move_assignable_v); + if (m_maxEntries == 0 || bytes > m_maxBytes) { + return false; + } + const auto found = m_index.find(key); + const auto old = found == m_index.end() ? m_entries.end() : found->second; + size_t retainedEntries = m_entries.size() - (old != m_entries.end() ? 1u : 0u); + size_t retainedBytes = m_storedBytes - (old != m_entries.end() ? old->bytes : 0u); + const auto fits = [&] { + return retainedEntries < m_maxEntries && retainedBytes <= m_maxBytes - bytes; + }; + + std::array victims{}; + size_t victimCount = 0; + auto next = m_entries.end(); + while (!fits() && next != m_entries.begin() && victimCount < victims.size()) { + const auto victim = --next; + if (victim == old) { + continue; + } + victims[victimCount++] = victim; + retainedBytes -= victim->bytes; + --retainedEntries; + } + // Retire at most one batch so later attempts can make progress too. + if (!fits()) { + for (size_t i = 0; i < victimCount; ++i) { + Erase(victims[i]); + } + return false; + } + + if (old == m_entries.end()) { + m_entries.push_front({key, std::move(value), bytes}); + try { + m_index.emplace(key, m_entries.begin()); + } catch (...) { + m_entries.pop_front(); + throw; + } + } else { + old->value = std::move(value); + m_storedBytes -= old->bytes; + old->bytes = bytes; + m_entries.splice(m_entries.begin(), m_entries, old); + } + for (size_t i = 0; i < victimCount; ++i) { + Erase(victims[i]); + } + m_storedBytes += bytes; + return true; + } + + void Clear() { + m_index.clear(); + m_entries.clear(); + m_storedBytes = 0; + } + + size_t Size() const { return m_entries.size(); } + size_t StoredBytes() const { return m_storedBytes; } + +private: + void Erase(Iterator entry) { + m_storedBytes -= entry->bytes; + m_index.erase(entry->key); + m_entries.erase(entry); + } + + size_t m_maxEntries; + size_t m_maxBytes; + Entries m_entries; + std::unordered_map m_index; + size_t m_storedBytes = 0; +}; + +} // namespace GxDisplayListScanCache diff --git a/runtime/src/hle/gx/gx_fifo.cpp b/runtime/src/hle/gx/gx_fifo.cpp index 139b86dba..4bacfc726 100644 --- a/runtime/src/hle/gx/gx_fifo.cpp +++ b/runtime/src/hle/gx/gx_fifo.cpp @@ -13,6 +13,7 @@ HleGxState g_hleGxState; namespace aurora::gx::fifo { bool submit_raw_draw(GXPrimitive prim, GXVtxFmt fmt, const uint8_t* vertices, uint16_t vtxCount, uint32_t vertexBytes); +void write_raw_draw_data(const uint8_t* data, uint32_t length); } void HleGxState::ResetVertex() { @@ -83,6 +84,73 @@ static bool TryGetRawDirectFifoVertexSize(GXVtxFmt fmt, uint32_t& vertexSize) { return vertexSize != 0; } +void GxStream::PrepareImmediateDirectDraw() { + auto& g = g_hleGxState; + g.fifoDirectBytesRemaining = 0; + g.fifoDirectVertexSize = 0; + g.fifoPendingDrawGeneration = g.vtxStateGeneration; + if (!g.inBegin || g.vertsRemaining > 0xffffu || g.currentVtxFmt >= GX_MAX_VTXFMT) { + return; + } + uint32_t vertexSize = 0; + for (int attr = 0; attr < 26; ++attr) { + if (g.vtxDesc[attr] == GX_NONE) { + continue; + } + if (g.vtxDesc[attr] != GX_DIRECT || attr == GX_VA_NRM || attr > GX_VA_TEX7) { + return; + } + const auto& fmt = g.vtxAttrFmt[g.currentVtxFmt][attr]; + if (attr == GX_VA_CLR0 || attr == GX_VA_CLR1) { + // Packed colors and RGBX padding are re-encoded by the attribute writer. + if (!((fmt.type == GX_RGB8 && fmt.cnt == GX_CLR_RGB) || + (fmt.type == GX_RGBA8 && fmt.cnt == GX_CLR_RGBA))) { + return; + } + } else if (!IsMatrixIndexAttr(static_cast(attr))) { + if (static_cast(fmt.type) > GX_F32 || fmt.frac > 31 || + static_cast(fmt.cnt) > 1u) { + return; + } + } + vertexSize += GetDirectAttrByteSizeForFifo(static_cast(attr), fmt); + } + g.fifoDirectVertexSize = vertexSize; + g.fifoDirectBytesRemaining = vertexSize * g.vertsRemaining; +} + +static uint32_t WriteImmediateDirectBytes(const uint8_t* bytes, uint32_t size) { + auto& g = g_hleGxState; + if (!g.inBegin || g.fifoDirectBytesRemaining == 0) { + return 0; + } + if (!g.auroraBeginCalled) { + if (g.fifoPendingDrawGeneration != g.vtxStateGeneration) { + GxStream::PrepareImmediateDirectDraw(); + if (g.fifoDirectBytesRemaining == 0) { + return 0; + } + } + EnsureAuroraFrameActive(); + ApplyAuroraVtxStateForRawBegin(g.currentVtxFmt); + EnsureDefaultGxAlphaCompare(); + GXBegin(g.currentPrim, g.currentVtxFmt, static_cast(g.vertsRemaining)); + g.auroraBeginCalled = true; + GXMarkFrameWork(); + } + const uint32_t count = std::min(size, g.fifoDirectBytesRemaining); + aurora::gx::fifo::write_raw_draw_data(bytes, count); + g.fifoDirectBytesRemaining -= count; + g.vertsRemaining = (g.fifoDirectBytesRemaining + g.fifoDirectVertexSize - 1u) / g.fifoDirectVertexSize; + if (g.fifoDirectBytesRemaining == 0) { + g.inBegin = false; + g.auroraBeginCalled = false; + g.ResetVertex(); + GXEnd(); + } + return count; +} + static bool TrySubmitRawDirectFifoDraw(const uint8_t* packet, uint32_t packetBytes, GXPrimitive prim, GXVtxFmt vtxFmt, uint16_t vtxCount) { if (packet == nullptr || packetBytes == 0 || vtxCount == 0) { @@ -369,6 +437,19 @@ static MKW_PPC_NO_INLINE void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes); // `val` is a raw big-endian bit pattern: the FIFO stream is type-agnostic, and // the float entry point converts before it gets here. void HleFifoWrite(u32 val, uint32_t sizeBytes) { + if (!IsDisplayListActive() && g_hleGxState.fifoDirectBytesRemaining != 0) { + const uint32_t count = sizeBytes == 0 || sizeBytes > 4 ? 4u : sizeBytes; + uint8_t bytes[4]; + BigEndian::Write32(bytes, val); + const uint8_t* data = bytes + 4u - count; + const uint32_t consumed = WriteImmediateDirectBytes(data, count); + if (consumed != 0) { + for (uint32_t i = consumed; i < count; ++i) { + HleFifoWriteGeneral(data[i], 1); + } + return; + } + } if ((sizeBytes == 0 || sizeBytes == 4) && !IsDisplayListActive() && TryAppendIncompleteXfWord(val)) { return; } @@ -385,6 +466,7 @@ static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { auto resetFifoBuffer = [&]() { g_hleGxState.fifoReadOffset = 0; g_hleGxState.fifoByteCount = 0; + g_hleGxState.fifoPendingDrawBytes = 0; }; auto compactFifoBuffer = [&]() { @@ -440,6 +522,7 @@ static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { } g_hleGxState.fifoReadOffset += count; g_hleGxState.fifoByteCount -= count; + g_hleGxState.fifoPendingDrawBytes = 0; if (g_hleGxState.fifoByteCount == 0) { g_hleGxState.fifoReadOffset = 0; } @@ -448,9 +531,17 @@ static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { }; pushBytes(val, sizeBytes == 0 ? 4 : sizeBytes); + if (g_hleGxState.fifoPendingDrawBytes != 0) { + if (g_hleGxState.fifoPendingDrawGeneration == g_hleGxState.vtxStateGeneration && + g_hleGxState.fifoByteCount < g_hleGxState.fifoPendingDrawBytes) { + return; + } + g_hleGxState.fifoPendingDrawBytes = 0; + } // Parse raw FIFO command packets written directly to the gather pipe // (e.g. NW4R/G3D paths that do not call the GXBegin wrapper function). +parseBufferedPackets: while (!g_hleGxState.inBegin) { if (g_hleGxState.fifoByteCount < 1) { break; @@ -535,8 +626,10 @@ static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { ? 3u + static_cast(vtxCount) * rawVertexSize : 0u; // A packet that cannot fit the buffer is streamed per vertex instead. - if (packetBytes != 0 && packetBytes <= g_hleGxState.fifoBytes.size()) { + if (packetBytes != 0 && packetBytes + 3u <= g_hleGxState.fifoBytes.size()) { if (g_hleGxState.fifoByteCount < packetBytes) { + g_hleGxState.fifoPendingDrawBytes = packetBytes; + g_hleGxState.fifoPendingDrawGeneration = g_hleGxState.vtxStateGeneration; break; } if (TrySubmitRawDirectFifoDraw(data, packetBytes, prim, vtxFmt, vtxCount)) { @@ -695,6 +788,9 @@ static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { } break; } + if (!g_hleGxState.inBegin && g_hleGxState.fifoByteCount != 0) { + goto parseBufferedPackets; + } } // Must behave exactly like GX_HLE_FIFO_Write32/16/8 run byte by byte. The ring is not @@ -851,6 +947,15 @@ static void FifoWriteBurstImpl(const uint8_t* data, uint32_t sizeBytes) { return; } + if (!IsDisplayListActive()) { + const uint32_t consumed = WriteImmediateDirectBytes(data, sizeBytes); + data += consumed; + sizeBytes -= consumed; + if (sizeBytes == 0) { + return; + } + } + const uint32_t applied = ApplyFifoPacketsDirect(data, sizeBytes); if (applied < sizeBytes) { HleFifoWriteBurstChunked(data + applied, sizeBytes - applied); diff --git a/runtime/src/hle/gx/gx_init.cpp b/runtime/src/hle/gx/gx_init.cpp index edf6f9c12..8243f8bfc 100644 --- a/runtime/src/hle/gx/gx_init.cpp +++ b/runtime/src/hle/gx/gx_init.cpp @@ -30,6 +30,7 @@ extern "C" uint32_t GX__Init_8016b850(uint32_t fifoBase, uint32_t fifoSize) constexpr uint32_t kGXDataSize = 0x600u; GXInit(GuestToHostPtr(fifoBase, fifoSize), fifoSize); + InvalidateTexObjLoads(); // Initialize GXData structure in guest memory try { diff --git a/runtime/src/hle/gx/gx_internal.h b/runtime/src/hle/gx/gx_internal.h index b8270a9b9..9addd1c07 100644 --- a/runtime/src/hle/gx/gx_internal.h +++ b/runtime/src/hle/gx/gx_internal.h @@ -154,6 +154,8 @@ struct TexObjSlot : TexObjMeta { uint64_t guestShadow[4] = {0, 0, 0, 0}; bool hasGuestShadow = false; + GxTextureBindingContract::ValidatedLoad validatedLoad; + // DCStoreRange interval-index bookkeeping: what this slot currently // contributes to g_texObjIntervals, so a pending update that turns out to // be a no-op costs a compare instead of an erase + insert. @@ -256,6 +258,10 @@ struct HleGxState { std::array fifoBytes{}; size_t fifoReadOffset = 0; size_t fifoByteCount = 0; + uint32_t fifoPendingDrawBytes = 0; + uint64_t fifoPendingDrawGeneration = 0; + uint32_t fifoDirectBytesRemaining = 0; + uint32_t fifoDirectVertexSize = 0; void ResetVertex(); void InvalidateVtxLayoutHash() noexcept { @@ -347,6 +353,10 @@ bool ValidateTlutData(uint32_t objAddr, const TlutObjMeta& meta); TexObjMeta& GetTexObjMeta(uint32_t addr); TexObjMeta ExtractTexObjMetaFromGuest(uint32_t addr); bool TryGetOrExtractTexObjMeta(uint32_t addr, TexObjMeta& outMeta); +// Lookup and publication require g_texObjMutex; reset acquires it itself. +TexObjSlot* TryGetValidatedTexObjSlot(uint32_t addr); +void MarkTexObjLoadValidated(uint32_t addr); +void InvalidateTexObjLoads(); // Guest GXTexObj bytes captured on the game thread; while a scope is active on a thread, texobj // decoding for that address reads the captured bytes instead of live guest memory. diff --git a/runtime/src/hle/gx/gx_objects.cpp b/runtime/src/hle/gx/gx_objects.cpp index 645658dfb..d87665610 100644 --- a/runtime/src/hle/gx/gx_objects.cpp +++ b/runtime/src/hle/gx/gx_objects.cpp @@ -520,6 +520,37 @@ void MarkTexObjSyncedLocked(TexObjSlot& slot) noexcept { } // namespace +TexObjSlot* TryGetValidatedTexObjSlot(uint32_t addr) { + TexObjSlot* slot = FindTexObjSlot(addr); + if (slot == nullptr) { + return nullptr; + } + if (slot->validatedLoad.CanReuse(TexObjSnapshotFor(addr), slot->needsUpload, + slot->host && slot->host->storageLive && slot->host->constructed, + IsPaletteTexFormat(slot->format))) { + return slot; + } + slot->validatedLoad.Invalidate(); + return nullptr; +} + +void MarkTexObjLoadValidated(uint32_t addr) { + TexObjSlot* slot = FindTexObjSlot(addr); + if (slot != nullptr && !slot->needsUpload && slot->host && slot->host->constructed && + !IsPaletteTexFormat(slot->format)) { + // Only call-time bytes may identify a completed load, never newer producer memory. + slot->validatedLoad.Record(TexObjSnapshotFor(addr)); + } +} + +void InvalidateTexObjLoads() { + std::lock_guard guard(g_texObjMutex); + for (auto& [addr, slot] : g_TexObjMeta) { + slot.validatedLoad.Invalidate(); + } + g_boundTexMaps = {}; +} + bool TryGetOrExtractTexObjMeta(uint32_t addr, TexObjMeta& outMeta) { TexObjSlot* slot = FindTexObjSlot(addr); if (slot != nullptr) { @@ -553,6 +584,7 @@ bool TryGetOrExtractTexObjMeta(uint32_t addr, TexObjMeta& outMeta) { const bool extractedUsable = IsUsableTexObjMeta(extracted); if (extractedUsable) { QueueTexObjIntervalUpdate(*slot); + slot->validatedLoad.Invalidate(); static_cast(*slot) = extracted; MarkTexObjSyncedLocked(*slot); outMeta = extracted; @@ -575,6 +607,7 @@ bool TryGetOrExtractTexObjMeta(uint32_t addr, TexObjMeta& outMeta) { // queues the interval-index patch and re-captures the guest shadow before that happens. TexObjMeta& GetTexObjMeta(uint32_t addr) { TexObjSlot& slot = FindOrCreateTexObjSlot(addr); + slot.validatedLoad.Invalidate(); QueueTexObjIntervalUpdate(slot); MarkTexObjSyncedLocked(slot); return slot; @@ -604,6 +637,7 @@ GXTexObj* GetHostTexObj(uint32_t addr) { GXTexObj* CreateHostTexObj(uint32_t addr) { TexObjSlot& slot = FindOrCreateTexObjSlot(addr); + slot.validatedLoad.Invalidate(); // GXInitTexObj/GXInitTexObjCI land here and rewrite the entire guest struct, // resetting the LOD/filter words the HLE does not mirror into this cache. // The shadow captured before that rewrite mismatches afterwards, so the diff --git a/runtime/src/hle/gx/gx_stream_common.h b/runtime/src/hle/gx/gx_stream_common.h index 006ab4711..8167f37de 100644 --- a/runtime/src/hle/gx/gx_stream_common.h +++ b/runtime/src/hle/gx/gx_stream_common.h @@ -36,6 +36,9 @@ constexpr uint8_t GX_DRAW_POINTS_CMD = 0xB8; namespace GxStream { +// Called after the immediate begin state has been initialized. +void PrepareImmediateDirectDraw(); + // --- Big-endian scalar reads out of a command stream --------------------- inline uint16_t ReadBE16(const uint8_t* ptr) { return BigEndian::Read16(ptr); } diff --git a/runtime/src/hle/gx/gx_texture.cpp b/runtime/src/hle/gx/gx_texture.cpp index 0f7953334..f643d9daf 100644 --- a/runtime/src/hle/gx/gx_texture.cpp +++ b/runtime/src/hle/gx/gx_texture.cpp @@ -511,6 +511,34 @@ static void LoadTexObjImpl(uint32_t oa, uint32_t tid) { return; } + if (TexObjSlot* slot = TryGetValidatedTexObjSlot(oa)) { + try { + BoundTexInfo newBound{}; + static_cast(newBound) = *slot; + newBound.objAddr = oa; + newBound.dataAddr = CanonicalizeGxMainRamAddress(slot->dataAddr); + if (!GxTextureBindingContract::CanSkipHostLoad( + g_boundTexMaps[tid], newBound, false, false, false)) { + GXLoadTexObj(slot->host->PublicPtr(), (GXTexMapID)tid); + } + g_boundTexMaps[tid] = newBound; + } catch (const std::exception& ex) { + slot->validatedLoad.Invalidate(); + if (s_hostExceptionLogCount++ < 64) { + RT_LOGF(RT_TAG_GX, "GXLoadTexObj rejected reason=host-exception (%s) oa=0x%08X tid=%u\n", + ex.what(), oa, tid); + } + BindUnloadableTexturePlaceholder(tid); + } catch (...) { + slot->validatedLoad.Invalidate(); + if (s_hostExceptionLogCount++ < 64) { + RT_LOGF(RT_TAG_GX, "GXLoadTexObj rejected reason=host-exception-unknown oa=0x%08X tid=%u\n", oa, tid); + } + BindUnloadableTexturePlaceholder(tid); + } + return; + } + // Check if we have metadata for this texture, or try to extract it from guest memory TexObjMeta meta; if (!TryGetOrExtractTexObjMeta(oa, meta)) { @@ -661,6 +689,7 @@ static void LoadTexObjImpl(uint32_t oa, uint32_t tid) { if (!canSkipHostLoad) { GXLoadTexObj(obj, (GXTexMapID)tid); } + MarkTexObjLoadValidated(oa); } catch (const std::exception& ex) { if (s_hostExceptionLogCount++ < 64) { RT_LOGF(RT_TAG_GX, @@ -681,24 +710,20 @@ static void LoadTexObjImpl(uint32_t oa, uint32_t tid) { } extern "C" void GX__LoadTexObj_80170f2c(uint32_t oa, uint32_t tid) { - if (GxWorker::ShouldDefer()) { - // The CPU-side SDK reads the GXTexObj at call time; the texture data itself is read later. - std::optional bytes; - try { - bytes = CaptureTexObjBytes(oa); - } catch (...) { - } - GxWorker::Submit([oa, tid, bytes] { - if (bytes) { - TexObjSnapshotScope snapshot(oa, *bytes); - LoadTexObjImpl(oa, tid); - } else { - LoadTexObjImpl(oa, tid); - } - }); - } else { - LoadTexObjImpl(oa, tid); + // Inline and deferred loads use the same complete call-time snapshot. + std::optional bytes; + try { + bytes = CaptureTexObjBytes(oa); + } catch (...) { } + GxWorker::Run([oa, tid, bytes] { + if (bytes) { + TexObjSnapshotScope snapshot(oa, *bytes); + LoadTexObjImpl(oa, tid); + } else { + LoadTexObjImpl(oa, tid); + } + }); try { uint32_t gd = Memory::Read32(kGXDataPtrAddr); if (gd) { Memory::Write32(gd + 0x5FCu, Memory::Read32(gd + 0x5FCu) | 1u); Memory::Write16(gd + 2, 0); } } catch (...) {} } PPC_NATIVE_OVERRIDE_VOID(80170f2c, GX__LoadTexObj_80170f2c, (uint32_t oa, uint32_t tid), (oa, tid)); diff --git a/runtime/src/hle/gx/gx_texture_binding_contract.h b/runtime/src/hle/gx/gx_texture_binding_contract.h index f8ca7b233..0ede802b6 100644 --- a/runtime/src/hle/gx/gx_texture_binding_contract.h +++ b/runtime/src/hle/gx/gx_texture_binding_contract.h @@ -1,9 +1,31 @@ #pragma once #include +#include namespace GxTextureBindingContract { +// Independent of the metadata shadow, which getters and setters may refresh. +struct ValidatedLoad { + uint8_t bytes[32]{}; + bool valid = false; + + void Invalidate() noexcept { valid = false; } + + void Record(const uint8_t* snapshot) noexcept { + valid = snapshot != nullptr; + if (valid) { + std::memcpy(bytes, snapshot, sizeof(bytes)); + } + } + + bool CanReuse(const uint8_t* snapshot, bool needsUpload, bool constructed, + bool isPalette) const noexcept { + return valid && snapshot != nullptr && !needsUpload && constructed && !isPalette && + std::memcmp(bytes, snapshot, sizeof(bytes)) == 0; + } +}; + // Every piece of GX texture state that can change the bound view/sampler. Shared by `State` // (this plus the guest GXTexObj address) and HLE's TexObjMeta (this plus the `userData`/ // `needsUpload` bookkeeping fields), so a new sampler field here reaches both. diff --git a/runtime/src/hle/gx/gx_vertex.cpp b/runtime/src/hle/gx/gx_vertex.cpp index 3a8c43b89..e0e90bced 100644 --- a/runtime/src/hle/gx/gx_vertex.cpp +++ b/runtime/src/hle/gx/gx_vertex.cpp @@ -329,11 +329,15 @@ static void GxBeginImpl(uint32_t t, uint32_t vf, uint32_t nv) { g_hleGxState.vertsRemaining=nv; g_hleGxState.inBegin=nv!=0; g_hleGxState.auroraBeginCalled=false; g_hleGxState.fifoReadOffset = 0; g_hleGxState.fifoByteCount=0; g_hleGxState.ResetVertex(); + g_hleGxState.fifoPendingDrawBytes = 0; + GxStream::PrepareImmediateDirectDraw(); } PPC_NATIVE_OVERRIDE_VOID(8016f0f0, GX__Begin_8016f0f0, (uint32_t t, uint32_t vf, uint32_t nv), (t, vf, nv)); extern "C" void GX__End_80044b30() { - GX_DEFER_CALL0(GX__End_80044b30); g_hleGxState.inBegin=false; GXEnd(); } + GX_DEFER_CALL0(GX__End_80044b30); + g_hleGxState.fifoDirectBytesRemaining = 0; + g_hleGxState.inBegin=false; GXEnd(); } PPC_NATIVE_OVERRIDE_VOID(80044b30, GX__End_80044b30, (), ()); extern "C" void GX__End_80048c30() { GX__End_80044b30(); } diff --git a/runtime/src/hle/gx/gx_worker.cpp b/runtime/src/hle/gx/gx_worker.cpp index ac8983e05..32279eb94 100644 --- a/runtime/src/hle/gx/gx_worker.cpp +++ b/runtime/src/hle/gx/gx_worker.cpp @@ -54,7 +54,9 @@ uint64_t g_write = 0; std::atomic g_head{0}; std::atomic g_tail{0}; std::atomic g_consumerWaiting{false}; -std::atomic g_producerWaiting{false}; +constexpr uint64_t kNotWaiting = ~uint64_t{0}; +// Publish the required tail before rechecking it under g_tailMutex. +std::atomic g_waitTarget{kNotWaiting}; std::mutex g_headMutex; std::condition_variable g_headCv; std::mutex g_tailMutex; @@ -66,6 +68,14 @@ std::atomic g_exited{false}; std::thread* g_thread = nullptr; std::vector g_fifoBytes; +#if defined(MKW_GX_WORKER_TESTING) +bool g_testEnabled = true; +bool g_testWakeEachRecord = false; +std::atomic g_testNotifications{0}; +std::atomic g_testWaits{0}; +void (*g_testBeforeTailRecheck)() = nullptr; +#endif + constexpr size_t AlignUp(size_t value) { return (value + detail::kAlign - 1) & ~(detail::kAlign - 1); } inline void CpuRelax() { @@ -117,11 +127,22 @@ void WaitForTail(uint64_t minTail, bool service) { { PerfStats::WaitScope wait; std::unique_lock lock{g_tailMutex}; - g_producerWaiting.store(true, std::memory_order_seq_cst); + g_waitTarget.store(minTail, std::memory_order_seq_cst); +#if defined(MKW_GX_WORKER_TESTING) + if (g_testWakeEachRecord) { + g_waitTarget.store(0, std::memory_order_seq_cst); + } + if (g_testBeforeTailRecheck != nullptr) { + g_testBeforeTailRecheck(); + } +#endif if (g_tail.load(std::memory_order_seq_cst) < minTail) { +#if defined(MKW_GX_WORKER_TESTING) + g_testWaits.fetch_add(1, std::memory_order_relaxed); +#endif WaitTailBriefly(lock); } - g_producerWaiting.store(false, std::memory_order_relaxed); + g_waitTarget.store(kNotWaiting, std::memory_order_relaxed); } if (service && g_waitCallback != nullptr && g_tail.load(std::memory_order_acquire) < minTail) { g_waitCallback(); @@ -234,8 +255,11 @@ void WorkerMain() { } tail += header->size; g_tail.store(tail, std::memory_order_seq_cst); - if (g_producerWaiting.load(std::memory_order_seq_cst)) { + if (tail >= g_waitTarget.load(std::memory_order_seq_cst)) { std::lock_guard lock{g_tailMutex}; +#if defined(MKW_GX_WORKER_TESTING) + g_testNotifications.fetch_add(1, std::memory_order_relaxed); +#endif g_tailCv.notify_all(); } } @@ -342,7 +366,12 @@ void Start() { if (Active()) { return; } - if (!RuntimeConfigFile::GxWorkerEnabled(true)) { +#if defined(MKW_GX_WORKER_TESTING) + const bool enabled = g_testEnabled; +#else + const bool enabled = RuntimeConfigFile::GxWorkerEnabled(true); +#endif + if (!enabled) { RT_LOGF(RT_TAG_GX, "GX worker disabled\n"); return; } @@ -350,6 +379,7 @@ void Start() { g_ring = static_cast(::operator new(kCapacity, std::align_val_t{64})); } g_fifoBytes.reserve(kFifoFlushBytes + 4096); + g_waitTarget.store(kNotWaiting, std::memory_order_relaxed); g_stop.store(false, std::memory_order_relaxed); g_abandon.store(false, std::memory_order_relaxed); g_exited.store(false, std::memory_order_relaxed); diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 3026abd6e..5266e25d0 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -1464,6 +1464,8 @@ int RuntimeMain(int argc, char** argv) { // aurora_initialize does. Wiimote::ConfigureSdlHints(RuntimeConfigFile::WiiRemotesEnabled()); + aurora_set_shader_mode(RuntimeConfigFile::ShaderMode() == "hybrid" + ? AURORA_SHADER_HYBRID : AURORA_SHADER_SPECIALIZED); const AuroraInfo auroraInfo = aurora_initialize(0, nullptr, &auroraConfig); if (auroraInfo.initializationStatus != AURORA_INITIALIZATION_SUCCESS) { throw std::runtime_error(auroraInfo.initializationError != nullptr diff --git a/runtime/src/settings_overlay.cpp b/runtime/src/settings_overlay.cpp index 6efe7e31b..0e32c1334 100644 --- a/runtime/src/settings_overlay.cpp +++ b/runtime/src/settings_overlay.cpp @@ -110,6 +110,7 @@ int g_displayMode = [] { } return static_cast(AURORA_DISPLAY_MODE_WINDOWED); }(); +int g_shaderMode = RuntimeConfigFile::ShaderMode() == "hybrid" ? 1 : 0; bool g_skipUnreadyPipelines = RuntimeConfigFile::SkipUnreadyPipelines(true); bool g_vsync = RuntimeConfigFile::Vsync(); bool g_disableCopyFilter = RuntimeConfigFile::DisableCopyFilter(true); @@ -1278,11 +1279,16 @@ void DrawGraphicsSettings() { aurora_set_disable_copy_filter(g_disableCopyFilter); RuntimeConfigFile::SetDisableCopyFilter(g_disableCopyFilter); } + if (ImGui::Combo("Shaders (restart required)", &g_shaderMode, "Specialized\0Hybrid\0")) { + RuntimeConfigFile::SetShaderMode(g_shaderMode == 1 ? "hybrid" : "specialized"); + } + ImGui::BeginDisabled(g_shaderMode == 1); if (ImGui::Checkbox("Skip draws while shaders compile", &g_skipUnreadyPipelines)) { GX_SYNC_INLINE(); aurora_set_skip_unready_pipelines(g_skipUnreadyPipelines); RuntimeConfigFile::SetSkipUnreadyPipelines(g_skipUnreadyPipelines); } + ImGui::EndDisabled(); if (ImGui::Checkbox("VSync", &g_vsync)) { aurora_enable_vsync(g_vsync); RuntimeConfigFile::SetVsync(g_vsync); diff --git a/runtime/tests/ax_mix_kernels_tests.cpp b/runtime/tests/ax_mix_kernels_tests.cpp new file mode 100644 index 000000000..a9e6f398e --- /dev/null +++ b/runtime/tests/ax_mix_kernels_tests.cpp @@ -0,0 +1,317 @@ +#include "ax_mix_kernels.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(MKW_AX_MIX_REQUIRE_NEON) && !MKW_AX_MIX_NEON +#error The ARM kernel fixture must select NEON +#endif + +namespace { + +struct Backend { + const char* name; + decltype(&AxMixKernels::MixAddRampScalar) mix; + decltype(&AxMixKernels::ScaleRampScalar) scale; + decltype(&AxMixKernels::MixAccumRamp32Scalar) accum; + decltype(&AxMixKernels::StoreBigEndian32Scalar) store; + decltype(&AxMixKernels::LoadBigEndian32Scalar) load; +}; + +constexpr Backend kScalar = {"scalar", AxMixKernels::MixAddRampScalar, + AxMixKernels::ScaleRampScalar, AxMixKernels::MixAccumRamp32Scalar, + AxMixKernels::StoreBigEndian32Scalar, AxMixKernels::LoadBigEndian32Scalar}; +#if MKW_AX_MIX_NEON +constexpr const char* kSelectedName = "NEON"; +#elif MKW_AX_MIX_AVX2 +constexpr const char* kSelectedName = "AVX2"; +#else +constexpr const char* kSelectedName = "selected scalar"; +#endif +constexpr Backend kSelected = {kSelectedName, AxMixKernels::MixAddRamp, + AxMixKernels::ScaleRamp, AxMixKernels::MixAccumRamp32, + AxMixKernels::StoreBigEndian32, AxMixKernels::LoadBigEndian32}; + +uint32_t g_random = 0x12345678u; + +uint32_t NextRandom() { + g_random ^= g_random << 13; + g_random ^= g_random >> 17; + g_random ^= g_random << 5; + return g_random; +} + +void Require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +int32_t Wrap32(int64_t value) { + const uint32_t bits = static_cast(value); + return bits <= 0x7fffffffu ? static_cast(bits) + : static_cast(static_cast(bits) - 0x100000000ll); +} + +int64_t FloorQ15(int64_t product) { + return product >= 0 ? product / 32768 : -((-product + 32767) / 32768); +} + +int16_t ScaleSample(int16_t sample, uint16_t volume) { + const int64_t scaled = FloorQ15(static_cast(sample) * volume); + return static_cast(std::clamp(scaled, -32768, 32767)); +} + +constexpr int16_t kSamples[] = {-32768, 32767, -32767, 16384, -16385, 1, -1, 0}; +constexpr int32_t kWords[] = {std::numeric_limits::min(), + std::numeric_limits::max(), -2147483647, 2147450880, + -65537, 65537, -1, 0, 1}; +constexpr uint16_t kGains[] = {0, 1, 0x3fff, 0x4000, 0x7fff, 0x8000, 0xfffe, 0xffff}; + +std::vector Counts() { + std::vector counts; + for (uint32_t count = 0; count <= 40; ++count) counts.push_back(count); + for (uint32_t count : {63u, 64u, 65u, 95u, 96u, 97u, 159u, 160u, 161u}) { + counts.push_back(count); + } + return counts; +} + +void TestRamps(const Backend& backend, uint32_t count, size_t offset, + uint16_t volume, uint16_t delta, unsigned pattern) { + // Source ends at the allocation boundary so ASan also catches tail overreads. + std::vector input(offset + count, 12345); + std::vector initial(offset + count + 1, 0x13579bdf); + for (uint32_t i = 0; i < count; ++i) { + input[offset + i] = pattern == 0 ? kSamples[i % std::size(kSamples)] + : static_cast(static_cast(NextRandom() & 0xffffu) - 32768); + initial[offset + i] = pattern == 0 ? kWords[i % std::size(kWords)] + : Wrap32(NextRandom()); + } + const auto original = input; + auto expectedMix = initial; + auto expectedScale = input; + int16_t expectedDpop = -23456; + uint16_t expectedVolume = volume; + for (uint32_t i = 0; i < count; ++i) { + const int16_t scaled = ScaleSample(input[offset + i], expectedVolume); + expectedMix[offset + i] = Wrap32(static_cast(initial[offset + i]) + scaled); + expectedScale[offset + i] = scaled; + expectedDpop = scaled; + expectedVolume = static_cast(expectedVolume + delta); + } + auto mixed = initial; + int16_t dpop = -23456; + Require(backend.mix(mixed.data() + offset, input.data() + offset, count, + volume, delta, dpop) == expectedVolume, "mix final volume"); + Require(mixed == expectedMix, "mix output or sentinel"); + Require(dpop == expectedDpop, "mix de-pop sample"); + Require(input == original, "mix modified its source"); + + auto scaled = input; + Require(backend.scale(scaled.data() + offset, count, volume, delta) == expectedVolume, + "scale final volume"); + Require(scaled == expectedScale, "scale output or prefix"); +} + +void TestAccum(const Backend& backend, uint32_t count, size_t offset, unsigned pattern) { + std::vector source(offset + count, 0x2468ace); + std::vector ramp(offset + count, 12345); + std::vector initial(offset + count + 1, 0x13579bdf); + for (uint32_t i = 0; i < count; ++i) { + source[offset + i] = pattern == 0 ? kWords[i % std::size(kWords)] : Wrap32(NextRandom()); + initial[offset + i] = pattern == 0 ? kWords[(i + 1) % std::size(kWords)] + : Wrap32(NextRandom()); + ramp[offset + i] = pattern == 0 ? kGains[i % std::size(kGains)] + : static_cast(NextRandom()); + } + const auto originalSource = source; + const auto originalRamp = ramp; + auto expected = initial; + for (uint32_t i = 0; i < count; ++i) { + expected[offset + i] = Wrap32(static_cast(initial[offset + i]) + + FloorQ15(static_cast(source[offset + i]) * ramp[offset + i])); + } + auto actual = initial; + backend.accum(actual.data() + offset, source.data() + offset, ramp.data() + offset, count); + Require(actual == expected, "aux accumulation or sentinel"); + Require(source == originalSource && ramp == originalRamp, "aux modified its source"); + + auto aliased = initial; + expected = initial; + for (uint32_t i = 0; i < count; ++i) { + expected[offset + i] = Wrap32(static_cast(initial[offset + i]) + + FloorQ15(static_cast(initial[offset + i]) * ramp[offset + i])); + } + backend.accum(aliased.data() + offset, aliased.data() + offset, ramp.data() + offset, count); + Require(aliased == expected, "aux in-place accumulation"); +} + +void TestAccumEdges(const Backend& backend) { + for (int32_t source : kWords) for (int32_t initial : kWords) for (uint16_t gain : kGains) { + std::array src, dst; + std::array ramp; + src.fill(source); + dst.fill(initial); + ramp.fill(gain); + const int32_t expected = Wrap32(static_cast(initial) + + FloorQ15(static_cast(source) * gain)); + backend.accum(dst.data(), src.data(), ramp.data(), 17); + Require(std::all_of(dst.begin(), dst.end(), [expected](int32_t v) { return v == expected; }), + "aux full-width edge"); + } +} + +void TestEndian(const Backend& backend, size_t count, size_t offset) { + std::vector source(offset + count, 0x13579bdf); + for (size_t i = 0; i < count; ++i) { + source[offset + i] = i % 2 == 0 ? kWords[(i / 2) % std::size(kWords)] : Wrap32(NextRandom()); + } + const auto original = source; + std::vector expected(offset + count * 4 + 1, 0xa5); + for (size_t i = 0; i < count; ++i) { + const uint32_t word = static_cast(source[offset + i]); + for (unsigned byte = 0; byte < 4; ++byte) { + expected[offset + i * 4 + byte] = static_cast(word >> (24 - byte * 8)); + } + } + auto bytes = std::vector(expected.size(), 0xa5); + backend.store(bytes.data() + offset, source.data() + offset, count); + Require(bytes == expected, "big-endian store or sentinel"); + Require(source == original, "store modified its source"); + + // Drop the trailing sentinel before loading to expose overreads to ASan. + std::vector loadBytes(expected.begin(), expected.end() - 1); + std::vector loaded(offset + count + 1, 0x13579bdf); + auto expectedWords = loaded; + std::copy(source.begin() + offset, source.end(), expectedWords.begin() + offset); + backend.load(loaded.data() + offset, loadBytes.data() + offset, count); + Require(loaded == expectedWords, "big-endian load or sentinel"); + Require(std::equal(loadBytes.begin(), loadBytes.end(), expected.begin()), "load modified source"); +} + +void TestBackend(const Backend& backend) { + for (uint32_t count : Counts()) for (size_t offset : {1u, 2u, 3u}) { + for (uint16_t volume : kGains) for (uint16_t delta : kGains) { + TestRamps(backend, count, offset, volume, delta, 0); + } + for (unsigned pattern = 0; pattern < 8; ++pattern) TestAccum(backend, count, offset, pattern); + } + for (unsigned trial = 0; trial < 512; ++trial) { + TestRamps(backend, NextRandom() % 162, 1 + NextRandom() % 3, + static_cast(NextRandom()), static_cast(NextRandom()), 1); + } + TestRamps(backend, 65539, 1, 0xffff, 0xffff, 0); + TestAccumEdges(backend); + for (size_t count = 0; count <= 161; ++count) for (size_t offset = 1; offset <= 15; ++offset) { + TestEndian(backend, count, offset); + } +} + +uint64_t Checksum(uint64_t hash, uint32_t value) { + return (hash ^ value) * 1099511628211ull; +} + +#if defined(_MSC_VER) +#define AX_TEST_NOINLINE __declspec(noinline) +#else +#define AX_TEST_NOINLINE __attribute__((noinline)) +#endif + +AX_TEST_NOINLINE uint64_t Benchmark(const Backend& backend, const std::string& kernel, + uint32_t count, uint32_t iterations) { + std::vector samples(count); + std::vector source(count), bus(count); + std::vector ramp(count); + std::vector bytes(count * 4 + 1); + for (uint32_t i = 0; i < count; ++i) { + samples[i] = kSamples[i % std::size(kSamples)]; + source[i] = kWords[i % std::size(kWords)]; + bus[i] = source[i]; + ramp[i] = kGains[i % std::size(kGains)]; + } + AxMixKernels::StoreBigEndian32Scalar(bytes.data() + 1, source.data(), count); + uint16_t volume = 0xfffe; + int16_t dpop = -12345; + const auto start = std::chrono::steady_clock::now(); + if (kernel == "mix") { + for (uint32_t i = 0; i < iterations; ++i) { + volume = backend.mix(bus.data(), samples.data(), count, volume, 0xffff, dpop); + } + } else if (kernel == "scale") { + for (uint32_t i = 0; i < iterations; ++i) { + samples[i % count] = kSamples[i % std::size(kSamples)]; + volume = backend.scale(samples.data(), count, volume, 0xffff); + } + } else if (kernel == "accum") { + for (uint32_t i = 0; i < iterations; ++i) { + backend.accum(bus.data(), source.data(), ramp.data(), count); + } + } else if (kernel == "store") { + for (uint32_t i = 0; i < iterations; ++i) { + source[i % count] = Wrap32(i); + backend.store(bytes.data() + 1, source.data(), count); + } + } else if (kernel == "load") { + for (uint32_t i = 0; i < iterations; ++i) { + bytes[1 + i % (count * 4)] = static_cast(i); + backend.load(bus.data(), bytes.data() + 1, count); + } + } + const auto elapsed = std::chrono::duration( + std::chrono::steady_clock::now() - start).count(); + uint64_t hash = Checksum(Checksum(14695981039346656037ull, volume), static_cast(dpop)); + for (int32_t value : bus) hash = Checksum(hash, static_cast(value)); + for (int16_t value : samples) hash = Checksum(hash, static_cast(value)); + for (uint8_t value : bytes) hash = Checksum(hash, value); + std::printf("%s,%s,%u,%u,%.3f,%.3f,%016llx\n", backend.name, kernel.c_str(), count, + iterations, elapsed / iterations, elapsed / (static_cast(iterations) * count), + static_cast(hash)); + return hash; +} + +void RunBenchmarks(uint32_t iterations) { + std::puts("backend,kernel,samples,iterations,ns_per_block,ns_per_sample,checksum"); + for (uint32_t count : {6u, 18u, 32u, 96u, 160u}) { + for (const char* kernel : {"mix", "scale", "accum", "store", "load"}) { + const uint64_t scalar = Benchmark(kScalar, kernel, count, iterations); + const uint64_t selected = Benchmark(kSelected, kernel, count, iterations); + Require(scalar == selected, "benchmark checksum differs"); + } + } +} + +} // namespace + +int main(int argc, char** argv) { + try { + if (argc >= 2 && std::string(argv[1]) == "--benchmark") { + uint32_t iterations = 200000; + if (argc == 3) { + const std::string argument = argv[2]; + const auto result = std::from_chars(argument.data(), argument.data() + argument.size(), iterations); + Require(result.ec == std::errc{} && result.ptr == argument.data() + argument.size() && + iterations > 0 && iterations <= 100000000, "invalid benchmark iterations"); + } + Require(argc <= 3, "usage: ax_mix_kernels_tests [--benchmark [iterations]]"); + RunBenchmarks(iterations); + } else { + Require(argc == 1, "usage: ax_mix_kernels_tests [--benchmark [iterations]]"); + g_random = 0x12345678u; + TestBackend(kScalar); + g_random = 0x12345678u; + TestBackend(kSelected); + std::printf("AX mix kernels (%s): passed\n", kSelected.name); + } + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/runtime/tests/gx_copy_tests.cpp b/runtime/tests/gx_copy_tests.cpp new file mode 100644 index 000000000..8a8e04ab9 --- /dev/null +++ b/runtime/tests/gx_copy_tests.cpp @@ -0,0 +1,245 @@ +#include "gx_test_common.hpp" +#include "../src/hle/gx/gx_internal.h" +#include "gx_sync_support/copy_observation.h" +#include "gx/readback.hpp" +#include "__gx.h" + +// Exercise the wrappers without registering guest ABI entry points in the host fixture. +#undef REGISTER_NATIVE_FUNCTION +#define REGISTER_NATIVE_FUNCTION(address, fn) +#include "../src/hle/gx/gx_copy.cpp" + +extern "C" void AuroraResetCopyProbesForTesting(); + +namespace aurora::gfx { +extern uint32_t g_testDrawCount; +namespace testing { +extern std::atomic resolvePassCount; +extern std::vector> readbackJobs; +extern std::vector readbackJobCopies; +std::vector copyObservations; + +void observe_copy(TextureHandle target, ClipRect rect, bool clearColor, bool clearAlpha, bool clearDepth, + GXTexFmt format, const ResolveOptions& options) { + copyObservations.push_back({std::move(target), rect, clearColor, clearAlpha, clearDepth, format, options, + g_testDrawCount}); +} +} // namespace testing +} // namespace aurora::gfx + +extern "C" { +const uint8_t kMkwGuestRegion = 0; +int g_gxFrameCount = 0; +} +TexCopyState g_texCopyState; + +namespace { +constexpr uint32_t kGuestBase = 0x80001000; +std::array g_guestBytes{}; +std::atomic g_drawDoneCallbacks{0}; +uint32_t g_frameJoins = 0; +uint32_t g_readyXfb = 0; +uint32_t g_presents = 0; +uint32_t g_overlayDraws = 0; + +void DrawDoneCallback() { ++g_drawDoneCallbacks; } +} // namespace + +void* GuestToHostPtr(uint32_t address, size_t length) { + const uint32_t canonical = CanonicalizeGxMainRamAddress(address); + const uint32_t base = CanonicalizeGxMainRamAddress(kGuestBase); + if (canonical < base || canonical - base >= g_guestBytes.size() || + length > g_guestBytes.size() - (canonical - base)) { + return nullptr; + } + return g_guestBytes.data() + canonical - base; +} +void EnsureAuroraFrameActive() {} +void VI_HLE_SetXfbReady(uint32_t address) { g_readyXfb = address; } +void VI_HLE_PresentFrame(bool presentedXfb, bool paceToRetrace) { + EXPECT_TRUE(presentedXfb); + EXPECT_TRUE(paceToRetrace); + GxWorker::Sync(); + aurora::gx::fifo::drain(); + ++g_presents; +} +extern "C" void aurora_wait_for_frame_worker() { ++g_frameJoins; } +namespace settings_overlay { +void Draw() noexcept { ++g_overlayDraws; } +} // namespace settings_overlay + +namespace { +namespace stub = aurora::gfx::testing; + +class GxCopyTest : public GXFifoTest, public ::testing::WithParamInterface { +protected: + void SetUp() override { + aurora::gx::fifo::init(); + GXFifoTest::SetUp(); + AuroraResetCopyProbesForTesting(); + AuroraSetGxProducerThread(); + g_texCopyState = {}; + g_gxFrameCount = 0; + g_drawDoneCallbacks = 0; + g_frameJoins = g_readyXfb = g_presents = g_overlayDraws = 0; + g_efbCopyDestinations.clear(); + g_efbCopyBuckets.clear(); + g_largestEfbCopyDestination = 0; + stub::copyObservations.clear(); + stub::resolvePassCount = 0; + stub::readbackJobs.clear(); + stub::readbackJobCopies.clear(); + aurora::gfx::g_testDrawCount = 0; + aurora::gfx::readback::set_supported(true); + GXSetDrawDoneCallback(DrawDoneCallback); + aurora::gx::fifo::begin_frame(); + if (GetParam()) { + GxWorker::Start(); + } + GxWorker::Run([] { + GXSetColorUpdate(GX_TRUE); + GXSetAlphaUpdate(GX_TRUE); + GXSetZMode(GX_TRUE, GX_ALWAYS, GX_TRUE); + GXSetPixelFmt(GX_PF_RGBA6_Z24, GX_ZC_LINEAR); + }); + } + void TearDown() override { + GxWorker::Stop(); + aurora::gx::fifo::drain(); + aurora::gx::fifo::end_frame(); + GXSetDrawDoneCallback(nullptr); + AuroraResetCopyProbesForTesting(); + aurora::gfx::readback::shutdown(); + aurora::gfx::readback::set_supported(false); + GXFifoTest::TearDown(); + } + void Drain() { + GxWorker::Sync(); + AuroraGXSync(); + } + void Draw() { + GxWorker::Run([] { + GXClearVtxDesc(); + GXSetVtxDesc(GX_VA_POS, GX_DIRECT); + GXSetVtxAttrFmt(GX_VTXFMT0, GX_VA_POS, GX_POS_XYZ, GX_U8, 0); + GXBegin(GX_TRIANGLES, GX_VTXFMT0, 3); + GXPosition3u8(0, 1, 2); + GXPosition3u8(3, 4, 5); + GXPosition3u8(6, 7, 8); + GXEnd(); + }); + } + void Copy(uint32_t address, GXBool clear, GXTexFmt format = GX_TF_RGBA8, u16 width = 8, u16 height = 8) { + GX__SetTexCopySrc_8016f478(2, 4, width, height); + GX__SetTexCopyDst_8016f4dc(width, height, format, GX_FALSE); + GX__CopyTex_8016fd74(address, clear); + } +}; + +TEST_P(GxCopyTest, InterleavedDrawsClearsAndReusedDestinationsNeedNoCompletionTokens) { + Draw(); + Copy(kGuestBase, GX_TRUE); + Draw(); + Copy(kGuestBase + 512, GX_FALSE); + Copy(kGuestBase, GX_FALSE); + Drain(); + ASSERT_EQ(stub::copyObservations.size(), 3u); + EXPECT_EQ(stub::copyObservations[0].draws, 1u); + EXPECT_EQ(stub::copyObservations[1].draws, 2u); + EXPECT_EQ(stub::copyObservations[2].draws, 2u); + const auto& first = stub::copyObservations[0]; + EXPECT_TRUE(first.clearColor); + EXPECT_TRUE(first.clearAlpha); + EXPECT_TRUE(first.clearDepth); + EXPECT_EQ(first.rect.x, 2); + EXPECT_EQ(first.rect.y, 4); + EXPECT_EQ(first.rect.width, 8); + EXPECT_EQ(first.rect.height, 8); + EXPECT_FALSE(stub::copyObservations[1].clearColor); + EXPECT_EQ(first.target, stub::copyObservations[2].target); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); + EXPECT_EQ(g_frameJoins, 0u); + EXPECT_EQ(g_efbCopyDestinations.size(), 2u); + GXDrawDone(); + EXPECT_EQ(g_drawDoneCallbacks.load(), 1u); +} + +TEST_P(GxCopyTest, ReadbackIsRecordedAfterItsCopyWithoutAnImplicitFence) { + Draw(); + Copy(kGuestBase, GX_FALSE); + const auto stream = AuroraReadbackCreateStream(); + ASSERT_NE(stream, 0u); + const AuroraReadbackCopySrc source{.key = 7, .src = GuestToHostPtr(kGuestBase)}; + ASSERT_TRUE(AuroraReadbackCaptureCopies(stream, &source, 1)); + Drain(); + ASSERT_EQ(stub::readbackJobs.size(), 1u); + ASSERT_EQ(stub::readbackJobCopies.size(), 1u); + ASSERT_EQ(stub::copyObservations.size(), 1u); + EXPECT_EQ(stub::readbackJobCopies[0], 1u); + ASSERT_EQ(stub::readbackJobs[0]->copies.size(), 1u); + EXPECT_EQ(stub::readbackJobs[0]->copies[0].texture, stub::copyObservations[0].target); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); + AuroraReadbackDestroyStream(stream); +} + +TEST_P(GxCopyTest, CopyStillFlushesUnpublishedDirtyState) { + GxWorker::Run([] { + GXClearVtxDesc(); + GXSetVtxDesc(GX_VA_POS, GX_DIRECT); + }); + GxWorker::Sync(); + ASSERT_NE(__gx->dirtyState, 0u); + Copy(kGuestBase, GX_FALSE); + EXPECT_EQ(__gx->dirtyState, 0u); + Drain(); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); +} + +TEST_P(GxCopyTest, InvalidationRetiresTheQueuedCopyBeforeLaterConsumers) { + Copy(kGuestBase, GX_FALSE); + Copy(kGuestBase + 512, GX_FALSE); + InvalidateEfbCopyDestinationsForRange(kGuestBase, 256); + Drain(); + EXPECT_FALSE(aurora::gx::g_gxState.copyTextures.contains(GuestToHostPtr(kGuestBase))); + EXPECT_TRUE(aurora::gx::g_gxState.copyTextures.contains(GuestToHostPtr(kGuestBase + 512))); + EXPECT_EQ(g_efbCopyDestinations.size(), 1u); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); +} + +TEST_P(GxCopyTest, PersistentCopyPolicyAndDepthProbesSurviveWithoutTheWait) { + Copy(kGuestBase, GX_FALSE); + Drain(); + ASSERT_EQ(stub::copyObservations.size(), 1u); + EXPECT_TRUE(stub::copyObservations.back().options.persistent); + // The renderer stub retains this field, allowing the real copy policy to see a warm destination. + aurora::gx::g_gxState.copyTextures[GuestToHostPtr(kGuestBase)].consecutiveFrames = 8; + Copy(kGuestBase, GX_FALSE, GX_TF_RGBA8, 640, 480); + Drain(); + EXPECT_FALSE(stub::copyObservations.back().options.persistent); + const auto probe = kGuestBase + (GetParam() ? 2048u : 3072u); + Copy(probe, GX_FALSE, GX_TF_Z8, 4, 4); + Drain(); + EXPECT_FALSE(stub::copyObservations.back().options.persistent); + EXPECT_FALSE(stub::readbackJobs.empty()); + EXPECT_EQ(g_guestBytes[probe - kGuestBase], 0xff); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); +} + +TEST_P(GxCopyTest, DisplayCopyRetainsPresentationAndInlineRenderWorkerJoin) { + Draw(); + GxWorker::Run([] { + GXSetDispCopySrc(0, 0, 640, 480); + GXSetDispCopyDst(640, 480); + }); + GX__CopyDisp_8016fc38(kGuestBase, GX_TRUE); + EXPECT_EQ(stub::copyObservations.size(), 1u); + EXPECT_EQ(g_readyXfb, kGuestBase); + EXPECT_EQ(g_presents, 1u); + EXPECT_EQ(g_overlayDraws, 1u); + EXPECT_EQ(g_gxFrameCount, 1); + EXPECT_EQ(g_frameJoins, GetParam() ? 0u : 1u); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); +} + +INSTANTIATE_TEST_SUITE_P(WorkerDisabledAndEnabled, GxCopyTest, ::testing::Bool()); +} // namespace diff --git a/runtime/tests/gx_direct_draw_tests.cpp b/runtime/tests/gx_direct_draw_tests.cpp new file mode 100644 index 000000000..6ddd64f41 --- /dev/null +++ b/runtime/tests/gx_direct_draw_tests.cpp @@ -0,0 +1,280 @@ +#include "gx_test_common.hpp" +#include "../src/hle/gx/gx_internal.h" +#include "../src/hle/gx/gx_stream_common.h" +#include "isa/big_endian.h" + +#include +#include + +namespace { +class RuntimeDirectDrawTest : public GXFifoTest { +protected: + void SetUp() override { + GXFifoTest::SetUp(); + reset_runtime(); + } + + void reset_runtime() { + GXInit(nullptr, 0); + aurora::gx::fifo::clear_buffer(); + g_hleGxState = {}; + g_dlRecordState = {}; + g_alphaCompareValid = false; + GxWorker::detail::g_active = false; + GxWorker::detail::t_onWorker = false; + } + + void position(GXCompCnt cnt = GX_POS_XYZ, GXCompType type = GX_U8, u8 frac = 0) { + g_hleGxState.vtxDesc[GX_VA_POS] = GX_DIRECT; + g_hleGxState.vtxAttrFmt[GX_VTXFMT0][GX_VA_POS] = {cnt, type, frac}; + g_hleGxState.InvalidateVtxLayoutHash(); + } + + void begin(u16 count, bool direct = true) { + auto& g = g_hleGxState; + GxStream::PublishAuroraVtxState(GX_VTXFMT0, {false, GX_VA_POS, GX_VA_TEX7}); + g.currentVtxFmt = GX_VTXFMT0; + g.currentPrim = GX_TRIANGLES; + g.vertsRemaining = count; + g.inBegin = count != 0; + g.auroraBeginCalled = false; + g.fifoReadOffset = 0; + g.fifoByteCount = 0; + g.fifoPendingDrawBytes = 0; + g.ResetVertex(); + GxStream::PrepareImmediateDirectDraw(); + if (!direct) { + g.fifoDirectBytesRemaining = 0; + } + } + + void write(const std::vector& bytes, u32 width) { + size_t pos = 0; + for (; pos + width <= bytes.size(); pos += width) { + const u32 value = width == 4 ? BigEndian::Read32(bytes.data() + pos) + : width == 2 ? BigEndian::Read16(bytes.data() + pos) : bytes[pos]; + HleFifoWrite(value, width); + } + for (; pos < bytes.size(); ++pos) { + HleFifoWrite(bytes[pos], 1); + } + } +}; + +TEST_F(RuntimeDirectDrawTest, DirectNumericPayloadMatchesIncrementalForEveryWriteWidth) { + for (GXCompType type : {GX_U8, GX_S8, GX_U16, GX_S16, GX_F32}) { + for (GXCompCnt cnt : {GX_POS_XY, GX_POS_XYZ}) { + for (u32 width : {1u, 2u, 4u}) { + const u32 bytesPerVertex = (cnt == GX_POS_XY ? 2u : 3u) * GetCompSizeBytes(type); + std::vector payload(bytesPerVertex * 3); + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast(i * 17u + 11u); + } + reset_runtime(); + position(cnt, type, 7); + begin(3, false); + write(payload, width); + const auto reference = capture_fifo(); + reset_runtime(); + position(cnt, type, 7); + begin(3); + ASSERT_EQ(g_hleGxState.fifoDirectBytesRemaining, payload.size()); + write(payload, width); + EXPECT_EQ(capture_fifo(), reference) << "type=" << type << " count=" << cnt << " width=" << width; + EXPECT_FALSE(g_hleGxState.inBegin); + EXPECT_EQ(g_hleGxState.vertsRemaining, 0u); + } + } + } +} + +TEST_F(RuntimeDirectDrawTest, MixedMatrixColorsAndAllTextureCoordinatesMatchIncremental) { + std::vector reference; + for (bool direct : {false, true}) { + reset_runtime(); + position(GX_POS_XY, GX_S16, 11); + auto& g = g_hleGxState; + g.vtxDesc[GX_VA_PNMTXIDX] = GX_DIRECT; + g.vtxDesc[GX_VA_TEX7MTXIDX] = GX_DIRECT; + g.vtxDesc[GX_VA_CLR0] = GX_DIRECT; + g.vtxAttrFmt[0][GX_VA_CLR0] = {GX_CLR_RGB, GX_RGB8, 0}; + g.vtxDesc[GX_VA_CLR1] = GX_DIRECT; + g.vtxAttrFmt[0][GX_VA_CLR1] = {GX_CLR_RGBA, GX_RGBA8, 0}; + for (int tex = GX_VA_TEX0; tex <= GX_VA_TEX7; ++tex) { + g.vtxDesc[tex] = GX_DIRECT; + g.vtxAttrFmt[0][tex] = {GX_TEX_ST, GX_S16, static_cast(tex)}; + } + begin(3, direct); + ASSERT_EQ(g.fifoDirectBytesRemaining, direct ? 45u * 3u : 0u); + const std::vector payload(45u * 3u, 0x12); + GX_HLE_FIFO_WriteBurst(payload.data(), static_cast(payload.size())); + if (!direct) { + reference = capture_fifo(); + } else { + EXPECT_EQ(capture_fifo(), reference); + } + } +} + +TEST_F(RuntimeDirectDrawTest, NormalAliasesIndexedAndReencodedColorsKeepFallback) { + for (GXCompCnt cnt : {GX_NRM_XYZ, GX_NRM_NBT, GX_NRM_NBT3}) { + reset_runtime(); + position(); + g_hleGxState.vtxDesc[GX_VA_NRM] = GX_DIRECT; + g_hleGxState.vtxAttrFmt[0][GX_VA_NRM] = {cnt, GX_S16, 14}; + begin(3); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + } + for (GXCompType type : {GX_RGB565, GX_RGBX8, GX_RGBA4, GX_RGBA6}) { + reset_runtime(); + position(); + g_hleGxState.vtxDesc[GX_VA_CLR0] = GX_DIRECT; + g_hleGxState.vtxAttrFmt[0][GX_VA_CLR0] = {GX_CLR_RGBA, type, 0}; + begin(3); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + } + for (GXAttrType type : {GX_INDEX8, GX_INDEX16}) { + reset_runtime(); + position(); + g_hleGxState.vtxDesc[GX_VA_POS] = type; + begin(3); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + } +} + +TEST_F(RuntimeDirectDrawTest, CompletingWordAndBurstTailAreParsedImmediately) { + for (bool burst : {false, true}) { + reset_runtime(); + position(); + begin(1); + const std::vector stream{0x11, 0x22, 0x33, 0x48}; + if (burst) { + GX_HLE_FIFO_WriteBurst(stream.data(), static_cast(stream.size())); + } else { + HleFifoWrite(0x11223348, 4); + } + const auto bytes = capture_fifo(); + ASSERT_FALSE(bytes.empty()); + EXPECT_EQ(bytes.back(), 0x48); + EXPECT_EQ(g_hleGxState.fifoByteCount, 0u); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + } +} + +TEST_F(RuntimeDirectDrawTest, RawPacketsReserveCompletingWordTailAtBufferBoundary) { + for (u16 packetBytes : {4093, 4094, 4095, 4096}) { + reset_runtime(); + g_hleGxState.vtxDesc[GX_VA_PNMTXIDX] = GX_DIRECT; + const u16 count = packetBytes - 3; + HleFifoWrite(0x90, 1); + HleFifoWrite(count, 2); + EXPECT_EQ(g_hleGxState.fifoPendingDrawBytes, packetBytes == 4093 ? packetBytes : 0u); + for (u32 i = 0; i < count - 1u; ++i) { + HleFifoWrite(0x12, 1); + } + HleFifoWrite(0x12480000, 4); + const auto bytes = capture_fifo(); + ASSERT_GE(bytes.size(), count + 4u); + const auto drawStart = bytes.size() - count - 4u; + EXPECT_EQ(bytes[drawStart], 0x90); + EXPECT_EQ(BigEndian::Read16(bytes.data() + drawStart + 1u), count); + for (size_t i = drawStart + 3u; i < drawStart + count + 3u; ++i) { + ASSERT_EQ(bytes[i], 0x12) << "packet=" << packetBytes; + } + EXPECT_EQ(bytes.back(), 0x48); + EXPECT_EQ(g_hleGxState.fifoByteCount, 0u); + EXPECT_EQ(g_hleGxState.fifoPendingDrawBytes, 0u); + EXPECT_FALSE(g_hleGxState.inBegin); + } +} + +TEST_F(RuntimeDirectDrawTest, PendingRawLengthIsRecomputedAfterLayoutInvalidation) { + position(); + HleFifoWrite(0x90, 1); + HleFifoWrite(1, 2); + ASSERT_EQ(g_hleGxState.fifoPendingDrawBytes, 6u); + position(GX_POS_XY); + HleFifoWrite(0x1122, 2); + EXPECT_EQ(g_hleGxState.fifoPendingDrawBytes, 0u); + EXPECT_EQ(g_hleGxState.fifoByteCount, 0u); + EXPECT_FALSE(g_hleGxState.inBegin); +} + +TEST_F(RuntimeDirectDrawTest, CompletingWordCanStartPartialCpPacket) { + position(GX_POS_XY); + begin(1); + HleFifoWrite(0x11220850, 4); + EXPECT_FALSE(g_hleGxState.inBegin); + EXPECT_EQ(g_hleGxState.fifoByteCount, 2u); + HleFifoWrite(3u << 9, 4); + EXPECT_EQ(g_hleGxState.vtxDesc[GX_VA_POS], GX_INDEX16); + EXPECT_EQ(g_hleGxState.fifoByteCount, 0u); +} + +TEST_F(RuntimeDirectDrawTest, LayoutChangeBeforeFirstPayloadRechecksDirectEligibility) { + position(); + begin(1); + g_hleGxState.vtxDesc[GX_VA_NRM] = GX_DIRECT; + g_hleGxState.vtxAttrFmt[0][GX_VA_NRM] = {GX_NRM_XYZ, GX_S16, 14}; + g_hleGxState.InvalidateVtxLayoutHash(); + HleFifoWrite(0x11, 1); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + EXPECT_TRUE(g_hleGxState.auroraBeginCalled); +} + +TEST_F(RuntimeDirectDrawTest, DirectPayloadLargerThanQueueStillMatchesIncremental) { + std::vector reference; + const std::vector payload(4096u * 3u, 0x32); + for (bool direct : {false, true}) { + reset_runtime(); + position(); + begin(4096, direct); + GX_HLE_FIFO_WriteBurst(payload.data(), static_cast(payload.size())); + EXPECT_FALSE(g_hleGxState.inBegin); + EXPECT_EQ(g_hleGxState.fifoByteCount, 0u); + if (!direct) { + reference = capture_fifo(); + } else { + EXPECT_EQ(capture_fifo(), reference); + } + } +} + +TEST_F(RuntimeDirectDrawTest, RecordingBypassesPreparedDirectPayload) { + position(); + begin(1); + g_dlRecordState = {0x80001000, 32, 0x80001000, 0, true}; + HleFifoWrite(0x11223344, 4); + EXPECT_EQ(g_dlRecordState.count, 4u); + EXPECT_EQ(Memory::Read32(0x80001000), 0x11223344u); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 3u); + EXPECT_TRUE(capture_fifo().empty()); + g_dlRecordState.active = false; +} + +TEST_F(RuntimeDirectDrawTest, ZeroCountDoesNotPrepareDirectPayload) { + position(); + begin(0); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + EXPECT_FALSE(g_hleGxState.inBegin); +} + +TEST_F(RuntimeDirectDrawTest, DISABLED_SubmissionBenchmark) { + constexpr u32 draws = 20000; + const std::vector payload(3u * 4u, 0x32); + for (bool direct : {false, true, true, false}) { + reset_runtime(); + position(); + const auto start = std::chrono::steady_clock::now(); + for (u32 i = 0; i < draws; ++i) { + begin(4, direct); + write(payload, 4); + } + const auto end = std::chrono::steady_clock::now(); + const auto bytes = capture_fifo(); + std::printf("direct=%d draws=%u elapsed_us=%lld fifo_bytes=%zu\n", direct, draws, + static_cast(std::chrono::duration_cast(end - start).count()), + bytes.size()); + } +} +} // namespace diff --git a/runtime/tests/gx_dl_cache_tests.cpp b/runtime/tests/gx_dl_cache_tests.cpp new file mode 100644 index 000000000..585eb0e90 --- /dev/null +++ b/runtime/tests/gx_dl_cache_tests.cpp @@ -0,0 +1,301 @@ +#include "../src/hle/gx/gx_dl_cache.h" +#include "../src/hle/gx/gx_internal.h" + +#include + +#include +#include +#include +#include +#include + +extern "C" const uint8_t kMkwGuestRegion = 0; + +namespace { + +void Require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +struct Record { + uint32_t listAddr = 0; + uint32_t nbytes = 0; + uint64_t layoutHash = 0; + uint64_t contentDigest = 0; + uint64_t writeGeneration = GxGuestWrite::kUntracked; + std::vector cpWrites; + std::vector flattened; + + size_t Bytes() const { return flattened.size() + cpWrites.size() * sizeof(uint64_t); } +}; + +using Cache = GxDisplayListScanCache::BoundedCache; + +bool Store(Cache& cache, uint64_t key, size_t flattenedBytes, size_t cpWrites = 0) { + Record record{}; + record.cpWrites.resize(cpWrites, key); + record.flattened.resize(flattenedBytes, static_cast(key)); + const size_t bytes = record.Bytes(); + return cache.Insert(key, std::move(record), bytes); +} + +void EntryBudgetAndRecency() { + Cache cache{3, 1024}; + for (uint64_t key = 1; key <= 3; ++key) Require(Store(cache, key, 0), "initial insert failed"); + auto* hot = cache.Find(1); + cache.Touch(1); + Require(Store(cache, 4, 0), "entry-budget insert failed"); + Require(cache.Size() == 3 && cache.StoredBytes() == 0, "entry budget exceeded"); + Require(cache.Find(1) == hot && cache.Find(2) == nullptr && cache.Find(3) && cache.Find(4), + "entry-budget eviction discarded the hot working set"); + cache.Find(3); + cache.Touch(999); + Require(Store(cache, 5, 0), "second entry-budget insert failed"); + Require(!cache.Find(3) && cache.Find(1), "unvalidated probe refreshed recency"); +} + +void ByteBudgetAndReplacement() { + Cache cache{3, 12}; + Require(Store(cache, 1, 4) && Store(cache, 2, 4) && Store(cache, 3, 4), "byte-budget setup failed"); + auto* retained = cache.Find(3); + Require(Store(cache, 1, 8), "protected replacement failed"); + Require(cache.Size() == 2 && cache.StoredBytes() == 12, "replacement byte accounting failed"); + Require(cache.Find(1) && !cache.Find(2) && cache.Find(3) == retained, + "oldest replacement was evicted or unrelated record moved"); + Require(cache.Find(1)->flattened == std::vector(8, 1), "replacement payload was lost"); + Require(Store(cache, 4, 4), "byte-budget eviction failed"); + Require(cache.Size() == 2 && cache.StoredBytes() == 12 && !cache.Find(3) && cache.Find(1), + "replacement was not promoted to most recent"); + Require(Store(cache, 1, 0), "shrinking replacement failed"); + Require(cache.Size() == 2 && cache.StoredBytes() == 4, "shrinking replacement leaked bytes"); + Require(Store(cache, 5, 0, 1), "CP-write insertion failed"); + Require(cache.Size() == 3 && cache.StoredBytes() == 12, "CP-write bytes were not counted"); + Require(Store(cache, 5, 4), "CP-write replacement failed"); + Require(cache.StoredBytes() == 8, "CP-write replacement counted old bytes twice"); +} + +void RejectedInsertsPreserveRecords() { + Cache cache{2, 12}; + Require(Store(cache, 1, 8) && Store(cache, 2, 4), "oversize setup failed"); + const auto* old = cache.Find(1); + Require(!Store(cache, 3, 13) && !Store(cache, 1, 13), "oversized record was admitted"); + Require(!cache.Insert(1, Record{}, std::numeric_limits::max()), "overflowing size was admitted"); + Require(cache.Size() == 2 && cache.StoredBytes() == 12 && cache.Find(1) == old && cache.Find(2), + "rejected insertion changed the working set"); + Require(Store(cache, 3, 4) && !cache.Find(1) && cache.Find(2), "rejected insertion changed recency"); + + Cache disabled{0, 12}; + Require(!Store(disabled, 1, 0) && disabled.Size() == 0, "disabled entry budget accepted a record"); + Cache noBytes{1, 0}; + Require(Store(noBytes, 1, 0) && !Store(noBytes, 1, 1) && noBytes.Find(1), + "zero byte budget mishandled an empty record"); +} + +void EvictionLimitAndProtection() { + Cache cache{256, 128}; + for (uint64_t key = 1; key <= 128; ++key) Require(Store(cache, key, 1), "bounded setup failed"); + const auto* protectedRecord = cache.Find(1); + Require(Store(cache, 1, 65), "replacement needing exactly 64 victims failed"); + Require(cache.Size() == 64 && cache.StoredBytes() == 128 && cache.Find(1) == protectedRecord, + "protected replacement was counted as a victim"); + for (uint64_t key = 2; key <= 65; ++key) Require(!cache.Find(key), "wrong LRU victim selected"); + for (uint64_t key = 66; key <= 128; ++key) Require(cache.Find(key), "too many records evicted"); + + cache.Clear(); + for (uint64_t key = 1; key <= 128; ++key) Require(Store(cache, key, 1), "bounded refill failed"); + Require(Store(cache, 129, 64), "new insertion needing exactly 64 victims failed"); + Require(cache.Size() == 65 && cache.StoredBytes() == 128 && !cache.Find(64) && cache.Find(65), + "new insertion evicted the wrong count"); +} + +void BoundedRejectionsMakeProgress() { + Cache cache{256, 128}; + for (uint64_t key = 1; key <= 128; ++key) Require(Store(cache, key, 1), "rejection setup failed"); + Require(!Store(cache, 129, 65), "new insertion exceeded 64 victims"); + Require(cache.Size() == 64 && cache.StoredBytes() == 64 && !cache.Find(64) && cache.Find(65), + "rejected insertion did not stop at 64 victims"); + Require(Store(cache, 129, 65) && cache.Size() == 64 && cache.StoredBytes() == 128, + "rejected insertion did not make progress for the next attempt"); + + cache.Clear(); + for (uint64_t key = 1; key <= 128; ++key) Require(Store(cache, key, 1), "protected rejection setup failed"); + const auto* protectedRecord = cache.Find(1); + Require(!Store(cache, 1, 66), "replacement exceeded 64 victims"); + Require(cache.Size() == 64 && cache.StoredBytes() == 64 && cache.Find(1) == protectedRecord && + protectedRecord->flattened.size() == 1, "bounded rejection lost the protected replacement"); + Require(Store(cache, 1, 66) && cache.Size() == 63 && cache.StoredBytes() == 128, + "replacement did not make progress for the next attempt"); + + cache.Clear(); + for (uint64_t key = 1; key <= 64; ++key) Require(Store(cache, key, 0), "empty record setup failed"); + Require(Store(cache, 65, 128), "large record setup failed"); + Require(!Store(cache, 66, 1) && cache.Size() == 1 && cache.StoredBytes() == 128, + "zero-byte victims bypassed the eviction work limit"); + Require(Store(cache, 66, 1) && cache.Size() == 1 && cache.StoredBytes() == 1, + "zero-byte victims caused repeated insertion starvation"); +} + +void ClearAndPointerLifetime() { + Cache cache{8192, 8192}; + Require(Store(cache, 0, 1), "pointer setup failed"); + const auto* first = cache.Find(0); + const auto* payload = first->flattened.data(); + for (uint64_t key = 1; key < 8192; ++key) Require(Store(cache, key, 1), "growth insert failed"); + cache.Touch(0); + for (uint64_t key = 8192; key < 16384; ++key) { + Require(Store(cache, key, 1), "churn insert failed"); + Require(cache.Find(0) == first && first->flattened.data() == payload, + "rehash or unrelated eviction invalidated a retained stream"); + cache.Touch(0); + } + Require(cache.Size() == 8192 && cache.StoredBytes() == 8192, "churn exceeded a budget"); + cache.Clear(); + cache.Clear(); + Require(cache.Size() == 0 && cache.StoredBytes() == 0 && !cache.Find(0), "clear left stale accounting"); + Require(Store(cache, 0, 0), "insertion after clear failed"); +} + +void LiveBytesStillInvalidateRetainedRecords() { + constexpr uint32_t address = 0x80010000; + constexpr uint64_t layout = 0x12345678; + std::array bytes{0x08, 0x50, 0, 0, 0, 1, 0, 0}; + const auto digest = [&] { return static_cast(XXH3_64bits(bytes.data(), bytes.size())); }; + const auto key = GxDisplayListScanCache::MakeIdentityKey(address, bytes.size(), layout); + Require(key == GxDisplayListScanCache::MakeIdentityKey(0xC0010000, bytes.size(), layout), + "cached and uncached aliases use different keys"); + Cache cache{2, 32}; + Record record{}; + record.listAddr = CanonicalizeGxMainRamAddress(address); + record.nbytes = bytes.size(); + record.layoutHash = layout; + record.contentDigest = digest(); + record.writeGeneration = GxGuestWrite::GenerationForRange(address, bytes.size()); + Require(cache.Insert(key, std::move(record), 0), "validation setup failed"); + auto* retained = cache.Find(key); + const auto reusable = [&](uint32_t addr, uint32_t count, uint64_t currentLayout, uint64_t currentDigest) { + return GxDisplayListScanCache::CanReuse(retained->listAddr, retained->nbytes, retained->layoutHash, + retained->contentDigest, addr, count, currentLayout, + currentDigest); + }; + Require(reusable(0xC0010000, bytes.size(), layout, digest()), "unchanged alias was rejected"); + Require(!reusable(address, bytes.size() - 1, layout, digest()) && + !reusable(address, bytes.size(), layout + 1, digest()) && + !reusable(address + 32, bytes.size(), layout, digest()), "identity mismatch was reused"); + GxGuestWrite::NotifyWrite(0xC0010000, bytes.size()); + const auto generation = GxGuestWrite::GenerationForRange(address, bytes.size()); + Require(!GxGuestWrite::CanSkipDigest(retained->writeGeneration, generation), "write failed to invalidate digest"); + Require(reusable(address, bytes.size(), layout, digest()), "unchanged bytes after a write were rejected"); + bytes[5] ^= 0x80; + Require(!reusable(address, bytes.size(), layout, digest()), "changed command bytes reused a retained record"); + Require(!GxGuestWrite::CanSkipDigest(GxGuestWrite::kUntracked, GxGuestWrite::kUntracked), + "untracked writes bypassed digest validation"); + Require(Store(cache, 1, 16), "validation churn setup failed"); + cache.Touch(key); + Require(Store(cache, 2, 16) && cache.Find(key) == retained && !cache.Find(1), + "unrelated eviction discarded the retained record"); + Require(!reusable(address, bytes.size(), layout, digest()), "churn made a stale digest reusable"); + Record replacement = *retained; + replacement.contentDigest = digest(); + replacement.writeGeneration = generation; + Require(cache.Insert(key, std::move(replacement), 0) && cache.StoredBytes() == 16, + "invalidated replacement changed unrelated byte accounting"); + Require(reusable(address, bytes.size(), layout, digest()), "replacement did not refresh the digest"); +} + +struct LegacyCache { + std::unordered_map entries; + size_t storedBytes = 0; + size_t maxEntries; + size_t maxBytes; + + Record* Find(uint64_t key) { + const auto it = entries.find(key); + return it == entries.end() ? nullptr : &it->second; + } + void Touch(uint64_t) {} + size_t Size() const { return entries.size(); } + bool Insert(uint64_t key, Record&& value, size_t bytes) { + const auto it = entries.find(key); + const bool replacing = it != entries.end(); + size_t replacedBytes = replacing ? it->second.Bytes() : 0; + if ((!replacing && entries.size() >= maxEntries) || storedBytes - replacedBytes + bytes > maxBytes) { + entries.clear(); + storedBytes = 0; + replacedBytes = 0; + } + entries.insert_or_assign(key, std::move(value)); + storedBytes = storedBytes - replacedBytes + bytes; + return true; + } +}; + +template +void Benchmark(Storage& cache, std::string_view policy, std::string_view scenario, + size_t hotCount, size_t coldCount, size_t bytesPerEntry) { + using Clock = std::chrono::steady_clock; + uint64_t hits = 0, misses = 0, evictions = 0, skips = 0; + std::vector batches; + const auto access = [&](uint64_t key) { + if (cache.Find(key)) { + ++hits; + cache.Touch(key); + } else { + ++misses; + Record record{}; + record.flattened.resize(bytesPerEntry); + const auto before = cache.Size(); + const bool inserted = cache.Insert(key, std::move(record), bytesPerEntry); + skips += !inserted; + evictions += before + (inserted ? 1u : 0u) - cache.Size(); + } + }; + for (uint64_t key = 0; key < hotCount; ++key) access(key); + for (uint64_t round = 0; round < 256; ++round) { + const auto begin = Clock::now(); + for (uint64_t key = 0; key < hotCount; ++key) access(key); + for (uint64_t key = 0; key < coldCount; ++key) access(hotCount + round * coldCount + key); + batches.push_back(std::chrono::duration(Clock::now() - begin).count()); + } + std::sort(batches.begin(), batches.end()); + const auto percentile = [&](size_t percent) { return batches[(batches.size() - 1) * percent / 100]; }; + std::cout << policy << ',' << scenario << ',' << hits << ',' << misses << ',' << evictions << ',' << skips + << ',' << percentile(50) << ',' << percentile(95) << ',' << percentile(99) << '\n'; +} + +void Benchmarks() { + constexpr size_t maxBytes = 8u * 1024u * 1024u; + std::cout << "policy,scenario,hits,misses,evictions,skips,batch_p50_us,batch_p95_us,batch_p99_us\n"; + for (const bool bytePressure : {false, true}) { + const size_t hotCount = bytePressure ? 256 : 4096; + const size_t coldCount = bytePressure ? 64 : 1024; + const size_t bytesPerEntry = bytePressure ? 16384 : 64; + const auto scenario = bytePressure ? "bytes" : "entries"; + Cache cache{8192, maxBytes}; + Benchmark(cache, "incremental", scenario, hotCount, coldCount, bytesPerEntry); + LegacyCache legacy{{}, 0, 8192, maxBytes}; + Benchmark(legacy, "whole-cache", scenario, hotCount, coldCount, bytesPerEntry); + } +} + +} // namespace + +int main(int argc, char** argv) { + try { + if (argc == 2 && std::string_view(argv[1]) == "--benchmark") { + Benchmarks(); + } else { + EntryBudgetAndRecency(); + ByteBudgetAndReplacement(); + RejectedInsertsPreserveRecords(); + EvictionLimitAndProtection(); + BoundedRejectionsMakeProgress(); + ClearAndPointerLifetime(); + LiveBytesStillInvalidateRetainedRecords(); + std::cout << "display-list scan cache tests passed\n"; + } + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/runtime/tests/gx_sync_support/copy_observation.h b/runtime/tests/gx_sync_support/copy_observation.h new file mode 100644 index 000000000..c54d56531 --- /dev/null +++ b/runtime/tests/gx_sync_support/copy_observation.h @@ -0,0 +1,22 @@ +#pragma once + +#include "gfx/recording.hpp" + +#include + +namespace aurora::gfx::testing { +struct CopyObservation { + TextureHandle target; + ClipRect rect; + bool clearColor; + bool clearAlpha; + bool clearDepth; + GXTexFmt format; + ResolveOptions options; + uint32_t draws; +}; + +extern std::vector copyObservations; +void observe_copy(TextureHandle target, ClipRect rect, bool clearColor, bool clearAlpha, bool clearDepth, + GXTexFmt format, const ResolveOptions& options); +} // namespace aurora::gfx::testing diff --git a/runtime/tests/gx_texture_binding_support/runtime_seams.h b/runtime/tests/gx_texture_binding_support/runtime_seams.h new file mode 100644 index 000000000..20a4d5e31 --- /dev/null +++ b/runtime/tests/gx_texture_binding_support/runtime_seams.h @@ -0,0 +1,100 @@ +#pragma once + +#include "../memory_slow_test_support.h" + +#include + +extern "C" const uint8_t kMkwGuestRegion = 0; + +namespace TextureBindingTest { +std::array memory{}; +uint32_t unreadableObject = 0; +uint64_t rangeChecks = 0; +std::vector> retiredRanges; + +uint8_t* Pointer(uint32_t address, size_t length) { + const uint32_t offset = CanonicalizeGxMainRamAddress(address); + if ((unreadableObject != 0 && offset >= unreadableObject && offset < unreadableObject + 32) || + offset > memory.size() || length > memory.size() - offset) { + throw Memory::AccessViolation(address, length, "test unmapped range"); + } + return memory.data() + offset; +} + +struct Command { + std::unique_ptr storage; + void (*run)(void*); +}; +std::deque commands; +} // namespace TextureBindingTest + +Memory::AccessViolation::AccessViolation(uint32_t address, size_t length, std::string_view reason) + : std::runtime_error(std::string(reason)), address_(address), length_(length), reason_(reason) {} + +bool Memory::Contains(uint32_t address, size_t length) { + ++TextureBindingTest::rangeChecks; + try { + TextureBindingTest::Pointer(address, length); + return true; + } catch (const Memory::AccessViolation&) { + return false; + } +} +uint8_t* Memory::GetPointer(uint32_t address) { return TextureBindingTest::Pointer(address, 1); } +uint8_t* Memory::GetPointer(uint32_t address, size_t length) { return TextureBindingTest::Pointer(address, length); } + +uint8_t MemoryInline::Read8Slow(uint32_t address) { return *TextureBindingTest::Pointer(address, 1); } +uint16_t MemoryInline::Read16Slow(uint32_t address) { return BigEndian::Read16(TextureBindingTest::Pointer(address, 2)); } +uint32_t MemoryInline::Read32Slow(uint32_t address) { return BigEndian::Read32(TextureBindingTest::Pointer(address, 4)); } +uint64_t MemoryInline::Read64Slow(uint32_t address) { + return (static_cast(Read32Slow(address)) << 32) | Read32Slow(address + 4); +} +void MemoryInline::Write8Slow(uint32_t address, uint8_t value) { *TextureBindingTest::Pointer(address, 1) = value; } +void MemoryInline::Write16Slow(uint32_t address, uint16_t value) { + BigEndian::Write16(TextureBindingTest::Pointer(address, 2), value); +} +void MemoryInline::Write32Slow(uint32_t address, uint32_t value) { + BigEndian::Write32(TextureBindingTest::Pointer(address, 4), value); +} +void MemoryInline::WriteFloat32Slow(uint32_t address, double value) { + const float narrowed = static_cast(value); + uint32_t bits; + std::memcpy(&bits, &narrowed, sizeof(bits)); + Write32Slow(address, bits); +} + +namespace GxWorker { +namespace detail { +std::atomic g_active{false}; +constinit thread_local bool t_onWorker = false; +constinit thread_local int t_inlineDepth = 0; +bool g_fifoPending = false; + +void* Reserve(size_t size, void (*run)(void*)) { + auto storage = std::make_unique(size); + void* result = storage.get(); + TextureBindingTest::commands.push_back({std::move(storage), run}); + return result; +} +void Commit() {} +void FlushFifo() { g_fifoPending = false; } +} // namespace detail + +void Sync() { + const bool wasWorker = detail::t_onWorker; + detail::t_onWorker = true; + while (!TextureBindingTest::commands.empty()) { + auto command = std::move(TextureBindingTest::commands.front()); + TextureBindingTest::commands.pop_front(); + command.run(command.storage.get()); + } + detail::t_onWorker = wasWorker; +} +void BeginInline() { Sync(); ++detail::t_inlineDepth; } +void EndInline() { --detail::t_inlineDepth; } +} // namespace GxWorker + +void InvalidateEfbCopyDestinationsForRange(uint32_t address, uint32_t size) { + TextureBindingTest::retiredRanges.emplace_back(address, size); +} +extern "C" void GxNotifyDisplayListMemoryWrite(uint32_t, uint32_t) {} diff --git a/runtime/tests/gx_texture_binding_tests.cpp b/runtime/tests/gx_texture_binding_tests.cpp new file mode 100644 index 000000000..1220fdab6 --- /dev/null +++ b/runtime/tests/gx_texture_binding_tests.cpp @@ -0,0 +1,335 @@ +#include "gx_test_common.hpp" +#include "../src/hle/gx/gx_internal.h" +#include "isa/big_endian.h" + +// Keep the production loaders and metadata writers, without a guest dispatch registry. +#undef PPC_NATIVE_OVERRIDE +#undef PPC_NATIVE_OVERRIDE_VOID +#define PPC_NATIVE_OVERRIDE(...) +#define PPC_NATIVE_OVERRIDE_VOID(...) +#include "../src/hle/gx/gx_utils.cpp" +#include "../src/hle/gx/gx_objects.cpp" +#include "../src/hle/gx/gx_texture.cpp" +#include "gx_texture_binding_support/runtime_seams.h" + +namespace { +constexpr uint32_t kObject = 0x80001000; +constexpr uint32_t kOtherObject = kObject + 32; +constexpr uint32_t kPixels = 0x80010000; +constexpr uint32_t kOtherPixels = kPixels + 4096; +constexpr uint32_t kGuestGxData = 0x80003000; + +class RuntimeTextureBindingTest : public GXFifoTest { +protected: + void SetUp() override { + GXFifoTest::SetUp(); + AuroraSetGxProducerThread(); + GxWorker::detail::g_active = false; + GxWorker::detail::t_inlineDepth = 0; + GxWorker::detail::t_onWorker = false; + TextureBindingTest::memory.fill(0); + TextureBindingTest::unreadableObject = 0; + TextureBindingTest::rangeChecks = 0; + TextureBindingTest::retiredRanges.clear(); + InvalidateTexObjLoads(); + // The production table never erases slots; reset its memo before fixture teardown. + g_texObjSlotMemo = nullptr; + g_texObjSlotMemoAddr = 0; + g_texObjIntervals.clear(); + g_texObjIntervalsPending.clear(); + g_texObjIntervalsRebuildAll = true; + g_TexObjMeta.clear(); + g_TlutObjMeta.clear(); + g_HostTlutObjMap.clear(); + g_tlutObjBoundsDirty = true; + Memory::Write32(kGXDataPtrAddr, kGuestGxData); + } + + void TearDown() override { + GxWorker::Sync(); + GxWorker::detail::g_active = false; + g_texObjSlotMemo = nullptr; + g_texObjSlotMemoAddr = 0; + g_texObjIntervals.clear(); + g_texObjIntervalsPending.clear(); + g_TexObjMeta.clear(); + g_HostTlutObjMap.clear(); + GXFifoTest::TearDown(); + } + + void Init(uint32_t object = kObject, uint32_t pixels = kPixels, uint32_t format = GX_TF_RGBA8) { + if (IsPaletteTexFormat(format)) { + GX__InitTexObjCI_80170a04(object, pixels, 8, 8, format, GX_REPEAT, GX_CLAMP, 0, 0); + } else { + GX__InitTexObj_801707f8(object, pixels, 8, 8, format, GX_REPEAT, GX_CLAMP, 0); + } + } + + void Load(uint32_t object = kObject, uint32_t map = 0) { GX__LoadTexObj_80170f2c(object, map); } + + void Warm(uint32_t object = kObject, uint32_t map = 0) { + Load(object, map); + GxWorker::Sync(); + EnsureTexObjIntervalsLocked(); + capture_fifo(); + } + + void ExpectFast(uint32_t object = kObject, uint32_t map = 0) { + const auto checks = TextureBindingTest::rangeChecks; + Load(object, map); + GxWorker::Sync(); + EXPECT_EQ(TextureBindingTest::rangeChecks, checks); + EXPECT_TRUE(g_texObjIntervalsPending.empty()); + } + + void ExpectFull(uint32_t object = kObject, uint32_t map = 0) { + const auto checks = TextureBindingTest::rangeChecks; + Load(object, map); + GxWorker::Sync(); + EXPECT_GT(TextureBindingTest::rangeChecks, checks); + } +}; + +TEST_F(RuntimeTextureBindingTest, RepeatedBindingSkipsMetadataAndExistingHostLoad) { + Init(); + Warm(); + ExpectFast(); + EXPECT_TRUE(capture_fifo().empty()); + EXPECT_EQ(Memory::Read32(kGuestGxData + 0x5FC) & 1u, 1u); + EXPECT_EQ(Memory::Read16(kGuestGxData + 2), 0); +} + +TEST_F(RuntimeTextureBindingTest, AlternatingObjectsAndMapsStillEmitRequiredBindings) { + Init(); + Init(kOtherObject, kOtherPixels); + Warm(); + Warm(kOtherObject, 1); + ExpectFast(kObject, 1); + auto bytes = capture_fifo(); + ASSERT_FALSE(bytes.empty()); + decode_fifo(bytes); + EXPECT_EQ(gxState().loadedTextures[1].data, GuestToHostPtr(kPixels)); + ExpectFast(kOtherObject, 0); + bytes = capture_fifo(); + ASSERT_FALSE(bytes.empty()); + decode_fifo(bytes); + EXPECT_EQ(gxState().loadedTextures[0].data, GuestToHostPtr(kOtherPixels)); +} + +TEST_F(RuntimeTextureBindingTest, BindOnlyAndFullPathsEmitIdenticalPackets) { + for (uint32_t format : {GX_TF_I4, GX_TF_RGBA8, GX_TF_CMPR, GX_TF_Z8, GX_TF_Z16, GX_TF_Z24X8}) { + Init(kObject, kPixels, format); + // Include values the full path has to sanitize before memoizing them. + Memory::Write32(kObject, Memory::Read32(kObject) | 3u | (3u << 19)); + Memory::Write32(kObject + 4, 0xFFFF); + Warm(); + g_boundTexMaps[2] = {}; + ExpectFast(kObject, 2); + const auto fast = capture_fifo(); + ASSERT_FALSE(fast.empty()); + g_boundTexMaps[2] = {}; + g_TexObjMeta.at(kObject).validatedLoad.Invalidate(); + ExpectFull(kObject, 2); + EXPECT_EQ(capture_fifo(), fast) << format; + } +} + +TEST_F(RuntimeTextureBindingTest, EverySnapshotByteParticipatesInValidation) { + Init(); + Warm(); + const auto original = CaptureTexObjBytes(kObject); + for (uint32_t offset = 0; offset < original.size(); ++offset) { + const uint8_t changed = original[offset] ^ 0x80; + Memory::Write8(kObject + offset, changed); + const auto snapshot = CaptureTexObjBytes(kObject); + TexObjSnapshotScope scope(kObject, snapshot); + EXPECT_EQ(TryGetValidatedTexObjSlot(kObject), nullptr) << offset; + Memory::Write8(kObject + offset, original[offset]); + // Re-arm through a real successful load, using the original bytes. + { + TexObjSnapshotScope restored(kObject, original); + LoadTexObjImpl(kObject, 0); + } + } +} + +TEST_F(RuntimeTextureBindingTest, GetterShadowCannotValidateAnotherSnapshot) { + Init(); + Warm(); + auto newer = CaptureTexObjBytes(kObject); + newer[0x1C] ^= 1; + Memory::Write8(kObject + 0x1C, newer[0x1C]); + GX__GetTexObjAll_80170bf8(kObject, 0, 0, 0, 0, 0, 0, 0); + // This getter refreshes the decode shadow without doing a full texture load. + ExpectFull(); + Warm(); + ExpectFast(); +} + +TEST_F(RuntimeTextureBindingTest, ProducerBacklogUsesExecutedBytes) { + Init(); + Warm(); + const auto oldBytes = CaptureTexObjBytes(kObject); + GxWorker::detail::g_active = true; + Load(kObject, 1); + Memory::Write32(kObject + 0x0C, CanonicalizeGxMainRamAddress(kOtherPixels) >> 5); + Load(kObject, 2); + // Producer has already moved on again before either queued load executes. + Memory::Write32(kObject + 0x0C, BigEndian::Read32(oldBytes.data() + 0x0C)); + GxWorker::Sync(); + auto bytes = capture_fifo(); + decode_fifo(bytes); + EXPECT_EQ(gxState().loadedTextures[1].data, GuestToHostPtr(kPixels)); + EXPECT_EQ(gxState().loadedTextures[2].data, GuestToHostPtr(kOtherPixels)); + EXPECT_EQ(g_TexObjMeta.at(kObject).dataAddr, CanonicalizeGxMainRamAddress(kOtherPixels)); + ExpectFull(kObject, 2); + Warm(kObject, 2); + ExpectFast(kObject, 2); +} + +TEST_F(RuntimeTextureBindingTest, MetadataAndHostWritersRequireFullLoad) { + Init(); + Warm(); + GX__InitTexObjLOD_80170a4c(kObject, GX_NEAR, GX_LINEAR, 0, 0, 0.5f, 1, 0, GX_ANISO_2); + ExpectFull(); + Warm(); + GX__InitTexObjWrapMode_80170b50(kObject, GX_MIRROR, GX_REPEAT); + ExpectFull(); + Warm(); + GX__InitTexObjTlut_80170b64(kObject, 3); + ExpectFull(); + Warm(); + GX__InitTexObjFilter_80170b6c(kObject, GX_LINEAR, GX_NEAR); + ExpectFull(); + Warm(); + GX__InitTexObjLODBias_80170b94(kObject, -0.5f); + ExpectFull(); + Warm(); + GX__InitTexObjUserData_80170be8(kObject, kOtherPixels); + ExpectFull(); + Warm(); + // Even an identical descriptor receives a new host texture identity. + Init(); + ExpectFull(); + EXPECT_FALSE(capture_fifo().empty()); + Warm(); + auto& meta = GetTexObjMeta(kObject); + meta.wrapT = GX_MIRROR; + ExpectFull(); +} + +TEST_F(RuntimeTextureBindingTest, PixelDmaBetweenQueuedLoadsCannotBeSkipped) { + Init(); + Warm(); + const auto* host = TryGetHostTexObj(kObject); + const auto oldVersion = reinterpret_cast(host)->texDataVersion; + GxWorker::detail::g_active = true; + Load(kObject, 1); + GxNotifyGuestRamDmaWrite(kPixels, 256); + Load(kObject, 1); + GxWorker::Sync(); + EXPECT_FALSE(g_TexObjMeta.at(kObject).needsUpload); + EXPECT_GT(reinterpret_cast(host)->texDataVersion, oldVersion); + EXPECT_EQ(TextureBindingTest::retiredRanges.size(), 1u); + auto bytes = capture_fifo(); + decode_fifo(bytes); + EXPECT_GT(gxState().loadedTextures[1].texDataVersion, oldVersion); +} + +TEST_F(RuntimeTextureBindingTest, DirtyAliasesAndAddressReuseUploadBeforeRebinding) { + Init(); + Warm(); + const auto oldVersion = reinterpret_cast(TryGetHostTexObj(kObject))->texDataVersion; + MarkTexObjsDirtyForRange(0xC0010000, 256); + EXPECT_TRUE(g_TexObjMeta.at(kObject).needsUpload); + ExpectFull(); + EXPECT_GT(reinterpret_cast(TryGetHostTexObj(kObject))->texDataVersion, oldVersion); + Warm(); + Memory::Write32(kObject + 0x0C, CanonicalizeGxMainRamAddress(kOtherPixels) >> 5); + ExpectFull(); + EXPECT_EQ(g_TexObjMeta.at(kObject).dataAddr, CanonicalizeGxMainRamAddress(kOtherPixels)); + auto bytes = capture_fifo(); + decode_fifo(bytes); + EXPECT_EQ(gxState().loadedTextures[0].data, GuestToHostPtr(kOtherPixels)); +} + +TEST_F(RuntimeTextureBindingTest, PaletteFormatsAlwaysKeepFullPathAcrossTlutChanges) { + for (uint32_t format : {GX_TF_C4, GX_TF_C8, GX_TF_C14X2}) { + Init(kObject, kPixels, format); + Warm(); + ExpectFull(); + GX__InitTlutObj_80170f80(kOtherObject, kOtherPixels, GX_TL_RGB565, 16); + GX__LoadTlut_80170fa8(kOtherObject, 0); + MarkTlutObjsDirtyForRange(kOtherPixels, 32); + GX__LoadTlut_80170fa8(kOtherObject, 0); + ExpectFull(); + } +} + +TEST_F(RuntimeTextureBindingTest, InvalidationAndCopyEvictionPreserveMetadataValidation) { + Init(); + Warm(); + GX__InvalidateTexAll_80171110(); + ExpectFast(); + EXPECT_FALSE(capture_fifo().empty()); + GXDestroyCopyTex(GuestToHostPtr(kPixels)); + ExpectFast(); + EXPECT_FALSE(capture_fifo().empty()); +} + +TEST_F(RuntimeTextureBindingTest, ResetAndHostDestructionCannotReuseCompletedLoad) { + Init(); + Warm(); + InvalidateTexObjLoads(); + ExpectFull(); + EXPECT_FALSE(capture_fifo().empty()); + Warm(); + g_TexObjMeta.at(kObject).host->Destroy(); + ExpectFull(); + EXPECT_TRUE(g_TexObjMeta.at(kObject).host->constructed); +} + +TEST_F(RuntimeTextureBindingTest, UnreadableSnapshotAndBadMapDoNotBecomeValidated) { + Init(); + Warm(); + TextureBindingTest::unreadableObject = CanonicalizeGxMainRamAddress(kObject); + ExpectFull(); + EXPECT_FALSE(g_TexObjMeta.at(kObject).validatedLoad.valid); + TextureBindingTest::unreadableObject = 0; + Warm(); + Load(kObject, 8); + EXPECT_TRUE(capture_fifo().empty()); + ExpectFast(); +} + +TEST_F(RuntimeTextureBindingTest, InvalidGuestDescriptorKeepsPlaceholderFallback) { + WriteGuestTexObj(kObject, kPixels, 8, 8, GX_CTF_R4, GX_CLAMP, GX_CLAMP, false, false, 0); + Load(); + auto bytes = capture_fifo(); + ASSERT_FALSE(bytes.empty()); + EXPECT_EQ(g_boundTexMaps[0].objAddr, 0u); + Init(); + Warm(); + ExpectFast(); +} + +TEST_F(RuntimeTextureBindingTest, DISABLED_BenchmarkRepeatedLoads) { + Init(); + Warm(); + constexpr uint32_t iterations = 250000; + auto& validation = g_TexObjMeta.at(kObject).validatedLoad; + for (bool forceFull : {true, false}) { + const auto checks = TextureBindingTest::rangeChecks; + const auto begin = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < iterations; ++i) { + if (forceFull) validation.Invalidate(); + Load(); + } + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - begin).count(); + std::printf("texture-load full=%d iterations=%u ns=%lld range-checks=%llu\n", forceFull, iterations, + static_cast(elapsed), + static_cast(TextureBindingTest::rangeChecks - checks)); + } +} +} // namespace diff --git a/runtime/tests/gx_worker_tests.cpp b/runtime/tests/gx_worker_tests.cpp new file mode 100644 index 000000000..5f9660238 --- /dev/null +++ b/runtime/tests/gx_worker_tests.cpp @@ -0,0 +1,283 @@ +#define MKW_GX_WORKER_TESTING 1 +#include "../src/hle/gx/gx_worker.cpp" + +#include + +#include +#include +#include + +namespace PerfStats { +bool Enabled() noexcept { return false; } +void AddWait(std::chrono::steady_clock::duration) noexcept {} +} // namespace PerfStats + +namespace { +std::vector> g_fifoReplay; +std::function g_service; +std::function g_beforeRecheck; + +void ServiceWait() { g_service(); } +void BeforeRecheck() { g_beforeRecheck(); } + +template +bool WaitUntil(F&& ready) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{2}; + while (!ready()) { + if (std::chrono::steady_clock::now() >= deadline) { + return false; + } + std::this_thread::yield(); + } + return true; +} + +class Gate { +public: + ~Gate() { Open(); } + void Block() { + std::unique_lock lock{m_mutex}; + m_entered = true; + m_cv.notify_all(); + m_cv.wait(lock, [&] { return m_open; }); + } + bool WaitEntered() { + std::unique_lock lock{m_mutex}; + return m_cv.wait_for(lock, std::chrono::seconds{2}, [&] { return m_entered; }); + } + void Open() { + std::lock_guard lock{m_mutex}; + m_open = true; + m_cv.notify_all(); + } + +private: + std::mutex m_mutex; + std::condition_variable m_cv; + bool m_entered = false; + bool m_open = false; +}; + +class GxWorkerTest : public ::testing::Test { +protected: + void SetUp() override { + GxWorker::g_testEnabled = true; + GxWorker::g_testWakeEachRecord = false; + GxWorker::g_testNotifications = 0; + GxWorker::g_testWaits = 0; + g_fifoReplay.clear(); + GxWorker::Start(); + } + void TearDown() override { + GxWorker::g_testBeforeTailRecheck = nullptr; + GxWorker::SetWaitCallback(nullptr); + GxWorker::Stop(); + g_service = {}; + g_beforeRecheck = {}; + } +}; + +TEST_F(GxWorkerTest, DoesNotNotifyBeforeTheRequestedTail) { + Gate first; + Gate last; + uint32_t completed = 0; + GxWorker::Submit([&] { first.Block(); }); + for (uint32_t i = 0; i < 512; ++i) { + GxWorker::Submit([&, i] { EXPECT_EQ(completed++, i); }); + } + GxWorker::Submit([&] { last.Block(); }); + std::thread release{[&] { + EXPECT_TRUE(WaitUntil([] { return GxWorker::g_waitTarget.load() != GxWorker::kNotWaiting; })); + first.Open(); + EXPECT_TRUE(last.WaitEntered()); + EXPECT_EQ(GxWorker::g_testNotifications.load(), 0u); + last.Open(); + }}; + GxWorker::Sync(); + release.join(); + EXPECT_EQ(completed, 512u); + EXPECT_GE(GxWorker::g_testWaits.load(), 1u); + EXPECT_EQ(GxWorker::g_waitTarget.load(), GxWorker::kNotWaiting); +} + +TEST_F(GxWorkerTest, RechecksCompletionAfterPublishingTheWaitTarget) { + Gate gate; + GxWorker::Submit([&] { gate.Block(); }); + g_beforeRecheck = [&] { + GxWorker::g_testBeforeTailRecheck = nullptr; + gate.Open(); + EXPECT_TRUE(WaitUntil([] { return GxWorker::g_tail.load() >= GxWorker::g_waitTarget.load(); })); + }; + GxWorker::g_testBeforeTailRecheck = BeforeRecheck; + GxWorker::Sync(); + EXPECT_EQ(GxWorker::g_testWaits.load(), 0u); + EXPECT_EQ(GxWorker::g_waitTarget.load(), GxWorker::kNotWaiting); +} + +TEST_F(GxWorkerTest, ServicesCallbacksAndDrainsTheirNestedSyncAndNewWork) { + Gate gate; + uint32_t callbacks = 0; + uint32_t completed = 0; + GxWorker::Submit([&] { gate.Block(); ++completed; }); + g_service = [&] { + if (++callbacks == 1) { + GxWorker::Submit([&] { ++completed; }); + gate.Open(); + GxWorker::Sync(); + } + }; + GxWorker::SetWaitCallback(ServiceWait); + GxWorker::Sync(); + EXPECT_GE(callbacks, 1u); + EXPECT_EQ(completed, 2u); + EXPECT_EQ(GxWorker::g_waitTarget.load(), GxWorker::kNotWaiting); +} + +TEST_F(GxWorkerTest, RingBackpressureAndWrapPreserveOrderWithoutServicingGuestCallbacks) { + Gate gate; + uint32_t callbacks = 0; + uint32_t completed = 0; + GxWorker::Submit([&] { gate.Block(); }); + std::thread release{[&] { + EXPECT_TRUE(WaitUntil([] { return GxWorker::g_waitTarget.load() != GxWorker::kNotWaiting; })); + gate.Open(); + }}; + g_service = [&] { ++callbacks; }; + GxWorker::SetWaitCallback(ServiceWait); + for (uint32_t i = 0; i < 160; ++i) { + std::array bytes{}; + bytes.front() = static_cast(i); + bytes.back() = static_cast(i ^ 0xff); + GxWorker::Submit([&, bytes, i] { + EXPECT_EQ(completed++, i); + EXPECT_EQ(bytes.front(), static_cast(i)); + EXPECT_EQ(bytes.back(), static_cast(i ^ 0xff)); + }); + } + release.join(); + EXPECT_EQ(callbacks, 0u); + GxWorker::SetWaitCallback(nullptr); + GxWorker::Sync(); + EXPECT_EQ(completed, 160u); +} + +TEST_F(GxWorkerTest, FifoFlushInlineAndRestartRetainTheirBoundaries) { + GxWorker::FifoWrite(0x61, 1); + GxWorker::FifoWrite(0x12345678, 4); + const std::array burst{1, 2, 3}; + GxWorker::FifoBurst(burst.data(), burst.size()); + GxWorker::Submit([] { EXPECT_EQ(g_fifoReplay.size(), 5u); }); + { + GxWorker::InlineScope scope; + EXPECT_FALSE(GxWorker::ShouldDefer()); + EXPECT_EQ(g_fifoReplay, (std::vector>{ + {0x61, 1}, {0x12345678, 4}, {1, 1}, {2, 1}, {3, 1}})); + } + EXPECT_TRUE(GxWorker::ShouldDefer()); + auto lifetime = std::make_shared(0); + std::weak_ptr weak = lifetime; + GxWorker::Submit([owned = std::move(lifetime)] {}); + GxWorker::Stop(); + EXPECT_TRUE(weak.expired()); + GxWorker::g_testEnabled = false; + GxWorker::Start(); + EXPECT_FALSE(GxWorker::Active()); + bool ran = false; + GxWorker::Run([&] { ran = true; }); + EXPECT_TRUE(ran); + GxWorker::g_testEnabled = true; + GxWorker::Start(); + GxWorker::Submit([&] { ran = false; }); + GxWorker::Sync(); + EXPECT_FALSE(ran); +} + +TEST_F(GxWorkerTest, ThrowingCommandsStillCompleteTheirRecordAndReleaseTheirPayload) { + auto lifetime = std::make_shared(0); + std::weak_ptr weak = lifetime; + bool completed = false; + GxWorker::Submit([owned = std::move(lifetime)] { throw std::runtime_error("injected failure"); }); + GxWorker::Submit([&] { completed = true; }); + GxWorker::Sync(); + EXPECT_TRUE(completed); + EXPECT_TRUE(weak.expired()); +} + +#if GTEST_HAS_DEATH_TEST +TEST(GxWorkerDeathTest, AbandonWakesTheIdleConsumerAndSignalsExit) { + ASSERT_EXIT({ + GxWorker::Start(); + GxWorker::Abandon(); + std::_Exit(!GxWorker::Active() && GxWorker::g_thread == nullptr && GxWorker::g_exited.load() ? 0 : 1); + }, ::testing::ExitedWithCode(0), ""); +} + +TEST(GxWorkerDeathTest, AbandonDoesNotExecuteTheFollowingRecord) { + ASSERT_EXIT({ + Gate gate; + std::atomic ran{false}; + GxWorker::Start(); + GxWorker::Submit([&] { gate.Block(); }); + GxWorker::Submit([&] { ran = true; }); + GxWorker::Publish(); + if (!gate.WaitEntered()) { + std::_Exit(2); + } + std::thread release{[&] { + WaitUntil([] { return GxWorker::g_abandon.load(); }); + gate.Open(); + }}; + GxWorker::Abandon(); + release.join(); + std::_Exit(!ran.load() && GxWorker::g_exited.load() ? 0 : 1); + }, ::testing::ExitedWithCode(0), ""); +} +#endif + +TEST_F(GxWorkerTest, DISABLED_WakeBenchmark) { + std::array, 2> times; + std::array notifications{}; + std::array waits{}; + for (uint32_t batch = 0; batch < 40; ++batch) { + for (uint32_t order = 0; order < 2; ++order) { + const uint32_t mode = (batch + order) % 2; + GxWorker::g_testWakeEachRecord = mode != 0; + GxWorker::g_testNotifications = 0; + GxWorker::g_testWaits = 0; + uint32_t completed = 0; + Gate gate; + const auto start = std::chrono::steady_clock::now(); + GxWorker::Submit([&] { gate.Block(); }); + for (uint32_t i = 0; i < 20000; ++i) { + GxWorker::Submit([&] { ++completed; }); + } + std::thread release{[&] { + EXPECT_TRUE(WaitUntil([] { return GxWorker::g_waitTarget.load() != GxWorker::kNotWaiting; })); + gate.Open(); + }}; + GxWorker::Sync(); + release.join(); + times[mode].push_back(std::chrono::duration_cast( + std::chrono::steady_clock::now() - start).count()); + notifications[mode] += GxWorker::g_testNotifications.load(); + waits[mode] += GxWorker::g_testWaits.load(); + EXPECT_EQ(completed, 20000u); + } + } + for (uint32_t mode = 0; mode < 2; ++mode) { + std::sort(times[mode].begin(), times[mode].end()); + std::printf("wake_each=%u records=800000 notifications=%llu waits=%llu median_us=%lld p95_us=%lld\n", mode, + static_cast(notifications[mode]), static_cast(waits[mode]), + static_cast((times[mode][19] + times[mode][20]) / 2), + static_cast(times[mode][37])); + } + GxWorker::g_testWakeEachRecord = false; +} +} // namespace + +void GxWorkerReplayFifoWrite(uint32_t value, uint32_t sizeBytes) { g_fifoReplay.emplace_back(value, sizeBytes); } +void GxWorkerReplayFifoBurst(const uint8_t* data, uint32_t sizeBytes) { + for (uint32_t i = 0; i < sizeBytes; ++i) { + GxWorkerReplayFifoWrite(data[i], 1); + } +} diff --git a/runtime/tests/runtime_config_tests.cpp b/runtime/tests/runtime_config_tests.cpp index 32e42f679..98f8fadcd 100644 --- a/runtime/tests/runtime_config_tests.cpp +++ b/runtime/tests/runtime_config_tests.cpp @@ -34,6 +34,26 @@ bool SetsOffset(const std::string& original, const char* message) { } // namespace int main() { + for (const std::string mode : {"specialized", "hybrid"}) { + std::istringstream input("[video]\nshader_mode = '" + mode + "'\n"); + const auto config = RuntimeConfigFile::ParseConfig(input, "shader.toml"); + if (!Require(config.shaderMode == mode, "valid shader mode was not loaded")) return 1; + } + for (const std::string text : {"", "video = 4\n", "[[video]]\nshader_mode = 'hybrid'\n", + "[video]\nshader_mode = true\n", "[video]\nshader_mode = 1\n", + "[video]\nshader_mode = []\n", "[video]\nshader_mode = {}\n", + "[video]\nshader_mode = 'force_generic'\n", "[video]\nshader_mode = 'HYBRID'\n", + "[video]\nshader_mode = 'hybrid'\nshader_mode = 'specialized'\n", + "[video]\nshader_mode = 'hybrid'\n[video]\n"}) { + std::istringstream input(text); + const auto config = RuntimeConfigFile::ParseConfig(input, "invalid-shader.toml"); + if (!Require(config.shaderMode.value_or("specialized") == "specialized", "invalid shader mode did not default")) return 1; + } + { + const auto doc = Update("[video]\nshader_mode = 'specialized'\n", "video", "shader_mode", "'hybrid'"); + if (!Require(doc && doc->at("video").at("shader_mode").as_string() == "hybrid", "next-start shader mode was not saved")) return 1; + } + { std::istringstream input("[video]\nmetalfx_spatial_upscaling = true\n"); const RuntimeUserConfig config = RuntimeConfigFile::ParseConfig(input, "enabled.toml"); diff --git a/translator/src/Translator.Cli/Program.cs b/translator/src/Translator.Cli/Program.cs index e1164dd2a..064db8bca 100644 --- a/translator/src/Translator.Cli/Program.cs +++ b/translator/src/Translator.Cli/Program.cs @@ -461,6 +461,7 @@ int RunTranslateRecursive(string[] argsTail) var outputMetadataPath = OptionValue(argsTail, "--output-metadata"); var productionSourceBundlePath = OptionValue(argsTail, "--production-source-bundle"); var emitFunctionFiles = !HasFlag(argsTail, "--no-function-files"); + var boundedIr = HasFlag(argsTail, "--bounded-ir"); if (!emitFunctionFiles && string.IsNullOrWhiteSpace(productionSourceBundlePath)) throw new ArgumentException("--no-function-files requires --production-source-bundle."); BaseTranslationOutputMetadata? previousOutputMetadata = null; @@ -954,14 +955,17 @@ void LogPhase(string label) var emissionAddresses = baseSummaries.Keys.OrderBy(address => address).ToArray(); var localGuestAbiContracts = new Dictionary(emissionAddresses.Length); - var canonicalGuestFunctions = new Dictionary(emissionAddresses.Length); + var bufferedGuestFunctions = boundedIr ? null : new Dictionary(emissionAddresses.Length); + IReadOnlyDictionary canonicalGuestFunctions = bufferedGuestFunctions is not null + ? bufferedGuestFunctions + : new CanonicalIrDictionary(canonicalIrStore, emissionAddresses); var irInstructionCounts = new Dictionary(emissionAddresses.Length); // Each slot is an independent pure decode plus contract analysis, so // compute them in parallel into position-indexed arrays and fill the // dictionaries afterwards in the original ascending order. Dictionary // insertion order is preserved, so every downstream enumeration of // these maps is byte-for-byte the same as the serial version. - var canonicalByIndex = new IrFunction[emissionAddresses.Length]; + var canonicalByIndex = boundedIr ? null : new IrFunction[emissionAddresses.Length]; var localContractByIndex = new GuestAbiContract[emissionAddresses.Length]; var irInstructionCountByIndex = new int[emissionAddresses.Length]; IndexedParallel.For(emissionAddresses.Length, parallelOptions, i => @@ -969,14 +973,14 @@ void LogPhase(string label) var address = emissionAddresses[i]; if (!canonicalIrStore.TryGet(address, out var canonicalSsa)) throw new InvalidOperationException($"Missing canonical SSA while building ABI contract for 0x{address:X8}."); - canonicalByIndex[i] = canonicalSsa; + if (canonicalByIndex is not null) canonicalByIndex[i] = canonicalSsa; localContractByIndex[i] = GuestAbiContractAnalyzer.Analyze(canonicalSsa); irInstructionCountByIndex[i] = canonicalSsa.Blocks.Sum(block => block.Instructions.Count); }); for (var i = 0; i < emissionAddresses.Length; i++) { var address = emissionAddresses[i]; - canonicalGuestFunctions[address] = canonicalByIndex[i]; + if (bufferedGuestFunctions is not null) bufferedGuestFunctions[address] = canonicalByIndex![i]; localGuestAbiContracts[address] = localContractByIndex[i]; irInstructionCounts[address] = irInstructionCountByIndex[i]; } @@ -988,7 +992,9 @@ void LogPhase(string label) // rather than retaining every descendant dependency forever. var interproceduralGuestAbi = GuestAbiInterproceduralAnalyzer.Analyze( canonicalGuestFunctions, - nativeGuestEffects.Contracts); + nativeGuestEffects.Contracts, + retainBodies: !boundedIr, + maxDegreeOfParallelism: boundedIr ? maxThreads : null); var guestAbiContracts = interproceduralGuestAbi.Contracts .ToDictionary(static pair => pair.Key, static pair => pair.Value); var recursiveComponents = interproceduralGuestAbi.StronglyConnectedComponents @@ -3697,6 +3703,7 @@ static string[] KnownCommands() => new[] new("--production-source-bundle", "path"), new("--threads", "N"), new("--no-function-files"), + new("--bounded-ir"), new("--prune-stale") }), "translate-mod" => (null, new CommandOption[] diff --git a/translator/src/Translator.Core/Analysis/GuestAbiInterproceduralAnalyzer.cs b/translator/src/Translator.Core/Analysis/GuestAbiInterproceduralAnalyzer.cs index f6817d3a1..a5374ac88 100644 --- a/translator/src/Translator.Core/Analysis/GuestAbiInterproceduralAnalyzer.cs +++ b/translator/src/Translator.Core/Analysis/GuestAbiInterproceduralAnalyzer.cs @@ -18,7 +18,9 @@ public static class GuestAbiInterproceduralAnalyzer { public static GuestAbiInterproceduralResult Analyze( IReadOnlyDictionary functions, - IReadOnlyDictionary? externalContracts = null) + IReadOnlyDictionary? externalContracts = null, + bool retainBodies = true, + int? maxDegreeOfParallelism = null) { externalContracts ??= new Dictionary(); var addresses = functions.Keys.OrderBy(static address => address).ToArray(); @@ -26,12 +28,18 @@ public static GuestAbiInterproceduralResult Analyze( // GuestAbiContractAnalyzer.Analyze is pure and the round's contract snapshot is frozen, so // per-function analyses within a round can run on any thread. Determinism comes from // applying results serially in ascending-ordinal order, matching the old single-threaded loop. - var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; + var parallelOptions = new ParallelOptions + { + MaxDegreeOfParallelism = maxDegreeOfParallelism ?? Environment.ProcessorCount + }; - // Resolving the IR by ordinal once keeps the caller-supplied dictionary - // off every parallel path: only this loop and BuildComponents touch it. - var bodies = new IrFunction[addresses.Length]; - for (var index = 0; index < addresses.Length; ++index) bodies[index] = functions[addresses[index]]; + // Lazy callers must supply stable bodies and support concurrent reads. + var bodies = retainBodies ? new IrFunction[addresses.Length] : null; + if (bodies is not null) + { + for (var index = 0; index < addresses.Length; ++index) bodies[index] = functions[addresses[index]]; + } + IrFunction Body(int index) => bodies is not null ? bodies[index] : functions[addresses[index]]; // Mirror of `contracts` keyed by ordinal. The fixpoint compares against // it from the parallel phase, where the dictionary must stay untouched. @@ -40,7 +48,7 @@ public static GuestAbiInterproceduralResult Analyze( 0, addresses.Length, parallelOptions, - index => current[index] = GuestAbiContractAnalyzer.Analyze(bodies[index])); + index => current[index] = GuestAbiContractAnalyzer.Analyze(Body(index))); var contracts = new Dictionary(addresses.Length); for (var index = 0; index < addresses.Length; ++index) contracts.Add(addresses[index], current[index]); @@ -62,7 +70,7 @@ public static GuestAbiInterproceduralResult Analyze( (index, _, scratch) => { scratch.Clear(); - CollectDirectCallees(bodies[index], ordinals, scratch); + CollectDirectCallees(Body(index), ordinals, scratch); directCallees[index] = scratch.ToArray(); return scratch; }, @@ -95,7 +103,7 @@ public static GuestAbiInterproceduralResult Analyze( Parallel.For(0, worklistCount, parallelOptions, position => { var ordinal = worklist[position]; - var next = GuestAbiContractAnalyzer.Analyze(bodies[ordinal], visible); + var next = GuestAbiContractAnalyzer.Analyze(Body(ordinal), visible); roundResults[position] = Equivalent(current[ordinal], next) ? null : next; }); diff --git a/translator/src/Translator.Core/Translation/CanonicalIrDictionary.cs b/translator/src/Translator.Core/Translation/CanonicalIrDictionary.cs new file mode 100644 index 000000000..b707683e8 --- /dev/null +++ b/translator/src/Translator.Core/Translation/CanonicalIrDictionary.cs @@ -0,0 +1,58 @@ +using System.Collections; +using Translator.Core.Ir; + +namespace Translator.Core.Translation; + +/// +/// Fixed keys over canonical SSA decoded on demand without caching bodies. +/// Concurrent readers require a thread-safe provider, such as CanonicalIrStore. +/// +public sealed class CanonicalIrDictionary : IReadOnlyDictionary +{ + private readonly ICanonicalIrProvider _provider; + private readonly IReadOnlyList _addresses; + private readonly HashSet _membership; + + public CanonicalIrDictionary(ICanonicalIrProvider provider, IEnumerable addresses) + { + ArgumentNullException.ThrowIfNull(provider); + ArgumentNullException.ThrowIfNull(addresses); + _provider = provider; + var snapshot = addresses.ToArray(); + _membership = new HashSet(snapshot); + if (_membership.Count != snapshot.Length) + throw new ArgumentException("Canonical IR addresses must be unique.", nameof(addresses)); + _addresses = Array.AsReadOnly(snapshot); + } + + public int Count => _addresses.Count; + public IEnumerable Keys => _addresses; + public IEnumerable Values + { + get + { + foreach (var address in _addresses) yield return this[address]; + } + } + + public IrFunction this[uint key] => TryGetValue(key, out var value) + ? value + : throw new KeyNotFoundException($"Missing canonical SSA for 0x{key:X8}."); + + public bool ContainsKey(uint key) => _membership.Contains(key); + + public bool TryGetValue(uint key, out IrFunction value) + { + if (_membership.Contains(key) && _provider.TryGet(key, out value)) return true; + value = null!; + return false; + } + + public IEnumerator> GetEnumerator() + { + foreach (var address in _addresses) + yield return new KeyValuePair(address, this[address]); + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); +} diff --git a/translator/tests/Translator.Tests/CanonicalIrDictionaryTests.cs b/translator/tests/Translator.Tests/CanonicalIrDictionaryTests.cs new file mode 100644 index 000000000..7acc5bb5b --- /dev/null +++ b/translator/tests/Translator.Tests/CanonicalIrDictionaryTests.cs @@ -0,0 +1,285 @@ +using System.Buffers.Binary; +using System.Collections; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Translator.Core.Analysis; +using Translator.Core.Ir; +using Translator.Core.Loading; +using Translator.Core.Translation; +using Xunit; + +namespace Translator.Tests; + +public sealed class CanonicalIrDictionaryTests +{ + [Fact] + public void KeysAndMembershipUseAnIndependentSnapshotWithoutDecoding() + { + var store = new CanonicalIrStore(); + store.Put(20, Function("second", new IrReturn(null))); + store.Put(10, Function("first", new IrReturn(null))); + store.Put(30, Function("outside", new IrReturn(null))); + var provider = new CountingProvider(store); + var addresses = new uint[] { 20, 10 }; + var view = new CanonicalIrDictionary(provider, addresses); + addresses[0] = 0; + + Assert.Equal(2, view.Count); + Assert.Equal(new uint[] { 20, 10 }, view.Keys); + Assert.True(view.ContainsKey(10)); + Assert.False(view.ContainsKey(0)); + Assert.False(view.ContainsKey(30)); + Assert.False(view.TryGetValue(30, out var missing)); + Assert.Null(missing); + Assert.Throws(() => view[30]); + Assert.Equal(0, provider.Reads); + + var first = view[10]; + Assert.NotSame(first, view[10]); + Assert.Equal(2, provider.Reads); + Assert.Equal("second", Assert.Single(view.Values.Take(1)).Name); + Assert.Equal(3, provider.Reads); + using var entries = view.GetEnumerator(); + Assert.Equal(3, provider.Reads); + Assert.True(entries.MoveNext()); + Assert.Equal(20u, entries.Current.Key); + Assert.Equal(4, provider.Reads); + Assert.Equal(view.Keys, view.Select(static pair => pair.Key)); + Assert.Equal(new[] { "second", "first" }, view.Values.Select(static body => body.Name)); + Assert.Equal(view.Keys, ((IEnumerable)view).Cast>().Select(static pair => pair.Key)); + } + + [Fact] + public void MissingAndReplacedBodiesDoNotChangeMembershipOrLeaveCachedValues() + { + var store = new CanonicalIrStore(); + store.Put(10, Function("original", new IrReturn(null))); + var view = new CanonicalIrDictionary(store, new uint[] { 10 }); + Assert.Equal("original", view[10].Name); + store.Put(10, Function("replacement", new IrReturn(null))); + Assert.Equal("replacement", view[10].Name); + store.Remove(10); + + Assert.Single(view.Keys); + Assert.True(view.ContainsKey(10)); + Assert.Equal(new uint[] { 10 }, view.Keys); + Assert.False(view.TryGetValue(10, out var missing)); + Assert.Null(missing); + Assert.Throws(() => view[10]); + Assert.Throws(() => view.Values.ToArray()); + Assert.Throws(() => view.ToArray()); + } + + [Fact] + public void ConstructorRejectsNullInputsAndDuplicateKeys() + { + var store = new CanonicalIrStore(); + Assert.Throws(() => new CanonicalIrDictionary(null!, Array.Empty())); + Assert.Throws(() => new CanonicalIrDictionary(store, null!)); + Assert.Throws(() => new CanonicalIrDictionary(store, new uint[] { 10, 20, 10 })); + var empty = new CanonicalIrDictionary(store, Array.Empty()); + Assert.Empty(empty); + Assert.Empty(empty.Keys); + Assert.Empty(empty.Values); + } + + [Fact] + public void ConcurrentReadersEnumerateInSnapshotOrderAndDecodeIndependentBodies() + { + var store = new CanonicalIrStore(); + var addresses = Enumerable.Range(1, 16).Reverse().Select(static index => (uint)index).ToArray(); + foreach (var address in addresses) store.Put(address, Function($"f{address}", new IrReturn(null))); + var provider = new CountingProvider(store); + var view = new CanonicalIrDictionary(provider, addresses); + var bodies = new IrFunction[32]; + + Parallel.For(0, bodies.Length, new ParallelOptions { MaxDegreeOfParallelism = 4 }, index => + { + Assert.Equal(addresses, view.Keys); + Assert.All(addresses, address => Assert.True(view.ContainsKey(address))); + Assert.False(view.TryGetValue(0, out _)); + Assert.Equal(addresses, view.Select(static pair => pair.Key)); + Assert.Equal(addresses.Select(static address => $"f{address}"), view.Values.Select(static body => body.Name)); + bodies[index] = view[1]; + }); + + Assert.Equal(bodies.Length * (2 * addresses.Length + 1), provider.Reads); + for (var index = 1; index < bodies.Length; ++index) Assert.NotSame(bodies[0], bodies[index]); + } + + [Fact] + public void LazyCanonicalBodyPreservesSerializedIrAndLoweredSource() + { + var memory = new byte[8]; + BinaryPrimitives.WriteUInt32BigEndian(memory.AsSpan(0, 4), 0x38630001u); // addi r3,r3,1 + BinaryPrimitives.WriteUInt32BigEndian(memory.AsSpan(4, 4), 0x4E800020u); // blr + var image = new ProgramImage(memory, + AddressRange.FromStartAndSize(MemoryLayout.RamBase, (uint)memory.Length), + AddressRange.FromStartAndSize(MemoryLayout.RamBase, (uint)memory.Length), default, "bounded-ir-test"); + var translator = new FunctionTranslator(image); + var options = TranslationOptions.Default with { PreferredName = "bounded_ir_test" }; + var canonical = translator.Discover(MemoryLayout.RamBase, options).Ssa.Function; + var store = new CanonicalIrStore(); + store.Put(MemoryLayout.RamBase, canonical); + var view = new CanonicalIrDictionary(store, new[] { MemoryLayout.RamBase }); + var buffered = translator.LowerCanonical(MemoryLayout.RamBase, canonical, options); + var lazy = translator.LowerCanonical(MemoryLayout.RamBase, view[MemoryLayout.RamBase], options); + + Assert.Equal(buffered.CxxCode, lazy.CxxCode); + Assert.Equal(JsonSerializer.Serialize(canonical), JsonSerializer.Serialize(view[MemoryLayout.RamBase])); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(-1)] + public void LazyAnalysisMatchesBufferedContractsAndComponents(int parallelism) + { + var functions = new Dictionary + { + [0x80003000] = Function("leaf", + new IrAssign("r7", IrValue.Register("r6")), + new IrCall("lr", "0x80004000", Array.Empty()), new IrReturn(null)), + [0x80001000] = Function("first", + new IrAssign("r5", IrValue.Register("r3")), + new IrCall("lr", "0x80002000", Array.Empty()), new IrReturn(null)), + [0x80002000] = Function("second", + new IrAssign("r6", IrValue.Register("r4")), + new IrCall("lr", "0x80001000", Array.Empty()), + new IrCall("lr", "0x80003000", Array.Empty()), new IrReturn(null)), + [0x80005000] = Function("self", + new IrCall("lr", "0x80005000", Array.Empty()), new IrReturn(null)), + [0x80006000] = Function("unknown", + new IrCall("lr", "0x80007000", Array.Empty()), + new IrIndirectCall("lr", IrValue.Register("ctr"), Array.Empty()), new IrReturn(null)) + }; + var external = new Dictionary + { + [0x80004000] = GuestAbiContractAnalyzer.Analyze(Function("external", + new IrAssign("r8", IrValue.Register("r9")), new IrReturn(null))) + }; + var store = new CanonicalIrStore(); + foreach (var pair in functions) store.Put(pair.Key, pair.Value); + var view = new CanonicalIrDictionary(store, functions.Keys.Reverse()); + var buffered = GuestAbiInterproceduralAnalyzer.Analyze(functions, external); + var lazy = GuestAbiInterproceduralAnalyzer.Analyze(view, external, + retainBodies: false, maxDegreeOfParallelism: parallelism); + + Assert.Equal(JsonSerializer.Serialize(buffered), JsonSerializer.Serialize(lazy)); + Assert.Equal(functions.Keys.OrderBy(static address => address), lazy.Contracts.Keys); + Assert.Contains(lazy.StronglyConnectedComponents, + component => component.SequenceEqual(new uint[] { 0x80001000, 0x80002000 })); + Assert.Contains(lazy.StronglyConnectedComponents, + component => component.SequenceEqual(new uint[] { 0x80005000 })); + Assert.True(lazy.Contracts[0x80006000].HasFullSynchronizationFence); + Assert.NotEqual(0u, lazy.Contracts[0x80003000].GprReadBeforeWriteMask & (1u << 9)); + } + + [Fact] + public void DefaultAnalysisResolvesEachBodyOnceAndLazyAnalysisResolvesEachPass() + { + var store = new CanonicalIrStore(); + store.Put(10, Function("leaf", new IrReturn(null))); + var provider = new CountingProvider(store); + var view = new CanonicalIrDictionary(provider, new uint[] { 10 }); + + var buffered = GuestAbiInterproceduralAnalyzer.Analyze(view); + Assert.Equal(1, provider.Reads); + var lazy = GuestAbiInterproceduralAnalyzer.Analyze(view, retainBodies: false, maxDegreeOfParallelism: 1); + Assert.Equal(4, provider.Reads); + Assert.Equal(JsonSerializer.Serialize(buffered), JsonSerializer.Serialize(lazy)); + } + + [Fact] + public void LazyAnalysisRevisitsCallersOfInlineThunksAfterTheirCalleesChange() + { + var range = GuestSaveRestoreThunks.Current.SaveGpr; + Assert.NotNull(range); + var thunk = range!.BaseAddress; + var functions = new Dictionary + { + [0x80001000] = Function("caller", + new IrCall("lr", $"0x{thunk:X8}", Array.Empty()), new IrReturn(null)), + [thunk] = Function("thunk", + new IrCall("lr", "0x80002000", Array.Empty()), new IrReturn(null)), + [0x80002000] = Function("middle", + new IrCall("lr", "0x80003000", Array.Empty()), new IrReturn(null)), + [0x80003000] = Function("leaf", + new IrAssign("r8", IrValue.Register("r9")), new IrReturn(null)) + }; + var store = new CanonicalIrStore(); + foreach (var pair in functions) store.Put(pair.Key, pair.Value); + var buffered = GuestAbiInterproceduralAnalyzer.Analyze(functions); + var lazy = GuestAbiInterproceduralAnalyzer.Analyze(new CanonicalIrDictionary(store, functions.Keys), + retainBodies: false, maxDegreeOfParallelism: 2); + + Assert.Equal(JsonSerializer.Serialize(buffered), JsonSerializer.Serialize(lazy)); + Assert.DoesNotContain(thunk, lazy.Contracts[0x80001000].DirectCallTargets); + Assert.Equal(1u << 9, lazy.Contracts[0x80001000].GprReadBeforeWriteMask); + Assert.NotEqual(0u, lazy.Contracts[0x80001000].GprPossibleWriteMask & (1u << 8)); + } + + [Theory] + [InlineData(true, 0)] + [InlineData(false, 0)] + [InlineData(true, -2)] + [InlineData(false, -2)] + public void InvalidParallelismFailsBeforeDecoding(bool retainBodies, int parallelism) + { + var provider = new CountingProvider(new CanonicalIrStore()); + var view = new CanonicalIrDictionary(provider, new uint[] { 10 }); + Assert.Throws(() => GuestAbiInterproceduralAnalyzer.Analyze( + view, retainBodies: retainBodies, maxDegreeOfParallelism: parallelism)); + Assert.Equal(0, provider.Reads); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void EmptyAnalysisPreservesEmptyResults(bool retainBodies) + { + var view = new CanonicalIrDictionary(new CanonicalIrStore(), Array.Empty()); + var result = GuestAbiInterproceduralAnalyzer.Analyze(view, retainBodies: retainBodies, maxDegreeOfParallelism: 1); + Assert.Empty(result.Contracts); + Assert.Empty(result.StronglyConnectedComponents); + } + + [Fact] + public void SingleWorkerLimitAppliesToLazyProviderReadsInEveryAnalysisPass() + { + var store = new CanonicalIrStore(); + var addresses = Enumerable.Range(1, 32).Select(static index => (uint)index).ToArray(); + foreach (var address in addresses) store.Put(address, Function($"f{address}", new IrReturn(null))); + var provider = new CountingProvider(store, requireSerial: true); + var result = GuestAbiInterproceduralAnalyzer.Analyze(new CanonicalIrDictionary(provider, addresses), + retainBodies: false, maxDegreeOfParallelism: 1); + Assert.Equal(addresses.Length, result.Contracts.Count); + Assert.Equal(3 * addresses.Length, provider.Reads); + } + + private static IrFunction Function(string name, params IrInstruction[] instructions) => + new(name, "entry", new[] { new IrBasicBlock("entry", instructions) }); + + private sealed class CountingProvider(ICanonicalIrProvider inner, bool requireSerial = false) : ICanonicalIrProvider + { + private int _reads; + private int _active; + public int Reads => Volatile.Read(ref _reads); + + public bool TryGet(uint entryPoint, out IrFunction function) + { + var active = Interlocked.Increment(ref _active); + try + { + if (requireSerial) Assert.Equal(1, active); + Interlocked.Increment(ref _reads); + return inner.TryGet(entryPoint, out function); + } + finally + { + Interlocked.Decrement(ref _active); + } + } + } +} diff --git a/translator/tests/Translator.Tests/TranslateRecursiveRuntimeConfigTests.cs b/translator/tests/Translator.Tests/TranslateRecursiveRuntimeConfigTests.cs index 7f9254f25..982495af5 100644 --- a/translator/tests/Translator.Tests/TranslateRecursiveRuntimeConfigTests.cs +++ b/translator/tests/Translator.Tests/TranslateRecursiveRuntimeConfigTests.cs @@ -8,6 +8,91 @@ namespace Translator.Tests; public sealed class TranslateRecursiveRuntimeConfigTests { + [Fact] + public void BoundedIrMatchesBufferedFilesMetadataAndBundleAcrossThreadCounts() + { + var root = Path.Combine(Path.GetTempPath(), $"translator-bounded-ir-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + const uint entry = 0x80001000u; + File.WriteAllBytes(Path.Combine(root, "main.dol"), SyntheticDolFactory.CreateBytes(entry, + sections: + [ + SyntheticDolFactory.Text(0, entry, + 0x38A30001u, // addi r5,r3,1 + 0x4800001Du, // bl 0x80001020 + 0x38650001u, // addi r3,r5,1 + 0x4E800020u), + SyntheticDolFactory.Text(1, entry + 0x20, + 0x38C40001u, // addi r6,r4,1 + 0x4BFFFFDDu, // bl 0x80001000 + 0x38660001u, // addi r3,r6,1 + 0x4E800020u) + ])); + var projectPath = Path.Combine(root, "recomp.yml"); + File.WriteAllText(projectPath, + """ + schema_version: 1 + project: + id: bounded-ir-test + memory: + sda_base: 0x80002000 + sda2_base: 0x80003000 + inputs: + dol: + path: main.dol + translation: + entry_points: [0x80001000, 0x80001020] + output: + root: generated + """); + + (Dictionary Sources, byte[] Metadata, byte[] Bundle) Translate( + string name, int threads, bool bounded, bool emitFiles = true) + { + var output = Path.Combine(root, name); + var functions = Path.Combine(output, "functions"); + var metadata = Path.Combine(output, "metadata.json"); + var bundle = Path.Combine(output, "sources.bundle"); + var args = new List + { + "translate-recursive", $"0x{entry:X8}", "--project", projectPath, + "--outdir", functions, "--output-metadata", metadata, + "--production-source-bundle", bundle, "--threads", threads.ToString() + }; + if (bounded) args.Add("--bounded-ir"); + if (!emitFiles) args.Add("--no-function-files"); + Assert.Equal(0, InvokeCli(args.ToArray())); + var sources = Directory.Exists(functions) + ? Directory.GetFiles(functions, "*.cpp").ToDictionary( + static path => Path.GetFileName(path), static path => File.ReadAllText(path)) + : new Dictionary(); + return (sources, File.ReadAllBytes(metadata), File.ReadAllBytes(bundle)); + } + + var buffered = Translate("buffered", 1, bounded: false); + Assert.Equal(2, buffered.Sources.Count); + foreach (var threads in new[] { 1, 3 }) + { + var lazy = Translate($"lazy-{threads}", threads, bounded: true); + Assert.Equal(buffered.Sources.Keys.OrderBy(static name => name), + lazy.Sources.Keys.OrderBy(static name => name)); + foreach (var pair in buffered.Sources) Assert.Equal(pair.Value, lazy.Sources[pair.Key]); + Assert.Equal(buffered.Metadata, lazy.Metadata); + Assert.Equal(buffered.Bundle, lazy.Bundle); + } + var bundleOnly = Translate("bundle-only", 3, bounded: true, emitFiles: false); + Assert.Empty(bundleOnly.Sources); + Assert.Equal(buffered.Metadata, bundleOnly.Metadata); + Assert.Equal(buffered.Bundle, bundleOnly.Bundle); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } + [Fact] public void UnsupportedOpcodeFailsStrictTranslation() {