From efd6a44da2c938d583a6a89838be84ffefb1ac0b Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Fri, 5 Jun 2026 14:59:35 -0700 Subject: [PATCH 1/6] [Wasm] Spill live ref/byref values to pinned stack slots at calls so the GC won't move them --- src/coreclr/jit/codegen.h | 1 + src/coreclr/jit/codegenlinear.cpp | 3 + src/coreclr/jit/codegenwasm.cpp | 31 +++++++++ src/coreclr/jit/compiler.cpp | 5 ++ src/coreclr/jit/compiler.h | 2 + src/coreclr/jit/compmemkind.h | 1 + src/coreclr/jit/compphases.h | 1 + src/coreclr/jit/fgwasm.cpp | 106 ++++++++++++++++++++++++++++++ src/coreclr/jit/gentree.cpp | 3 + src/coreclr/jit/gtlist.h | 1 + src/coreclr/jit/regallocwasm.cpp | 6 ++ 11 files changed, 160 insertions(+) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 5a933a511f1d4a..421f3455fe4ac7 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -217,6 +217,7 @@ class CodeGen final : public CodeGenInterface ArrayStack* wasmControlFlowStack = nullptr; unsigned wasmCursor = 0; unsigned wasmExtraControlFlowDepth = 0; + unsigned wasmSpillRefIndex = 0; unsigned findTargetDepth(BasicBlock* target); void WasmProduceReg(GenTree* node); regNumber GetMultiUseOperandReg(GenTree* operand); diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 00b5ccf2740ac2..f69d63db476afa 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -459,6 +459,9 @@ void CodeGen::genCodeForBlock(BasicBlock* block) #endif #ifdef TARGET_WASM + // Reset spill counter at block boundaries. + wasmSpillRefIndex = 0; + // genHomeRegisterParams can generate arbitrary amounts of code on Wasm, so // we have moved it out of the prolog to the first basic block in order to // work around the restriction that the prolog can only be one insGroup. diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 0bf6b32b214fa5..4fd271c46ad8fe 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -858,6 +858,7 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) break; case GT_CALL: + wasmSpillRefIndex = 0; genCall(treeNode->AsCall()); break; @@ -911,6 +912,36 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) GetEmitter()->emitIns(INS_unreachable); break; + case GT_WASM_SPILL_REF: + { + const unsigned splashZoneVar = m_compiler->m_wasmSpillSlots->at(0); + noway_assert(wasmSpillRefIndex + 1 < m_compiler->m_wasmSpillSlots->size()); + const unsigned spillTargetVar = m_compiler->m_wasmSpillSlots->at(wasmSpillRefIndex + 1); + unsigned splashZoneLclIndex; + bool FPBased; + + { + LclVarDsc* varDsc = m_compiler->lvaGetDesc(splashZoneVar); + assert(genIsValidReg(varDsc->GetRegNum())); + splashZoneLclIndex = WasmRegToIndex(varDsc->GetRegNum()); + + GetEmitter()->emitIns_I(INS_local_tee, EA_PTRSIZE, splashZoneLclIndex); + } + + GetEmitter()->emitIns_I(INS_local_get, EA_PTRSIZE, GetFramePointerRegIndex()); + m_compiler->lvaFrameAddress(spillTargetVar, &FPBased); + GetEmitter()->emitIns_S(INS_I_const, EA_PTRSIZE, spillTargetVar, 0); + GetEmitter()->emitIns(INS_I_add); + + GetEmitter()->emitIns_I(INS_local_get, EA_PTRSIZE, splashZoneLclIndex); + + instruction ins = ins_Store(TYP_BYREF); + GetEmitter()->emitIns_I(ins, EA_PTRSIZE, 0); + + wasmSpillRefIndex++; + break; + } + case GT_CATCH_ARG: genCatchArg(treeNode); break; diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index a786daa966f13b..9dabdf18b98e36 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5053,6 +5053,11 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // keep the Virtual IP updated. // DoPhase(this, PHASE_WASM_VIRTUAL_IP, &Compiler::fgWasmVirtualIP); + + // Ensure that any refs or byrefs live at call sites are spilled + // to pinned stack slots so the objects aren't moved. + // + DoPhase(this, PHASE_WASM_SPILL_REFS, &Compiler::WasmSpillRefs); #endif FinalizeEH(); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index b25ce125243fa1..d0c9964704b85f 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4244,6 +4244,7 @@ class Compiler unsigned lvaWasmVirtualIP = BAD_VAR_NUM; // Wasm virtual IP slot unsigned lvaWasmFunctionIndex = BAD_VAR_NUM; // Wasm function index slot unsigned lvaWasmResumeIP = BAD_VAR_NUM; // Wasm catch resumption IP slot + jitstd::vector* m_wasmSpillSlots = nullptr; #endif // defined(TARGET_WASM) unsigned lvaInlinedPInvokeFrameVar = BAD_VAR_NUM; // variable representing the InlinedCallFrame @@ -6757,6 +6758,7 @@ class Compiler PhaseStatus fgWasmControlFlow(); PhaseStatus fgWasmTransformSccs(); PhaseStatus fgWasmVirtualIP(); + PhaseStatus WasmSpillRefs(); #ifdef DEBUG void fgDumpWasmControlFlow(); void fgDumpWasmControlFlowDot(); diff --git a/src/coreclr/jit/compmemkind.h b/src/coreclr/jit/compmemkind.h index b6cc1a7cfc251b..f35cf6ce0e0a64 100644 --- a/src/coreclr/jit/compmemkind.h +++ b/src/coreclr/jit/compmemkind.h @@ -73,6 +73,7 @@ CompMemKindMacro(RangeCheckCloning) CompMemKindMacro(WasmSccTransform) CompMemKindMacro(WasmCfgLowering) CompMemKindMacro(WasmEH) +CompMemKindMacro(WasmSpillRefs) //clang-format on #undef CompMemKindMacro diff --git a/src/coreclr/jit/compphases.h b/src/coreclr/jit/compphases.h index d49a6cafbb063d..8c7b83b60974d6 100644 --- a/src/coreclr/jit/compphases.h +++ b/src/coreclr/jit/compphases.h @@ -130,6 +130,7 @@ CompPhaseNameMacro(PHASE_DFS_BLOCKS_WASM, "Wasm remove unreachable bl CompPhaseNameMacro(PHASE_WASM_EH_FLOW, "Wasm eh control flow", false, -1, false) CompPhaseNameMacro(PHASE_WASM_TRANSFORM_SCCS, "Wasm transform sccs", false, -1, false) CompPhaseNameMacro(PHASE_WASM_CONTROL_FLOW, "Wasm control flow", false, -1, false) +CompPhaseNameMacro(PHASE_WASM_SPILL_REFS, "Wasm spill refs", false, -1, false) CompPhaseNameMacro(PHASE_WASM_VIRTUAL_IP, "Wasm virtual IP", false, -1, false) CompPhaseNameMacro(PHASE_ASYNC, "Transform async", false, -1, true) diff --git a/src/coreclr/jit/fgwasm.cpp b/src/coreclr/jit/fgwasm.cpp index b417a598e934f1..a87a816ae3b275 100644 --- a/src/coreclr/jit/fgwasm.cpp +++ b/src/coreclr/jit/fgwasm.cpp @@ -1671,6 +1671,112 @@ PhaseStatus Compiler::fgWasmControlFlow() return PhaseStatus::MODIFIED_EVERYTHING; } +PhaseStatus Compiler::WasmSpillRefs() +{ + bool anyChanges = false; + + size_t highWaterMark = 0; + jitstd::vector defs(getAllocator(CMK_WasmSpillRefs)); + + for (BasicBlock* const block : Blocks()) + { + // LIR edges cannot span blocks, so we can safely clear the list of live values per-block + defs.clear(); + + for (GenTree* tree : LIR::AsRange(block)) + { + if (tree->IsCall()) + { + highWaterMark = std::max(highWaterMark, defs.size()); + + if (defs.size()) + { + JITDUMP("Spilling %d live ref(s) for call\n", defs.size()); + DISPNODE(tree); + for (GenTree* def : defs) + { + JITDUMP(" "); + DISPNODE(def); + GenTreeUnOp* spill = gtNewOperNode(GT_WASM_SPILL_REF, def->TypeGet(), def); + LIR::Use use; + noway_assert(LIR::AsRange(block).TryGetUse(def, &use)); + use.ReplaceWith(spill); + LIR::AsRange(block).InsertAfter(def, spill); + anyChanges = true; + } + + defs.clear(); + } + } + + // FIXME: Should this happen before the spilling of the live defs list? + // I think the answer is no, because live defs being passed as arguments to the current call + // are not guaranteed to ever end up in memory where the GC can see them unless we spill + // them. If we can somehow guarantee that all callees will spill their ref parameters + // immediately, we could do this before the block above. + // Remove used nodes from defs list, they're no longer meaningfully 'live'. + tree->VisitOperands([&defs](GenTree* op) { + if (!op->IsValue()) + return GenTree::VisitResult::Continue; + if (!op->TypeIs(TYP_REF, TYP_BYREF)) + return GenTree::VisitResult::Continue; + + for (size_t i = defs.size(); i > 0; i--) + { + if (op == defs[i - 1]) + { + defs[i - 1] = defs[defs.size() - 1]; + defs.erase(defs.begin() + (defs.size() - 1), defs.end()); + break; + } + } + + return GenTree::VisitResult::Continue; + }); + + if (tree->IsValue() && tree->TypeIs(TYP_REF, TYP_BYREF) && !tree->OperIs(GT_WASM_SPILL_REF)) + { + // TODO: Can we skip this for GT_LCL_VAR when it lives in memory? Or is it possible + // that the LCL_VAR has been modified since it was loaded onto the Wasm stack? + defs.push_back(tree); + } + } + } + + JITDUMP("High water mark for refs was %d\n", highWaterMark); + if (highWaterMark == 0) + return PhaseStatus::MODIFIED_NOTHING; + + m_wasmSpillSlots = new (this, CMK_WasmSpillRefs) jitstd::vector(highWaterMark + 1, 0, getAllocator(CMK_WasmSpillRefs)); + + // Allocate a temporary wasm local to use as a scratch slot during spills + { + const unsigned varNum = lvaGrabTemp(false DEBUGARG("WasmSpillRefs splash zone")); + LclVarDsc* const varDsc = lvaGetDesc(varNum); + // HACK: Make this TYP_I_IMPL because if we make it a REF or BYREF that may block enregistration + varDsc->lvType = TYP_I_IMPL; + varDsc->lvHasExplicitInit = true; + varDsc->lvImplicitlyReferenced = true; + // If we don't make this var tracked, regalloc will crash when allocating a register for it + varDsc->lvTracked = true; + m_wasmSpillSlots->at(0) = varNum; + } + + // Allocate N temporary refs to act as GC-visible storage for all spills that occur during execution + for (size_t i = 0; i < highWaterMark; i++) + { + const unsigned varNum = lvaGrabTemp(false DEBUGARG("WasmSpillRefs spill slot")); + LclVarDsc* const varDsc = lvaGetDesc(varNum); + varDsc->lvType = TYP_BYREF; + varDsc->lvPinned = true; + varDsc->lvImplicitlyReferenced = true; + lvaSetVarDoNotEnregister(varNum, DoNotEnregisterReason::WasmGCVisibility); + m_wasmSpillSlots->at(i + 1) = varNum; + } + + return PhaseStatus::MODIFIED_EVERYTHING; +} + #ifdef DEBUG //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 3af72048706405..9c35c9b077a637 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -11767,6 +11767,9 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_RETURN_SUSPEND: case GT_PATCHPOINT_FORCED: case GT_NONLOCAL_JMP: +#ifdef TARGET_WASM + case GT_WASM_SPILL_REF: +#endif m_edge = &m_node->AsUnOp()->gtOp1; assert(*m_edge != nullptr); m_advance = &GenTreeUseEdgeIterator::Terminate; diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index b293df525ed695..b6c5e498113623 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -357,6 +357,7 @@ GTNODE(SWIFT_ERROR_RET , GenTreeOp ,0,1,GTK_BINOP|GTK_NOVALUE) // Retu GTNODE(WASM_JEXCEPT , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // Special jump for Wasm exception handling GTNODE(WASM_THROW_REF , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // Wasm rethrow host exception (exception is an implicit operand) +GTNODE(WASM_SPILL_REF , GenTreeOp ,0,0,GTK_UNOP|DBK_NOTHIR) //----------------------------------------------------------------------------- // Nodes used by Lower to generate a closer CPU representation of other nodes diff --git a/src/coreclr/jit/regallocwasm.cpp b/src/coreclr/jit/regallocwasm.cpp index e6e4a0969f3ba0..f9d1d353901800 100644 --- a/src/coreclr/jit/regallocwasm.cpp +++ b/src/coreclr/jit/regallocwasm.cpp @@ -190,6 +190,12 @@ void WasmRegAlloc::IdentifyCandidates() varIsRegCandidate = false; } + // HACK: Ensure that we always enregister the splash zone, even if we are not enregistering other locals + if (m_compiler->m_wasmSpillSlots && m_compiler->m_wasmSpillSlots->size() && m_compiler->m_wasmSpillSlots->at(0) == lclNum) + { + varIsRegCandidate = true; + } + if (varIsRegCandidate) { JITDUMP("RA candidate: V%02u\n", lclNum); From 02375bb06b03049d5e3f43a5dc0ff56ff98c04c0 Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Fri, 5 Jun 2026 16:21:20 -0700 Subject: [PATCH 2/6] Fix a bounds check --- src/coreclr/jit/compiler.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index d0c9964704b85f..1fa701c9f600c3 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4647,6 +4647,7 @@ class Compiler unsigned lvaTrackedIndexToLclNum(unsigned trackedIndex) { assert(trackedIndex < lvaTrackedCount); + assert(trackedIndex < lvaTrackedToVarNumSize); unsigned lclNum = lvaTrackedToVarNum[trackedIndex]; assert(lclNum < lvaCount); return lclNum; From 9a2f3da5bd18a6352771ca4297664ab74595e485 Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Sat, 6 Jun 2026 07:55:40 -0700 Subject: [PATCH 3/6] Set the spill slots as must-init --- src/coreclr/jit/fgwasm.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/fgwasm.cpp b/src/coreclr/jit/fgwasm.cpp index a87a816ae3b275..fe2c69a6e47ead 100644 --- a/src/coreclr/jit/fgwasm.cpp +++ b/src/coreclr/jit/fgwasm.cpp @@ -1757,7 +1757,7 @@ PhaseStatus Compiler::WasmSpillRefs() varDsc->lvType = TYP_I_IMPL; varDsc->lvHasExplicitInit = true; varDsc->lvImplicitlyReferenced = true; - // If we don't make this var tracked, regalloc will crash when allocating a register for it + // HACK: If we don't make this var tracked, regalloc will crash when allocating a register for it varDsc->lvTracked = true; m_wasmSpillSlots->at(0) = varNum; } @@ -1770,6 +1770,7 @@ PhaseStatus Compiler::WasmSpillRefs() varDsc->lvType = TYP_BYREF; varDsc->lvPinned = true; varDsc->lvImplicitlyReferenced = true; + varDsc->lvMustInit = true; lvaSetVarDoNotEnregister(varNum, DoNotEnregisterReason::WasmGCVisibility); m_wasmSpillSlots->at(i + 1) = varNum; } From 1e668919b2d8494f17cf126e6ebfda493551a5c9 Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Mon, 8 Jun 2026 11:04:50 -0700 Subject: [PATCH 4/6] Cleanup --- src/coreclr/jit/codegen.h | 1 + src/coreclr/jit/codegenwasm.cpp | 65 +++++++++++++++++++------------- src/coreclr/jit/compiler.h | 1 + src/coreclr/jit/fgwasm.cpp | 48 +++++++++++++++++------ src/coreclr/jit/regallocwasm.cpp | 5 ++- 5 files changed, 80 insertions(+), 40 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 421f3455fe4ac7..74a7b9d1c80f7c 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -795,6 +795,7 @@ class CodeGen final : public CodeGenInterface #if defined(TARGET_WASM) void genCodeForConstant(GenTree* treeNode); void genCatchArg(GenTree* treeNode); + void genWasmSpillRef(GenTree* treeNode); #endif #if defined(TARGET_X86) diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 4fd271c46ad8fe..569236ab3cf04a 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -913,34 +913,8 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) break; case GT_WASM_SPILL_REF: - { - const unsigned splashZoneVar = m_compiler->m_wasmSpillSlots->at(0); - noway_assert(wasmSpillRefIndex + 1 < m_compiler->m_wasmSpillSlots->size()); - const unsigned spillTargetVar = m_compiler->m_wasmSpillSlots->at(wasmSpillRefIndex + 1); - unsigned splashZoneLclIndex; - bool FPBased; - - { - LclVarDsc* varDsc = m_compiler->lvaGetDesc(splashZoneVar); - assert(genIsValidReg(varDsc->GetRegNum())); - splashZoneLclIndex = WasmRegToIndex(varDsc->GetRegNum()); - - GetEmitter()->emitIns_I(INS_local_tee, EA_PTRSIZE, splashZoneLclIndex); - } - - GetEmitter()->emitIns_I(INS_local_get, EA_PTRSIZE, GetFramePointerRegIndex()); - m_compiler->lvaFrameAddress(spillTargetVar, &FPBased); - GetEmitter()->emitIns_S(INS_I_const, EA_PTRSIZE, spillTargetVar, 0); - GetEmitter()->emitIns(INS_I_add); - - GetEmitter()->emitIns_I(INS_local_get, EA_PTRSIZE, splashZoneLclIndex); - - instruction ins = ins_Store(TYP_BYREF); - GetEmitter()->emitIns_I(ins, EA_PTRSIZE, 0); - - wasmSpillRefIndex++; + genWasmSpillRef(treeNode); break; - } case GT_CATCH_ARG: genCatchArg(treeNode); @@ -961,6 +935,43 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) } } +//------------------------------------------------------------------------ +// genWasmSpillRef: spill a ref/byref to one of the reserved spill slots on the +// stack so the GC can see it +// +// Arguments: +// treeNode - WASM_SPILL_REF node +// +void CodeGen::genWasmSpillRef(GenTree* treeNode) +{ + const unsigned splashZoneVar = m_compiler->lvaWasmSplashZone; + noway_assert(wasmSpillRefIndex < m_compiler->m_wasmSpillSlots->size()); + const unsigned spillTargetVar = m_compiler->m_wasmSpillSlots->at(wasmSpillRefIndex); + unsigned splashZoneLclIndex; + bool FPBased; + + { + LclVarDsc* varDsc = m_compiler->lvaGetDesc(splashZoneVar); + assert(genIsValidReg(varDsc->GetRegNum())); + splashZoneLclIndex = WasmRegToIndex(varDsc->GetRegNum()); + + GetEmitter()->emitIns_I(INS_local_tee, EA_PTRSIZE, splashZoneLclIndex); + } + + GetEmitter()->emitIns_I(INS_local_get, EA_PTRSIZE, GetFramePointerRegIndex()); + m_compiler->lvaFrameAddress(spillTargetVar, &FPBased); + // TODO-WASM: Emit this offset as the memarg of the store, below. + GetEmitter()->emitIns_S(INS_I_const, EA_PTRSIZE, spillTargetVar, 0); + GetEmitter()->emitIns(INS_I_add); + + GetEmitter()->emitIns_I(INS_local_get, EA_PTRSIZE, splashZoneLclIndex); + + instruction ins = ins_Store(TYP_BYREF); + GetEmitter()->emitIns_I(ins, EA_PTRSIZE, 0); + + wasmSpillRefIndex++; +} + //------------------------------------------------------------------------ // genCodeForJTrue: emit Wasm br_if // diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 1fa701c9f600c3..e10cc94533bc7c 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4244,6 +4244,7 @@ class Compiler unsigned lvaWasmVirtualIP = BAD_VAR_NUM; // Wasm virtual IP slot unsigned lvaWasmFunctionIndex = BAD_VAR_NUM; // Wasm function index slot unsigned lvaWasmResumeIP = BAD_VAR_NUM; // Wasm catch resumption IP slot + unsigned lvaWasmSplashZone = BAD_VAR_NUM; // Temporary local used for spilling GC refs on Wasm jitstd::vector* m_wasmSpillSlots = nullptr; #endif // defined(TARGET_WASM) diff --git a/src/coreclr/jit/fgwasm.cpp b/src/coreclr/jit/fgwasm.cpp index fe2c69a6e47ead..97e0d7f0e54d58 100644 --- a/src/coreclr/jit/fgwasm.cpp +++ b/src/coreclr/jit/fgwasm.cpp @@ -1689,9 +1689,11 @@ PhaseStatus Compiler::WasmSpillRefs() { highWaterMark = std::max(highWaterMark, defs.size()); + // For any ref/byref values live at the point of a call, spill them into pinned slots + // on the stack where the GC can see them so it won't move them. if (defs.size()) { - JITDUMP("Spilling %d live ref(s) for call\n", defs.size()); + JITDUMP("Spilling %zu live ref(s) for call\n", defs.size()); DISPNODE(tree); for (GenTree* def : defs) { @@ -1702,6 +1704,12 @@ PhaseStatus Compiler::WasmSpillRefs() noway_assert(LIR::AsRange(block).TryGetUse(def, &use)); use.ReplaceWith(spill); LIR::AsRange(block).InsertAfter(def, spill); + if (def->gtLIRFlags & LIR::Flags::MultiplyUsed) + { + JITDUMP("Transferring multiply-used flag from [%06u] to [%06u] for spill\n", Compiler::dspTreeID(def), Compiler::dspTreeID(spill)); + def->gtLIRFlags &= ~LIR::Flags::MultiplyUsed; + spill->gtLIRFlags |= LIR::Flags::MultiplyUsed; + } anyChanges = true; } @@ -1714,6 +1722,7 @@ PhaseStatus Compiler::WasmSpillRefs() // are not guaranteed to ever end up in memory where the GC can see them unless we spill // them. If we can somehow guarantee that all callees will spill their ref parameters // immediately, we could do this before the block above. + // Remove used nodes from defs list, they're no longer meaningfully 'live'. tree->VisitOperands([&defs](GenTree* op) { if (!op->IsValue()) @@ -1726,7 +1735,7 @@ PhaseStatus Compiler::WasmSpillRefs() if (op == defs[i - 1]) { defs[i - 1] = defs[defs.size() - 1]; - defs.erase(defs.begin() + (defs.size() - 1), defs.end()); + defs.pop_back(); break; } } @@ -1734,20 +1743,37 @@ PhaseStatus Compiler::WasmSpillRefs() return GenTree::VisitResult::Continue; }); - if (tree->IsValue() && tree->TypeIs(TYP_REF, TYP_BYREF) && !tree->OperIs(GT_WASM_SPILL_REF)) + // We only care about used values, and invariant nodes can't produce movable GC refs, so skip + // nodes appropriately + if (!tree->IsValue() || tree->IsUnusedValue() || tree->IsInvariant()) + { + continue; + } + + // If a value is just a GT_LCL_VAR that isn't address-exposed, by construction we ensure that + // it won't be mutated between its def (here) and its use (the call that would produce a spill) + // and we won't need to spill it. + if (tree->OperIs(GT_LCL_VAR)) + { + LclVarDsc* dsc = lvaGetDesc(tree->AsLclVarCommon()); + if (!dsc->IsAddressExposed()) + continue; + } + + // We have a ref sourced from something like a call result or an indirection that hasn't been + // spilled yet, so record it for potential spilling at the next call. + if (tree->TypeIs(TYP_REF, TYP_BYREF) && !tree->OperIs(GT_WASM_SPILL_REF)) { - // TODO: Can we skip this for GT_LCL_VAR when it lives in memory? Or is it possible - // that the LCL_VAR has been modified since it was loaded onto the Wasm stack? defs.push_back(tree); } } } - JITDUMP("High water mark for refs was %d\n", highWaterMark); - if (highWaterMark == 0) + JITDUMP("High water mark for refs was %zu\n", highWaterMark); + if (!anyChanges) return PhaseStatus::MODIFIED_NOTHING; - m_wasmSpillSlots = new (this, CMK_WasmSpillRefs) jitstd::vector(highWaterMark + 1, 0, getAllocator(CMK_WasmSpillRefs)); + m_wasmSpillSlots = new (this, CMK_WasmSpillRefs) jitstd::vector(highWaterMark, 0, getAllocator(CMK_WasmSpillRefs)); // Allocate a temporary wasm local to use as a scratch slot during spills { @@ -1758,8 +1784,8 @@ PhaseStatus Compiler::WasmSpillRefs() varDsc->lvHasExplicitInit = true; varDsc->lvImplicitlyReferenced = true; // HACK: If we don't make this var tracked, regalloc will crash when allocating a register for it - varDsc->lvTracked = true; - m_wasmSpillSlots->at(0) = varNum; + // varDsc->lvTracked = true; + lvaWasmSplashZone = varNum; } // Allocate N temporary refs to act as GC-visible storage for all spills that occur during execution @@ -1772,7 +1798,7 @@ PhaseStatus Compiler::WasmSpillRefs() varDsc->lvImplicitlyReferenced = true; varDsc->lvMustInit = true; lvaSetVarDoNotEnregister(varNum, DoNotEnregisterReason::WasmGCVisibility); - m_wasmSpillSlots->at(i + 1) = varNum; + m_wasmSpillSlots->at(i) = varNum; } return PhaseStatus::MODIFIED_EVERYTHING; diff --git a/src/coreclr/jit/regallocwasm.cpp b/src/coreclr/jit/regallocwasm.cpp index f9d1d353901800..631b940a8c73b8 100644 --- a/src/coreclr/jit/regallocwasm.cpp +++ b/src/coreclr/jit/regallocwasm.cpp @@ -190,8 +190,9 @@ void WasmRegAlloc::IdentifyCandidates() varIsRegCandidate = false; } - // HACK: Ensure that we always enregister the splash zone, even if we are not enregistering other locals - if (m_compiler->m_wasmSpillSlots && m_compiler->m_wasmSpillSlots->size() && m_compiler->m_wasmSpillSlots->at(0) == lclNum) + // HACK: Ensure that we always enregister the splash zone, even if we are not enregistering other locals. + // Spilling codegen won't work unless the splash zone lives in a native wasm local. + if ((m_compiler->lvaWasmSplashZone != BAD_VAR_NUM) && (lclNum == m_compiler->lvaWasmSplashZone)) { varIsRegCandidate = true; } From 5dd8f82e1dbed2390188d05df9b5d10f3456014b Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Mon, 8 Jun 2026 11:18:57 -0700 Subject: [PATCH 5/6] Improve GetLclVarNameInfo for Wasm --- src/coreclr/jit/gentree.cpp | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9c35c9b077a637..9c0f3833e1b657 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -13129,6 +13129,20 @@ void Compiler::gtGetLclVarNameInfo(unsigned lclNum, const char** ilKindOut, cons const char* ilName = nullptr; unsigned ilNum = compMap2ILvarNum(lclNum); +#if TARGET_WASM + int wasmSpillSlotIndex = -1; + if (m_wasmSpillSlots) + { + for (unsigned i = 0; i < m_wasmSpillSlots->size(); i++) + { + if (m_wasmSpillSlots->at(i) == lclNum) + { + wasmSpillSlotIndex = i; + break; + } + } + } +#endif if (ilNum == (unsigned)ICorDebugInfo::RETBUF_ILNUM) { @@ -13203,6 +13217,23 @@ void Compiler::gtGetLclVarNameInfo(unsigned lclNum, const char** ilKindOut, cons { ilName = "SP"; } + else if (lclNum == lvaWasmVirtualIP) + { + ilName = "VirtualIP"; + } + else if (lclNum == lvaWasmFunctionIndex) + { + ilName = "FuncIndex"; + } + else if (lclNum == lvaWasmSplashZone) + { + ilName = "SplashZone"; + } + else if (wasmSpillSlotIndex > -1) + { + ilKind = "spill"; + ilNum = wasmSpillSlotIndex; + } #endif // defined(TARGET_WASM) else { From c90720a8dcdb2c65669b681019eabee073249431 Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Mon, 8 Jun 2026 11:45:25 -0700 Subject: [PATCH 6/6] Use regular local var loads and stores instead of a dedicated wasm spill node Remove the splash zone concept --- src/coreclr/jit/codegen.h | 2 - src/coreclr/jit/codegenlinear.cpp | 3 -- src/coreclr/jit/codegenwasm.cpp | 42 ------------------ src/coreclr/jit/compiler.h | 1 - src/coreclr/jit/fgwasm.cpp | 72 +++++++++++++++---------------- src/coreclr/jit/gentree.cpp | 7 --- src/coreclr/jit/gtlist.h | 1 - src/coreclr/jit/regallocwasm.cpp | 7 --- 8 files changed, 35 insertions(+), 100 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 74a7b9d1c80f7c..5a933a511f1d4a 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -217,7 +217,6 @@ class CodeGen final : public CodeGenInterface ArrayStack* wasmControlFlowStack = nullptr; unsigned wasmCursor = 0; unsigned wasmExtraControlFlowDepth = 0; - unsigned wasmSpillRefIndex = 0; unsigned findTargetDepth(BasicBlock* target); void WasmProduceReg(GenTree* node); regNumber GetMultiUseOperandReg(GenTree* operand); @@ -795,7 +794,6 @@ class CodeGen final : public CodeGenInterface #if defined(TARGET_WASM) void genCodeForConstant(GenTree* treeNode); void genCatchArg(GenTree* treeNode); - void genWasmSpillRef(GenTree* treeNode); #endif #if defined(TARGET_X86) diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index f69d63db476afa..00b5ccf2740ac2 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -459,9 +459,6 @@ void CodeGen::genCodeForBlock(BasicBlock* block) #endif #ifdef TARGET_WASM - // Reset spill counter at block boundaries. - wasmSpillRefIndex = 0; - // genHomeRegisterParams can generate arbitrary amounts of code on Wasm, so // we have moved it out of the prolog to the first basic block in order to // work around the restriction that the prolog can only be one insGroup. diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 569236ab3cf04a..0bf6b32b214fa5 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -858,7 +858,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) break; case GT_CALL: - wasmSpillRefIndex = 0; genCall(treeNode->AsCall()); break; @@ -912,10 +911,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) GetEmitter()->emitIns(INS_unreachable); break; - case GT_WASM_SPILL_REF: - genWasmSpillRef(treeNode); - break; - case GT_CATCH_ARG: genCatchArg(treeNode); break; @@ -935,43 +930,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) } } -//------------------------------------------------------------------------ -// genWasmSpillRef: spill a ref/byref to one of the reserved spill slots on the -// stack so the GC can see it -// -// Arguments: -// treeNode - WASM_SPILL_REF node -// -void CodeGen::genWasmSpillRef(GenTree* treeNode) -{ - const unsigned splashZoneVar = m_compiler->lvaWasmSplashZone; - noway_assert(wasmSpillRefIndex < m_compiler->m_wasmSpillSlots->size()); - const unsigned spillTargetVar = m_compiler->m_wasmSpillSlots->at(wasmSpillRefIndex); - unsigned splashZoneLclIndex; - bool FPBased; - - { - LclVarDsc* varDsc = m_compiler->lvaGetDesc(splashZoneVar); - assert(genIsValidReg(varDsc->GetRegNum())); - splashZoneLclIndex = WasmRegToIndex(varDsc->GetRegNum()); - - GetEmitter()->emitIns_I(INS_local_tee, EA_PTRSIZE, splashZoneLclIndex); - } - - GetEmitter()->emitIns_I(INS_local_get, EA_PTRSIZE, GetFramePointerRegIndex()); - m_compiler->lvaFrameAddress(spillTargetVar, &FPBased); - // TODO-WASM: Emit this offset as the memarg of the store, below. - GetEmitter()->emitIns_S(INS_I_const, EA_PTRSIZE, spillTargetVar, 0); - GetEmitter()->emitIns(INS_I_add); - - GetEmitter()->emitIns_I(INS_local_get, EA_PTRSIZE, splashZoneLclIndex); - - instruction ins = ins_Store(TYP_BYREF); - GetEmitter()->emitIns_I(ins, EA_PTRSIZE, 0); - - wasmSpillRefIndex++; -} - //------------------------------------------------------------------------ // genCodeForJTrue: emit Wasm br_if // diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index e10cc94533bc7c..1fa701c9f600c3 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4244,7 +4244,6 @@ class Compiler unsigned lvaWasmVirtualIP = BAD_VAR_NUM; // Wasm virtual IP slot unsigned lvaWasmFunctionIndex = BAD_VAR_NUM; // Wasm function index slot unsigned lvaWasmResumeIP = BAD_VAR_NUM; // Wasm catch resumption IP slot - unsigned lvaWasmSplashZone = BAD_VAR_NUM; // Temporary local used for spilling GC refs on Wasm jitstd::vector* m_wasmSpillSlots = nullptr; #endif // defined(TARGET_WASM) diff --git a/src/coreclr/jit/fgwasm.cpp b/src/coreclr/jit/fgwasm.cpp index 97e0d7f0e54d58..31cf06b3ee36bc 100644 --- a/src/coreclr/jit/fgwasm.cpp +++ b/src/coreclr/jit/fgwasm.cpp @@ -1693,22 +1693,51 @@ PhaseStatus Compiler::WasmSpillRefs() // on the stack where the GC can see them so it won't move them. if (defs.size()) { + anyChanges = true; + + if (!m_wasmSpillSlots) + { + m_wasmSpillSlots = new (this, CMK_WasmSpillRefs) jitstd::vector(getAllocator(CMK_WasmSpillRefs)); + } + + unsigned spillSlotIndex = 0; JITDUMP("Spilling %zu live ref(s) for call\n", defs.size()); DISPNODE(tree); for (GenTree* def : defs) { JITDUMP(" "); DISPNODE(def); - GenTreeUnOp* spill = gtNewOperNode(GT_WASM_SPILL_REF, def->TypeGet(), def); + + unsigned spillSlot; + if (spillSlotIndex < m_wasmSpillSlots->size()) + { + spillSlot = m_wasmSpillSlots->at(spillSlotIndex); + } + else + { + spillSlot = lvaGrabTemp(false DEBUGARG("WasmSpillRefs spill slot")); + LclVarDsc* const varDsc = lvaGetDesc(spillSlot); + varDsc->lvType = TYP_BYREF; + varDsc->lvPinned = true; + varDsc->lvImplicitlyReferenced = true; + varDsc->lvMustInit = true; + lvaSetVarDoNotEnregister(spillSlot, DoNotEnregisterReason::WasmGCVisibility); + m_wasmSpillSlots->push_back(spillSlot); + } + spillSlotIndex++; + + GenTreeLclVar *spill = gtNewStoreLclVarNode(spillSlot, def); + GenTreeLclVar *reload = gtNewLclVarNode(spillSlot); LIR::Use use; noway_assert(LIR::AsRange(block).TryGetUse(def, &use)); - use.ReplaceWith(spill); + use.ReplaceWith(reload); LIR::AsRange(block).InsertAfter(def, spill); + LIR::AsRange(block).InsertAfter(spill, reload); if (def->gtLIRFlags & LIR::Flags::MultiplyUsed) { - JITDUMP("Transferring multiply-used flag from [%06u] to [%06u] for spill\n", Compiler::dspTreeID(def), Compiler::dspTreeID(spill)); + JITDUMP("Transferring multiply-used flag from [%06u] to [%06u] for spill\n", Compiler::dspTreeID(def), Compiler::dspTreeID(reload)); def->gtLIRFlags &= ~LIR::Flags::MultiplyUsed; - spill->gtLIRFlags |= LIR::Flags::MultiplyUsed; + reload->gtLIRFlags |= LIR::Flags::MultiplyUsed; } anyChanges = true; } @@ -1762,7 +1791,7 @@ PhaseStatus Compiler::WasmSpillRefs() // We have a ref sourced from something like a call result or an indirection that hasn't been // spilled yet, so record it for potential spilling at the next call. - if (tree->TypeIs(TYP_REF, TYP_BYREF) && !tree->OperIs(GT_WASM_SPILL_REF)) + if (tree->TypeIs(TYP_REF, TYP_BYREF)) { defs.push_back(tree); } @@ -1770,38 +1799,7 @@ PhaseStatus Compiler::WasmSpillRefs() } JITDUMP("High water mark for refs was %zu\n", highWaterMark); - if (!anyChanges) - return PhaseStatus::MODIFIED_NOTHING; - - m_wasmSpillSlots = new (this, CMK_WasmSpillRefs) jitstd::vector(highWaterMark, 0, getAllocator(CMK_WasmSpillRefs)); - - // Allocate a temporary wasm local to use as a scratch slot during spills - { - const unsigned varNum = lvaGrabTemp(false DEBUGARG("WasmSpillRefs splash zone")); - LclVarDsc* const varDsc = lvaGetDesc(varNum); - // HACK: Make this TYP_I_IMPL because if we make it a REF or BYREF that may block enregistration - varDsc->lvType = TYP_I_IMPL; - varDsc->lvHasExplicitInit = true; - varDsc->lvImplicitlyReferenced = true; - // HACK: If we don't make this var tracked, regalloc will crash when allocating a register for it - // varDsc->lvTracked = true; - lvaWasmSplashZone = varNum; - } - - // Allocate N temporary refs to act as GC-visible storage for all spills that occur during execution - for (size_t i = 0; i < highWaterMark; i++) - { - const unsigned varNum = lvaGrabTemp(false DEBUGARG("WasmSpillRefs spill slot")); - LclVarDsc* const varDsc = lvaGetDesc(varNum); - varDsc->lvType = TYP_BYREF; - varDsc->lvPinned = true; - varDsc->lvImplicitlyReferenced = true; - varDsc->lvMustInit = true; - lvaSetVarDoNotEnregister(varNum, DoNotEnregisterReason::WasmGCVisibility); - m_wasmSpillSlots->at(i) = varNum; - } - - return PhaseStatus::MODIFIED_EVERYTHING; + return anyChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; } #ifdef DEBUG diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9c0f3833e1b657..8ec3b939405363 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -11767,9 +11767,6 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_RETURN_SUSPEND: case GT_PATCHPOINT_FORCED: case GT_NONLOCAL_JMP: -#ifdef TARGET_WASM - case GT_WASM_SPILL_REF: -#endif m_edge = &m_node->AsUnOp()->gtOp1; assert(*m_edge != nullptr); m_advance = &GenTreeUseEdgeIterator::Terminate; @@ -13225,10 +13222,6 @@ void Compiler::gtGetLclVarNameInfo(unsigned lclNum, const char** ilKindOut, cons { ilName = "FuncIndex"; } - else if (lclNum == lvaWasmSplashZone) - { - ilName = "SplashZone"; - } else if (wasmSpillSlotIndex > -1) { ilKind = "spill"; diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index b6c5e498113623..b293df525ed695 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -357,7 +357,6 @@ GTNODE(SWIFT_ERROR_RET , GenTreeOp ,0,1,GTK_BINOP|GTK_NOVALUE) // Retu GTNODE(WASM_JEXCEPT , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // Special jump for Wasm exception handling GTNODE(WASM_THROW_REF , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // Wasm rethrow host exception (exception is an implicit operand) -GTNODE(WASM_SPILL_REF , GenTreeOp ,0,0,GTK_UNOP|DBK_NOTHIR) //----------------------------------------------------------------------------- // Nodes used by Lower to generate a closer CPU representation of other nodes diff --git a/src/coreclr/jit/regallocwasm.cpp b/src/coreclr/jit/regallocwasm.cpp index 631b940a8c73b8..e6e4a0969f3ba0 100644 --- a/src/coreclr/jit/regallocwasm.cpp +++ b/src/coreclr/jit/regallocwasm.cpp @@ -190,13 +190,6 @@ void WasmRegAlloc::IdentifyCandidates() varIsRegCandidate = false; } - // HACK: Ensure that we always enregister the splash zone, even if we are not enregistering other locals. - // Spilling codegen won't work unless the splash zone lives in a native wasm local. - if ((m_compiler->lvaWasmSplashZone != BAD_VAR_NUM) && (lclNum == m_compiler->lvaWasmSplashZone)) - { - varIsRegCandidate = true; - } - if (varIsRegCandidate) { JITDUMP("RA candidate: V%02u\n", lclNum);