Skip to content

Commit 872e969

Browse files
EgorBoEgorBo
committed
Inline CORINFO_HELP_ARRADDR_ST helper call
1 parent 2fcafaf commit 872e969

13 files changed

Lines changed: 151 additions & 25 deletions

File tree

‎src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,6 @@ private static object ChkCastAny_NoCacheLookup(void* toTypeHnd, object obj)
5353
return obj;
5454
}
5555

56-
[MethodImpl(MethodImplOptions.InternalCall)]
57-
private static extern void WriteBarrier(ref object? dst, object? obj);
58-
5956
// IsInstanceOf test used for unusual cases (naked type parameters, variant generic types)
6057
// Unlike the IsInstanceOfInterface and IsInstanceOfClass functions,
6158
// this test must deal with all kinds of type tests
@@ -454,7 +451,7 @@ private static void StelemRef(object?[] array, nint index, object? obj)
454451
goto notExactMatch;
455452

456453
doWrite:
457-
WriteBarrier(ref element, obj);
454+
RuntimeHelpers.WriteBarrierUnchecked(ref element, obj);
458455
return;
459456

460457
assigningNull:
@@ -475,7 +472,7 @@ private static void StelemRef_Helper(ref object? element, void* elementType, obj
475472
CastResult result = CastCache.TryGet(s_table!, (nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType);
476473
if (result == CastResult.CanCast)
477474
{
478-
WriteBarrier(ref element, obj);
475+
RuntimeHelpers.WriteBarrierUnchecked(ref element, obj);
479476
return;
480477
}
481478

@@ -493,7 +490,7 @@ private static void StelemRef_Helper_NoCacheLookup(ref object? element, void* el
493490
ThrowArrayMismatchException();
494491
}
495492

496-
WriteBarrier(ref element, obj2);
493+
RuntimeHelpers.WriteBarrierUnchecked(ref element, obj2);
497494
}
498495

499496
[DebuggerHidden]

‎src/coreclr/jit/compiler.h‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5129,6 +5129,30 @@ class Compiler
51295129
GenTree* dereferencedAddress,
51305130
InlArgInfo* inlArgInfo);
51315131

5132+
typedef JitHashTable<CORINFO_METHOD_HANDLE, JitPtrKeyFuncs<struct CORINFO_METHOD_STRUCT_>, CORINFO_METHOD_HANDLE> HelperToManagedMap;
5133+
HelperToManagedMap* m_helperToManagedMap = nullptr;
5134+
5135+
public:
5136+
HelperToManagedMap* GetHelperToManagedMap()
5137+
{
5138+
if (m_helperToManagedMap == nullptr)
5139+
{
5140+
m_helperToManagedMap = new (getAllocator()) HelperToManagedMap(getAllocator());
5141+
}
5142+
return m_helperToManagedMap;
5143+
}
5144+
bool HelperToManagedMapLookup(CORINFO_METHOD_HANDLE helperCallHnd, CORINFO_METHOD_HANDLE* userCallHnd)
5145+
{
5146+
if (m_helperToManagedMap == nullptr)
5147+
{
5148+
return false;
5149+
}
5150+
bool found = m_helperToManagedMap->Lookup(helperCallHnd, userCallHnd);
5151+
return found;
5152+
}
5153+
private:
5154+
5155+
void impConvertToUserCallAndMarkForInlining(GenTreeCall* call);
51325156
void impMarkInlineCandidate(GenTree* call,
51335157
CORINFO_CONTEXT_HANDLE exactContextHnd,
51345158
bool exactContextNeedsRuntimeLookup,

‎src/coreclr/jit/gentree.cpp‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2343,6 +2343,34 @@ bool GenTreeCall::IsHelperCall(Compiler* compiler, unsigned helper) const
23432343
return IsHelperCall(compiler->eeFindHelper(helper));
23442344
}
23452345

2346+
//-------------------------------------------------------------------------
2347+
// IsHelperCallOrUserEquivalent: Determine if this GT_CALL node is a specific helper call
2348+
// or its CT_USER equivalent.
2349+
//
2350+
// Arguments:
2351+
// compiler - the compiler instance so that we can call eeFindHelper
2352+
//
2353+
// Return Value:
2354+
// Returns true if this GT_CALL node is a call to the specified helper.
2355+
//
2356+
bool GenTreeCall::IsHelperCallOrUserEquivalent(Compiler* compiler, unsigned helper) const
2357+
{
2358+
CORINFO_METHOD_HANDLE helperCallHnd = Compiler::eeFindHelper(helper);
2359+
if (IsHelperCall())
2360+
{
2361+
return helperCallHnd == gtCallMethHnd;
2362+
}
2363+
2364+
CORINFO_METHOD_HANDLE userCallHnd = NO_METHOD_HANDLE;
2365+
2366+
auto mmap = compiler->impInlineRoot()->m_helperToManagedMap;
2367+
auto cc = mmap != nullptr ? mmap->GetCount() : -1;
2368+
if (ISMETHOD("Test"))
2369+
printf("1");
2370+
2371+
return compiler->impInlineRoot()->HelperToManagedMapLookup(helperCallHnd, &userCallHnd);
2372+
}
2373+
23462374
//-------------------------------------------------------------------------
23472375
// IsRuntimeLookupHelperCall: Determine if this GT_CALL node represents a runtime lookup helper call.
23482376
//
@@ -12846,6 +12874,9 @@ void Compiler::gtDispTree(GenTree* tree,
1284612874
case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant:
1284712875
printf(" isKnownConst");
1284812876
break;
12877+
case NI_System_Runtime_CompilerServices_RuntimeHelpers_WriteBarrierUnchecked:
12878+
printf(" writeBarrierUnchecked");
12879+
break;
1284912880
#if defined(FEATURE_SIMD)
1285012881
case NI_SIMD_UpperRestore:
1285112882
printf(" simdUpperRestore");

‎src/coreclr/jit/gentree.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5715,6 +5715,8 @@ struct GenTreeCall final : public GenTree
57155715

57165716
bool IsHelperCall(Compiler* compiler, unsigned helper) const;
57175717

5718+
bool IsHelperCallOrUserEquivalent(Compiler* compiler, unsigned helper) const;
5719+
57185720
bool IsRuntimeLookupHelperCall(Compiler* compiler) const;
57195721

57205722
bool IsSpecialIntrinsic(Compiler* compiler, NamedIntrinsic ni) const;

‎src/coreclr/jit/importer.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7364,7 +7364,12 @@ void Compiler::impImportBlockCode(BasicBlock* block)
73647364
// The array helper takes a native int for array length.
73657365
// So if we have an int, explicitly extend it to be a native int.
73667366
index = impImplicitIorI4Cast(index, TYP_I_IMPL);
7367-
op1 = gtNewHelperCallNode(CORINFO_HELP_ARRADDR_ST, TYP_VOID, array, index, value);
7367+
7368+
GenTreeCall* call = gtNewHelperCallNode(CORINFO_HELP_ARRADDR_ST, TYP_VOID, array, index, value);
7369+
INDEBUG(call->gtRawILOffset = opcodeOffs);
7370+
impConvertToUserCallAndMarkForInlining(call);
7371+
op1 = call;
7372+
73687373
goto SPILL_APPEND;
73697374
}
73707375

‎src/coreclr/jit/importercalls.cpp‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3425,6 +3425,8 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd,
34253425
// This one is just `return true/false`
34263426
case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant:
34273427

3428+
case NI_System_Runtime_CompilerServices_RuntimeHelpers_WriteBarrierUnchecked:
3429+
34283430
// Not expanding this can lead to noticeable allocations in T0
34293431
case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan:
34303432

@@ -3657,6 +3659,14 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd,
36573659
break;
36583660
}
36593661

3662+
case NI_System_Runtime_CompilerServices_RuntimeHelpers_WriteBarrierUnchecked:
3663+
{
3664+
GenTree* val = impPopStack().val;
3665+
GenTree* dst = impPopStack().val;
3666+
retNode = gtNewStoreIndNode(TYP_REF, dst, val, GTF_IND_TGT_HEAP);
3667+
break;
3668+
}
3669+
36603670
case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant:
36613671
{
36623672
GenTree* op1 = impPopStack().val;
@@ -7702,6 +7712,47 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call,
77027712
call->AddGDVCandidateInfo(this, pInfo);
77037713
}
77047714

7715+
//------------------------------------------------------------------------
7716+
// impConvertToUserCallAndMarkForInlining: convert a helper call to a user call
7717+
// and mark it for inlining. This is used for helper calls that are
7718+
// known to be backed by a user method that can be inlined.
7719+
//
7720+
// Arguments:
7721+
// call - the helper call to convert
7722+
//
7723+
void Compiler::impConvertToUserCallAndMarkForInlining(GenTreeCall* call)
7724+
{
7725+
assert(call->IsHelperCall());
7726+
7727+
if (!opts.OptEnabled(CLFLG_INLINING))
7728+
{
7729+
return;
7730+
}
7731+
7732+
CORINFO_METHOD_HANDLE helperCallHnd = call->gtCallMethHnd;
7733+
CORINFO_METHOD_HANDLE managedCallHnd = NO_METHOD_HANDLE;
7734+
CORINFO_CONST_LOOKUP pNativeEntrypoint = {};
7735+
info.compCompHnd->getHelperFtn(eeGetHelperNum(helperCallHnd), &pNativeEntrypoint, &managedCallHnd);
7736+
7737+
if (managedCallHnd != NO_METHOD_HANDLE)
7738+
{
7739+
call->gtCallMethHnd = managedCallHnd;
7740+
call->gtCallType = CT_USER_FUNC;
7741+
7742+
CORINFO_CALL_INFO hCallInfo = {};
7743+
hCallInfo.hMethod = managedCallHnd;
7744+
hCallInfo.methodFlags = info.compCompHnd->getMethodAttribs(hCallInfo.hMethod);
7745+
impMarkInlineCandidate(call, nullptr, false, &hCallInfo, compInlineContext);
7746+
7747+
if (ISMETHOD("Test"))
7748+
printf("");
7749+
7750+
impInlineRoot()->GetHelperToManagedMap()->Set(helperCallHnd, managedCallHnd, HelperToManagedMap::Overwrite);
7751+
JITDUMP("Converting helperCall [%06u] to user call [%s] and marking for inlining\n", dspTreeID(call),
7752+
eeGetMethodFullName(managedCallHnd));
7753+
}
7754+
}
7755+
77057756
//------------------------------------------------------------------------
77067757
// impMarkInlineCandidate: determine if this call can be subsequently inlined
77077758
//
@@ -10618,7 +10669,14 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
1061810669
{
1061910670
namespaceName += 8;
1062010671

10621-
if (strcmp(namespaceName, "CompilerServices") == 0)
10672+
if (strcmp(className, "TypeCast") == 0)
10673+
{
10674+
if (strcmp(methodName, "WriteBarrierUnchecked") == 0)
10675+
{
10676+
result = NI_System_Runtime_CompilerServices_RuntimeHelpers_WriteBarrierUnchecked;
10677+
}
10678+
}
10679+
else if (strcmp(namespaceName, "CompilerServices") == 0)
1062210680
{
1062310681
if (strcmp(className, "RuntimeHelpers") == 0)
1062410682
{
@@ -10634,6 +10692,10 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
1063410692
{
1063510693
result = NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant;
1063610694
}
10695+
else if (strcmp(methodName, "WriteBarrierUnchecked") == 0)
10696+
{
10697+
result = NI_System_Runtime_CompilerServices_RuntimeHelpers_WriteBarrierUnchecked;
10698+
}
1063710699
else if (strcmp(methodName, "IsReferenceOrContainsReferences") == 0)
1063810700
{
1063910701
result =

‎src/coreclr/jit/morph.cpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6487,13 +6487,22 @@ GenTree* Compiler::fgMorphCall(GenTreeCall* call)
64876487

64886488
// Morph stelem.ref helper call to store a null value, into a store into an array without the helper.
64896489
// This needs to be done after the arguments are morphed to ensure constant propagation has already taken place.
6490-
if (opts.OptimizationEnabled() && call->IsHelperCall(this, CORINFO_HELP_ARRADDR_ST))
6490+
if (opts.OptimizationEnabled() && call->IsHelperCallOrUserEquivalent(this, CORINFO_HELP_ARRADDR_ST))
64916491
{
64926492
assert(call->gtArgs.CountArgs() == 3);
64936493
GenTree* arr = call->gtArgs.GetArgByIndex(0)->GetNode();
64946494
GenTree* index = call->gtArgs.GetArgByIndex(1)->GetNode();
64956495
GenTree* value = call->gtArgs.GetArgByIndex(2)->GetNode();
64966496

6497+
if (!call->IsHelperCall())
6498+
{
6499+
// Convert back to helper call if it wasn't inlined.
6500+
// Currently, only helper calls are eligible to be direct calls if the target has reached
6501+
// its final tier. TODO: remove this workaround and convert this user call to direct as well.
6502+
call->gtCallMethHnd = eeFindHelper(CORINFO_HELP_ARRADDR_ST);
6503+
call->gtCallType = CT_HELPER;
6504+
}
6505+
64976506
if (gtCanSkipCovariantStoreCheck(value, arr))
64986507
{
64996508
// Either or both of the array and index arguments may have been spilled to temps by `fgMorphArgs`. Copy

‎src/coreclr/jit/namedintrinsiclist.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ enum NamedIntrinsic : unsigned short
123123
NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant,
124124
NI_System_Runtime_CompilerServices_RuntimeHelpers_IsReferenceOrContainsReferences,
125125
NI_System_Runtime_CompilerServices_RuntimeHelpers_GetMethodTable,
126+
NI_System_Runtime_CompilerServices_RuntimeHelpers_WriteBarrierUnchecked,
126127

127128
NI_System_Runtime_CompilerServices_AsyncHelpers_AsyncSuspend,
128129
NI_System_Runtime_CompilerServices_AsyncHelpers_Await,

‎src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,10 +148,6 @@ internal static int RhEndNoGCRegion()
148148
internal static extern unsafe object RhpNewFastMisalign(MethodTable * pEEType);
149149
#endif // FEATURE_64BIT_ALIGNMENT
150150

151-
[RuntimeImport(RuntimeLibrary, "RhpAssignRef")]
152-
[MethodImpl(MethodImplOptions.InternalCall)]
153-
internal static extern unsafe void RhpAssignRef(ref object? address, object? obj);
154-
155151
[MethodImplAttribute(MethodImplOptions.InternalCall)]
156152
[RuntimeImport(RuntimeLibrary, "RhpGcSafeZeroMemory")]
157153
internal static extern unsafe ref byte RhpGcSafeZeroMemory(ref byte dmem, nuint size);

‎src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -802,8 +802,8 @@ public static unsafe void StelemRef(object?[] array, nint index, object? obj)
802802
if (elementType != obj.GetMethodTable())
803803
goto notExactMatch;
804804

805-
doWrite:
806-
InternalCalls.RhpAssignRef(ref element, obj);
805+
doWrite:
806+
WriteBarrierUnchecked(ref element, obj);
807807
return;
808808

809809
assigningNull:
@@ -826,7 +826,7 @@ private static unsafe void StelemRef_Helper(ref object? element, MethodTable* el
826826
CastResult result = s_castCache.TryGet((nuint)obj.GetMethodTable() + (int)AssignmentVariation.BoxedSource, (nuint)elementType);
827827
if (result == CastResult.CanCast)
828828
{
829-
InternalCalls.RhpAssignRef(ref element, obj);
829+
WriteBarrierUnchecked(ref element, obj);
830830
return;
831831
}
832832

@@ -843,7 +843,7 @@ private static unsafe void StelemRef_Helper_NoCacheLookup(ref object? element, M
843843
throw elementType->GetClasslibException(ExceptionIDs.ArrayTypeMismatch);
844844
}
845845

846-
InternalCalls.RhpAssignRef(ref element, obj);
846+
WriteBarrierUnchecked(ref element, obj);
847847
}
848848

849849
private static unsafe object IsInstanceOfArray(MethodTable* pTargetType, object obj)
@@ -1275,5 +1275,8 @@ private static unsafe object CheckCastAny_NoCacheLookup(MethodTable* pTargetType
12751275

12761276
return obj;
12771277
}
1278+
1279+
[Intrinsic]
1280+
internal static void WriteBarrierUnchecked(ref object? dst, object? obj) => dst = obj;
12781281
}
12791282
}

0 commit comments

Comments
 (0)